From 5b3d2c7fce11a62d0df0105e9cc7290f50f2a0f8 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 17 Nov 2026 02:21:10 +0000 Subject: [PATCH] Extract user input preparation into `codex-user-input` (#61949) ## What changed Move local attachment snapshotting and conversion from `UserInput` to model request content out of `codex-protocol` into a dedicated `codex-user-input` crate. Update core, the TUI, and the queue extension to use the new crate, replacing the `ResponseInputItem` conversion methods with `from_user_input`. Move the associated attachment and message conversion tests alongside the implementation, preserving coverage for image positions, media labels, serialization, and error placeholders. GitOrigin-RevId: 77d0f94a4cd4c2edc008b9e2e63a989995aab058 --- .github/CODEOWNERS | 1 + codex-rs/Cargo.lock | 15 + codex-rs/Cargo.toml | 2 + .../app-server-exports-stable.json.zst | Bin 172469 -> 172421 bytes codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/compact.rs | 9 +- codex-rs/core/src/session/file_audio_tests.rs | 14 +- codex-rs/core/src/session/mod.rs | 6 +- .../core/src/session/turn_start_mcp_tests.rs | 1 + .../core/tests/suite/client_websockets.rs | 14 +- codex-rs/ext/queue/Cargo.toml | 1 + codex-rs/ext/queue/src/service.rs | 2 +- codex-rs/protocol/src/lib.rs | 1 - codex-rs/protocol/src/models.rs | 695 +----------------- codex-rs/protocol/src/user_input.rs | 5 +- codex-rs/tui/Cargo.toml | 1 + .../tui/src/chatwidget/image_submission.rs | 2 +- codex-rs/user-input/BUILD.bazel | 6 + codex-rs/user-input/Cargo.toml | 23 + codex-rs/user-input/src/lib.rs | 13 + .../src/local_media.rs | 24 +- .../src/local_media_tests.rs | 4 +- codex-rs/user-input/src/message.rs | 160 ++++ codex-rs/user-input/src/message_tests.rs | 537 ++++++++++++++ 24 files changed, 814 insertions(+), 723 deletions(-) create mode 100644 codex-rs/user-input/BUILD.bazel create mode 100644 codex-rs/user-input/Cargo.toml create mode 100644 codex-rs/user-input/src/lib.rs rename codex-rs/{protocol => user-input}/src/local_media.rs (83%) rename codex-rs/{protocol => user-input}/src/local_media_tests.rs (97%) create mode 100644 codex-rs/user-input/src/message.rs create mode 100644 codex-rs/user-input/src/message_tests.rs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 61addafe3b..5df4f68445 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,6 +6,7 @@ /codex-rs/exec-server-protocol/ @openai/codex-core-agent-team /codex-rs/ext/extension-api/ @openai/codex-core-agent-team /codex-rs/prompts/ @openai/codex-core-agent-team +/codex-rs/user-input/ @openai/codex-core-agent-team /codex-rs/utils/path-uri/ @openai/codex-core-agent-team # Keep macOS AKV signing changes reviewed by Codex maintainers. diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5b3393b593..759a75a9d4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3093,6 +3093,7 @@ dependencies = [ "codex-test-binary-support", "codex-thread-store", "codex-tools", + "codex-user-input", "codex-utils-absolute-path", "codex-utils-audio", "codex-utils-cache", @@ -4765,6 +4766,7 @@ dependencies = [ "codex-protocol", "codex-state", "codex-thread-store", + "codex-user-input", "codex-utils-absolute-path", "core_test_support", "pretty_assertions", @@ -5294,6 +5296,7 @@ dependencies = [ "codex-state", "codex-terminal-detection", "codex-uds", + "codex-user-input", "codex-utils-absolute-path", "codex-utils-approval-presets", "codex-utils-cargo-bin", @@ -5391,6 +5394,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "codex-user-input" +version = "0.0.0" +dependencies = [ + "anyhow", + "codex-protocol", + "codex-utils-image", + "pretty_assertions", + "serde_json", + "tempfile", +] + [[package]] name = "codex-user-verification" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index c7f4180ddc..4bccad35f3 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -118,6 +118,7 @@ members = [ "otel-trace-websocket", "tcp-tunnel", "tui", + "user-input", "user-verification", "tools", "v8-poc", @@ -339,6 +340,7 @@ codex-utils-sleep-inhibitor = { path = "utils/sleep-inhibitor" } codex-utils-stream-parser = { path = "utils/stream-parser" } codex-utils-string = { path = "utils/string" } codex-utils-template = { path = "utils/template" } +codex-user-input = { path = "user-input" } codex-user-verification = { path = "user-verification" } codex-workload-identity = { path = "workload-identity" } codex-windows-sandbox = { path = "windows-sandbox-rs" } diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index b968a3613ae3615881fdfbc31f894a53ec41f93c..e8d10d7451bfcbde6d32aeeafe851c5248816163 100644 GIT binary patch delta 6440 zcmV+@8Q12u!wQAN3V?(Gv;vxle~i5V_MTZI0Cn41dy8&La~Wv4_z5;r{)h~oNFjLc zhT2v*uP*)ne+U6j080Q;0MLS#vj63*@J^&M=buG5&hSEtaV}((8P@dWe>Y`oMfCGy zf%539BHQpECM(#yu0MoWq=Tea_hdllF*qpT#V7O8gUwqSZ|J3Rea5<)e{SV+E0f7s zLXuxxlz-poR3(}z=%S3e1)b;N&due`d)AA90Dj^(kB&Kue@4I#>(49-oES>>YNc58{Yc+Zy!AM)kP_3GEiSkdy;ZutrtG0oO8 zdasNKE|8eqom= zI-H7%E0da2@zfsA!|~2VCx@*|pFTE*@~Gqel1Y*bn*mcG5I`UxmmG=$2!A*<8Ir8~ zBT(?*SbPeHi@-2I1Q-wm0zm+cAP9mW2!bLAf*>dZfe;XZPNy|?zQoLb!b$G^9>dl( zvskQ5AQ)#fTQh;YjIxdhuM$litP+AK2hap+uP<`ObBg)ZBp645eqj|aM7ss<>@R+z zqtGuA-?}2JPJws68krLWS$})dB}HZ^l`P5%UafBuR~S>2+;A+6{ng%p95MkR(Sa_? zGsa_x8|wStz}<)7`Px)J1DM|J*vo#R5FPYDPxL%jn4e+ApaeP(oLutQmj>H-N+{wb4?uEI&17RktB|T~tcwNOp_&@{xJ6@{!@TE)P@^i?bj~QE zY0!a(Zv-Hm?z8XkXMc;lr;PM#K!Q~xR{X6YkXe!Lw#Cm<8eJwu#=%aJriJ)4V7E*# zwA1j4W#K!aHl8cx#Hb`@xyOZ}&4!9sr?e5e`0Mg*@;5jO&1->G`9JC50tQTi*hXji zZJ4qNM?`ToM%=v=EmI=(gAft)NU2f8`vm~Z%v+P>`L;ctV1H$yLUd$+w(j*{9Vi2Z z%1Rgmo|V!OfR2=)>snuh06zT6Wov^Mg-oWzvHl!&u+(SQbBT`KRKwc*W#-Y0!SUiXBL1>kDGKoHMDeT2@ zelUwTdeV?PqJJ7PBj^p6>orp8GY3JiIlMR;rDph;#O0Bv*(!!8JtJGYh1}M^4o3`y zJYEuC2%gn(q`ah#IQ;lK!zuY=*^YW&Cs&0tM}ICYFVEWzu<&6ED9u!B6b8pk zbAAlegnu)lqt;#Cx|793L1du;Pd6qC7Z5U;AQduE&}%P1Q5Xd#yR3)0w8$D~6{y-j zILnKTrobf%G>RHa!v8IIpaDDy>m?swM7)AfU!tUwKQ$MoA;V@$4BbLLqHxk{4oi*P z4`C|k;D6vt_6{KHC({GCfE$o2m)!)7%*W@00`SC4`T+WM%PywM9u^X!uWybqWZ1_q z4sD(wyVFW~=NY`*1rB{7`&Ozp|1e0X@Yu0GXZE_6)RZt1i(p@xS}bYi*E1*UKUU4`%m@MP(cbSC*47(8$j3|@uIxh2KY?EY zA>6tf9F$L-`#sF5Sk8xXN+lZ=-#Jbx@kf9rPOF`e;F4?h67HY@eX0-}f3}GIRUK4! z@$HN(i>8t+kc1espXwu%tC26M^fQCGdvbkN<#?%2#P6f zR9sz=VNQ@S8eJr?2lR*z{j346V>UR26rAHZ*f$6MA}lWeT5W{Y=%fOX*&Ij&0!Ae; zsLwWNJu&^M5}Ef0wVk*|D4&#XVuDOruphZP0zZ0Tc99|{7VwJuq!5~w0yt7T%75TJ zE4;j%o$fbQLt-LsUcpOyRg?_9A1Y(|Y08OZgnKbHThTZjv(O5a9)9h)rC~i=qz9H~ zHgI4ykFE*$ra#VFWv^fCad=VAoLP0t79H(f54(R@-3r&Jo( zC+$AZ!|P^vK;p0fhX?}!hwd=6wtuuhs4WD)(SExV_iXqjme?yy7vdj@>>DEuT5@v7 z--cXXSjgd5%lmj^@0ClyDw=)M!V3??)kj7W*N$QF&uVDYW^KR_#N~f@Xih?z zbJs&X2v-8`{^P^P)~OE|#(xiD$2W+C*3AS6T2vJz80YHi3yLD+uQDEJ&5$(`bTl^# zP3VQ|@SyFb|NA$)c^hYRDh;fjwS^Z;IZOJNtn_V)-Dbm);MfDLT*De@-VA4_w0RMf z4?3OB4Wf4fS9GX=>q*&kEa2D9bRIgvH-BI3M@@zp%E3g~_)M660Bk6yR8Pz|+2-?6XAf@QXfu<9bAHkZ zMp1>iRiZ!|skM$Ee3wW)pxg$wj=)haMPx7%ZfimSW4719NYFz(V&!f=9_(0eA10y- z6!b&G;)o)x`Cus5_5(EKWhy+M#9N3IT%L6Y$OLa!HwIiR@PFa*?bB@mQgo_ng2Zy#T(K~Y0f zU@^hk&7wQ8J`h0U zt40DAsM6LCB}L2OUWYvu?Z(a5lXMtdBiq?WBZsS0C4XA>lT-!+IP*@tw`#(C)5iD( z_)i)+PJsp=UbNjG>~&cJupcR#vI1JBJ!MU&t+ZmbrR_4=!r3q!6wz=YpHo#x(EiK^ zTr;arE7wrZc_eTD;?LQO+xozQlgcc3Q4LNyQVcbEU0aBo^~RDjvkP!YU8eqwi4Xy)|90wbTd2B2_%_Dx!BSAR5WBRgu6!XgYMgnE>Bqa~y`^fkYR z5LT!HWR&tpG)@24r#1*!rIXi^_s+mR(o3e=QAJ_~kVe7I!jWMqFRHymt11Lg0T&?7 z1M8T$(PqhARg=vQ%ZZ%2)9F`l?!G=j9#>~Dx_m&>lRWYu__@-shFq-rC<8dgf}IZz zcYj{hh-fxB}9TDK%!t+Q^2a;2C3B&1K6DXOREuP<{5BF|uOCjUd`9 z_o7knL%KS74!q4?26v@z=c4EVeGBrZegjR((MGmGpwLVJ zwmPR$Fva|7tq|#B<|DFnfTn7MR(~7_M*u{t8$JJqzRyjHODowjC_&^C6Vu!92{z$y zMfD(TGB|bZdOr&34BRNXP4aN3gMvX)9+E&|Gtdj0jz3yRKrytCu5Wy@grv#};j_|5 zF$hXz1U)17or(G!>*0A-d;NkeWeTpT&@V0vb zc?`)NK=RE;pg1gjH*F%y42zR$+b>w&vqrjqXk}q;trK!p_ z+>;A~F?yj-jrn-M^i^du1U1JovO=s8&z9T%qxKGjjwq+gQGb74HO&hVLq0GCC@NK7@@Y~~+8Tfi3B0y+?ePf*DsaNdsoYB1pnq)Uoc?#mL zxFrSK5OiC{uz!}2$w;KIcaA|cLlfq)J>NZ)9bki1af(w;(fIv7hJK~TU*a~Nla1F|-YvVan zc?b|6QFq&Y`yzJ+!Np0f#uG`c-1BXY6QSyREaS?`U9Cl#2A>ZHeozp zP`pXpp!99jscYIZn+%6k7qNhH*+3YVHv+AeMArdRHZEJg;pys?9iaydkT>#Bdcqrl z@ox+QT2xp~mJ2NPp;Q;vy7?h*Iw)o@VrO{3KuT9iWafSe^N#shmVAmFVJ##_MZ!Vj zL4S!PIM-16&Gyzv4_V{~g~R{2!>|_CAP1(I61EDXXqmxd^w1|N`P@}OJiImGpcoaQ zC~5@jOqU8S2u{#MbhbeW=j7Dk#f&7flL??;CSZ}N7!$_`6MVJr(c&X{^l=Q5#{;^c z1pR$6Rm>wG6H>YJ2v5Y;^&MP3O^fxdOMg#2oajtWlHXPyqy458KJJj-Ke6)b7dYC=Qvgitx#V2UqJLG= zQd=(w*z)2(d{zah^AmkffUN!%8%Yo84e|J~#>`{jf&ubc9>qIgpm*{>{S*u^nPR-3 znG_pg{*SUzyp~Xj>V!NH6e?#zp%(iw6vG-SvOg+i+oNSkD3L%_Bz|9tSFJ~kIyE&l z3p*^St;+;lX{~WDr{vi^R3Io})PK%GeW|!&T;Q~)XgVdnqq3io!)b_ag^#UQ|CA2Y z7DL~D<9?AzQJ+&N2q&h1pxI)0fDki0$zkF@Sk^AiqQ4$yMZsLMZtx!6wEF zPfcR+PJ*f2r^YX#y9XF|w;-3E_u1iP({B)Ue1jkb&G(=xtxX|tGnD}!vNJ3xWTX?(B z_Wn0ZBIqkx&zCTp^fvkC1%J96`2;I8h#(#f{HwtjqjK^RqvIx`jtN1hn7#iU9)sDOIV=1zISv(T5>qLlG$XpJ=Q z(D`{-8W@)t^iP41?kStkd5maX^s z&mXrbePF=~$Pc>X%%E7F;rI=Y;4FNv{%Fsv<$vTDb5wdv>>4TNh>7lS zTei*QD&-oc8`!As8Zu^-pB6CK%vF2(vhtr3yarJ?Q34_z8+pxMxa`Y7OZI-qS~Yi- zoZXGzlx*W!chGm}L43z6~ZHly50;LH6 zKxuAtV{LP1eQoR^1Xn;0jZ7qER}OsS{;?L5-2W;TH-DUWjG&^d-?dmMXC=*4=@L{m zPa80VKSn{yyl0^$XRh5DozKi*rKMVbs3LC(bI_7P$EO%_9sDA)8s(yrvCoB+FL3ZI4dwwu!HvNh8wYn1u~% zw4ZU94S)I*gFUXcOUe9oQ*R~-eQq~1JKFn*2KHO$0yCDyRwLUziW<{+ET*KZWE8vlaO-0JS?k`1_Zn;IE=Xe0$QNb%zP0 CmL5m| delta 6491 zcmV-h8KmZg!wR*-3V?(Gv;vxlf6TrBR^M790F&GFG)<+ldAH|udH@dAp5h`do zpxjmy=I#Ig{|_Zm08Ic>0R12aF?IjTS>c69b0@alE0J%k>>Ae{%wmIfE=2 zWSUIp5`z3HM)~-ypz1?2MZrj;eo;?r+_}lziO-4w08pRmomY=b7K(dnZV|$-?L4Pk z`TTlER^|Gu;$NA)myVV9eZ-9!Fn!vrI;VvK!&`qpZiA=(!sz6bU1C_a@WB^~7?(pW z$2pm3EGU84%)1;a7f;+tf7zQvxg{o(h(=xEcyne}1epU>-z(=a$J#1N&H67aSj3!l z8ad7PGK#OPdZLS)S1C_|em$Qz=f{KlQs(t#XH-nZL{Hs7x)~rG9JpNuFashlEp3@3 z(o{#u|2l?UiqLo@Dy~jy&c#!2w3hJ?MyJPZhX1J34Uf2!fyt1VTUrI-SP|J~`!7%Z zO;M`KDp7oK&Y+mCPNcTv(CehPi=QS4t_l6Dz$Qh9?qERze@5Ivvscy-@A2?}4X{DC zn~C=Q-eX^#qU5XX^yaskq%{Wzy*n1lUY<*mWg!I32&#Zp3-%(7=aIrAL%IMy9>Z( zQd_=;O0Z&SV3o6KPXs(-5^A%x zrVhv{@l+-%H7R*4Tl9aC6q97`uxCo`GTtu(Ff(tPf28%l?N|mYBN<{T12ng7w2vY zNXvifgX!&jytfUrx~-_L|xS?eX}rr-Fq{FhOmv!ypCZn2CQP zA-KE)LjItC(UxK#l{|Bhcs7R@N4aW-j|sV&JWW?IL?Ip7qb%gM{&hGq5b}6Q{vx-b zDf5z)+&+8-CztK)_~GXuta5W%;e_BKyCe- zkv(eNWvx3|OcXj6GI+W%QJ{d3@ySvl69rp)0g6IVG1&!ms0)y+aaN0}y}&FlU^E3T ze^Iz-)L26N-*N{F;0bvz$^IhZ6%F+zN`#DTb73kOHdA7V4f%)y*1hJib>#j|s)7!P zzGMjiS%0}6Isx2(>~PslP+>lP9Tb44%%l&XU$^XHs`RjskbQk~6hnr+{NnKD39=V$ z@;lF9YF!}J7qSadS^vWz!FZ2-@Hw-Cf4Ce6f?eL{dxvi)>lGxBDtW_bVHZMBfz|@# zT-EJ(6#7v)1we8ljhd$E-==W#?EDQ-mcR=PCe zEUSJtsV!CSm}6q%C_-rI4sL13vrD1#oe=<1T){IAluMr9vkvzNaQT!I*`m;T$#!-BzLm89&e?2k4g_UdQa<)KYK`rNQDH+lr*T6lnnitEQ701hYAm2&9cle_*hXZ6J2y z{!^s{?+?5?aVDYsUf#xGnX;e&a+M7~I>7A0a!xE&j{BsFJ1bXkkZUP}H*1V>Q!Mf^nR#~>C)VZWd!kJYP>|VI%WY2l^z7!Gna;?c#$58pBbI(-mTKz zFQB4_6&|Dq<)U;AE9X&j zUT9Lr4Ie^HUWgG8eVNP!a?IltBSZm1U(2|4Nel^k%ZE|tghTwy8mq?W=g`V(ai2}1lsiK7=XC^4~ET2*u=T(VFlqz zVB3FunA|$`VG`qq>G%ebaJre0lNOaZ4g6eveSk%g`BxbapfY5Q1Rl+e!V$V8b(ml) z>Hp>iH*bR(ol4`WXKn6dDQEfqCHr%mqPp2|B+%{kCA(V#f6d#`*?DA+1?7WLr*re; zUFC`nl~Be~wvGjSy$umDmd^+zdPQ?b+;HCr5d`~?0O~eQKm;Q85vTGq5osK=ap)hS z9XBbKB$2PweI(x}PHym@w#FejY@va0I7)G?%STWw30M!8s0=jRVew!>MbVnZA1rJ& z{um2z72{fxe_HGI`)zr*4-x?sl3)Uoj6DisNayme_2b2@sB>kF{Lj9Z^pP2-q}B^) zWId*bPJqo<;Zf5!hVmp4wo(MX$Rj5ae#K+fOB0kA5}t=bV7 zE49`Uym!R_hY4+9>j*^U>4^+Rf_Y6iJcfH6j06tEe63WJLF96}?f@C#c69@;zyhCHzWv>=w!>JoF_rnG5XDbKfXAu} zHT1vN-%G_^GePucRfsZK)Y#cOCl^Q3V(NqY5={dP%Dbm3@b>Yg8Wc4&tZNTGe{-oF{Rd;KSA zf<)_4p869}^+qWF=E+=c_z9j#FjdXeY_2CLyJQ)+&L}99ScVY1XE?bJW&p4*^Sm|x zJVKeLm|LId7MDxmF)mZ2v_6sdH!YW5LVmyPe?8l-Dz8+_ewfN&WM|&Fy;WbCZ&w(9i*&@>y?&gZ7D#eKsdXAgKioAtT|P+GTI-pfNP}sw2B() zf0Z8zE?+!P@nW<-uxq3`7QB{(lMbAL>aT0N;AXv%TV@t*E;ZWl9NuLg=)`3v&WNT< z`}l@_G_5)=qZLNTjQiPHndtI6 zW+Uj#WaiXEKH$3U$xP9NL%bIpkO;gSe=}}0r7tH6-kI<-Gr`-K^#d+XmI9avz%cx%8d+#hw57SxJHrOl3NqOb_2givgIH(EC44!ZU$31Kld!1q!} zMsxAMG_^rGQF=5j5uOHCb9xV`O!S_g3jdxG@S<{58Tg{ZfeMt)kiphV=vhG;JEWlM#Ot2 z#Xo<1azaQT!xfNnOsO$jtc`3)3Z5~BZ7y5oa0d0S4)tfh8>1F0ZiL)kxhER+4!r4f z>{S%_>oj+bzM2lCEOlAs!+lGte^=XcBWYDYyDFjVhe2bY^-)`r5aw5qtD&aqYgn!8 z>Q`qQ@IOTwk`lWRab;pI`(x9_UI0Fxt+foRV8I>?t3dmHuNKv^W4dF+#I5cw224^mXBefFd7TrT7UCRV_8%t*4Eh~b1bG6pZAAOR4J1O~jJ@hSyzsoy#=5I|toOWLy%|NKc; zJEc%eTvaOmAKeaQ9LmdBe_l^2TW6@3epVMnKj~XVKJ{~GN(UR+1|dQ-0ln2ZU4bd% zr?sluk7-`{@&WlFDY7)nt4#7i`@-PkxuR$+tdj4p^02G4<(xLm4rTr?7e-J+ZKZ-%cMl?&wsC7rd|KAT7SBBtO6xWJ?X?GNafgL zKnfwpPKTKZw4DYHE%;uzahL?CP5NHbsyIpn(aYe{76Ql(5&W{^6SY~e@*mS+x3GhM z`Rz((j9DK8CNC0he*ue$0$^2AHHTm%1>(?Dqm;L-y7`<^DZ0Vf^Ac?Pr}pDMOW?O> z9L3}`LuV%lOQkhM+0!6d1Mtwp<^o2z2eF{Fh=Q9CP*QCZ2M7r8s!xy9ZA~|-LPj*qkf0 zu4(&WW4c7^su~)_YVTE)N7hTP9I+NRP}xJfQTdvIB!Wag6LGIH@S54ecM3N7j0OgsrVM#IG?CXMX zBN=0n5evYa93PO!M3m5 zmbqC=02w6C?49F@W;ld-ESK*d`NY^@k{eovOEf_HRio?HqG-bVKgE9!CQ(Ko!=vUv zis(BwT;|3fcOcSu85HyzyAe(}YxCHgriu$cf4j$%Q#3Z(e|CCE_A>^TueG6< z_hhlCm?Qni^e#2Doz~X=Zs+jRA{rM*yIPNp$g5>>cR~C6gDi(ENkQ+$qlR(?dbrV7 ze>0rpo`Qh9dCCLTbZ!H&8ejG^rOJDW<*qE29?X6LDSef?pX-I~NV~q2iW9l0oee~Y zpm+Y%I82PxeWAJJ#>g%4&UJ^fR&dc}J3t^D$Bw8-4Qvt45w1bm>DtJ5ir@k0BdULU zWM8z+ASiPRE$c)Mlw1Gv`bA}hgh*qpe`Asp1R1EKwoz{H;Gq&k`xML-rV#iQUK=En zq>|DqARrReb>}b2JB84Co4_%-qJeq#K#l+`R+Sj@(?5`@UW|dsYSZBPBJ!KW3`(1g zKD9QQvy$*%brH+ER5lQX;*CHua&#T!+s0+<6i!zwup@NB04XjH**?5+0RP4=f1pK0 zYOkt3vq>nc;Ann5 zTvCsX*_WTw!xqGqMa1p*#tnPrWpt{z{C*PU*H+;8&{JSc?3w0V&3>y}X@FU<-5O$;@5289MQ8=1z#PMCXbhAp}f67v1P1c}k$ z?!ie|U!z-`h^CjyYuH43K`|IOajy<2KY!5*TQRNHgXEb*|2?)DsS`1jPLr=`((?Ub z&&xZ}J8t?&al8K`fBZ{G0rqs683>MN0BDG%FC2v1+n({|YTyrFV83v>^Y;EVc_OG) zw9%I^Yw2zG%?m0x@@cQoAX1ODm5WYS$@C`)q!YR&_KNq?h5rCwP92QHcqO}T*5d32 zYO`#ok&jPdUA2p27Wj8AHaX1cTSA{KQz@(AJ1lges<*o;e{Z75<9{?w(Y7Ct1-bmz zinUtp)(bEh@{XjJ*l3HiPWz${UZu3`lKVBkFJ-cynr4QQ|L+idK!pkjP`Sr}CcIwzJ3M|Lsf#iU3&rffZE_fC1zv(R^YqLlG$2#qxF(D{2<8hDo& z^l5=0?Jfygf4WcNW5J?f1VCsK+|C8hz+j%`?a<Tbq7>JY*Z?p;?StY;N ztjhQm13O~RPVmJbET~Le?TrYKZovHGw2w2}>=I?G*pP2R;NEtwrgx?e@DPE+Oj~U;P*m$$#aWx*|275RhiI2aVmGjM*xBdmoR)gSAbwTv9&kxEO6U7E%mp~4;RR<@a36(~E zA}3tJcFqZ_mQHy2#Ir9IT7#q_Asx4pJ2q64VC)pFDv=>1o&VU9XDmX#X^E+z@FJ_? z5N|!Pe9+)FPCBF$R5iqMr2=fqprryzs&uIUe`-TAI7YcH*G5#1BIOAHKxuAtV{LP1 zeQ@j{1V=a$8ktDQx*Y%H{;|+TxrkMPPoS6jVa3ir##pG?OhN(UC8&CkR%D1gN}H7V z$ZSr|EFT)gXCA$Qy-)^rDg%hTpAUo|zbPuWp8FuRjcF;?QqtQp2JhFbp!y8a2bv_H z5d%BYt5RgX%!e??=CuYGr2lM`qDBVqP4^W!fK0Rp^dL;TSep1^-DjH_$yGy|O?$d< zLp}%xc+uO0u(@@D-_fH}I?i}&Gt3JBwPSnmRV*FB>m@`Wws_#h^Q74RhjYt%oB=V_ BM5q7& diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index c485813d02..7e78f7fb3d 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -73,6 +73,7 @@ codex-state = { workspace = true } codex-terminal-detection = { workspace = true } codex-thread-store = { workspace = true } codex-tools = { workspace = true } +codex-user-input = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-audio = { workspace = true } codex-utils-cache = { workspace = true } diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 84a8b920d3..5316f85434 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -56,11 +56,12 @@ use codex_protocol::models::AgentMessageInputContent; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ContentItem; use codex_protocol::models::ContentItemKind; -use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::user_input::UserInput; use codex_rollout_trace::InferenceTraceContext; +use codex_user_input::LocalImagePreparation; +use codex_user_input::from_user_input; use codex_utils_output_truncation::approx_token_count; use tracing::error; @@ -286,7 +287,11 @@ async fn run_compact_task_inner_impl( sess.emit_turn_item_started(&turn_context, &compaction_item) .await; } - let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); + let initial_input_for_turn = from_user_input( + input, + LocalImagePreparation::Process, + &mut std::collections::HashMap::new(), + ); let mut history = sess.clone_history().await; let input_goal_ids = UserGoalUpdate::message_ids(history.raw_items()); diff --git a/codex-rs/core/src/session/file_audio_tests.rs b/codex-rs/core/src/session/file_audio_tests.rs index 5871dbf048..c9d0020f8a 100644 --- a/codex-rs/core/src/session/file_audio_tests.rs +++ b/codex-rs/core/src/session/file_audio_tests.rs @@ -4,8 +4,8 @@ use super::*; use crate::context_manager::ContextManager; use codex_protocol::models::BaseInstructions; +use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseInputItem; -use codex_protocol::user_input::UserInput; use codex_protocol_test_support::uuid_v7_item_id_generator_for_test; use codex_utils_output_truncation::TruncationPolicy; use pretty_assertions::assert_eq; @@ -13,15 +13,17 @@ use pretty_assertions::assert_eq; #[test] fn history_sums_measured_and_unknown_file_audio() { let mut history = ContextManager::new(uuid_v7_item_id_generator_for_test()); - let item = ResponseInputItem::from( - ["short", "unknown"] - .map(|file_id| UserInput::Audio { + let item = ResponseInputItem::Message { + role: "user".to_string(), + content: ["short", "unknown"] + .map(|file_id| ContentItem::InputAudio { audio: AudioReference::File { file_id: file_id.into(), }, }) - .to_vec(), - ); + .into(), + phase: None, + }; history.record_items( [&ResponseItem::from(item)], TruncationPolicy::Tokens(10_000), diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 84408e7d19..a20b470a30 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -390,8 +390,6 @@ use codex_protocol::config_types::Settings; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::mcp::ClientMcpExtensions; use codex_protocol::mcp::OPENAI_STANDARD_FORM_INPUT_EXTENSION_ID; -use codex_protocol::models::LocalImagePreparation; -use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; @@ -430,6 +428,8 @@ use codex_protocol::turn_input::TurnStartOptions; use codex_protocol::user_input::UserInput; use codex_skills_extension::HostSkillsService; use codex_tools::ToolName; +use codex_user_input::LocalImagePreparation; +use codex_user_input::from_user_input; use codex_utils_absolute_path::AbsolutePathBuf; #[cfg(test)] use codex_utils_stream_parser::ProposedPlanSegment; @@ -4016,7 +4016,7 @@ impl Session { input: Vec, user_image_content_indices: &mut HashMap, ) -> ResponseItem { - let mut item = ResponseItem::from(ResponseInputItem::from_user_input( + let mut item = ResponseItem::from(from_user_input( input, LocalImagePreparation::Defer, user_image_content_indices, diff --git a/codex-rs/core/src/session/turn_start_mcp_tests.rs b/codex-rs/core/src/session/turn_start_mcp_tests.rs index 1861172f00..c0f4084622 100644 --- a/codex-rs/core/src/session/turn_start_mcp_tests.rs +++ b/codex-rs/core/src/session/turn_start_mcp_tests.rs @@ -4,6 +4,7 @@ use super::*; use crate::McpRefreshReason; use codex_mcp::McpResourceClient; use codex_mcp::McpResourceClientCacheKey; +use codex_protocol::models::ResponseInputItem; use pretty_assertions::assert_eq; struct Recorder { diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index 8ebacae138..9b2fd2211d 100755 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -2092,11 +2092,15 @@ async fn responses_websocket_uses_incremental_create_on_prefix() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); - let audio = ResponseItem::from(ResponseInputItem::from(vec![UserInput::Audio { - audio: AudioReference::File { - file_id: "file_audio".to_string(), - }, - }])); + let audio = ResponseItem::from(ResponseInputItem::Message { + role: "user".to_string(), + content: vec![ContentItem::InputAudio { + audio: AudioReference::File { + file_id: "file_audio".to_string(), + }, + }], + phase: None, + }); let prompt_one = prompt_with_input(vec![audio.clone()]); let prompt_two = prompt_with_input(vec![ audio, diff --git a/codex-rs/ext/queue/Cargo.toml b/codex-rs/ext/queue/Cargo.toml index 4b5e5d04f4..8d2754113d 100644 --- a/codex-rs/ext/queue/Cargo.toml +++ b/codex-rs/ext/queue/Cargo.toml @@ -18,6 +18,7 @@ codex-core = { workspace = true } codex-extension-api = { workspace = true } codex-protocol = { workspace = true } codex-thread-store = { workspace = true } +codex-user-input = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "sync", "time"] } diff --git a/codex-rs/ext/queue/src/service.rs b/codex-rs/ext/queue/src/service.rs index 12671a0ef7..420181a04c 100644 --- a/codex-rs/ext/queue/src/service.rs +++ b/codex-rs/ext/queue/src/service.rs @@ -19,7 +19,6 @@ use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadResumeInput; use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; -use codex_protocol::models::snapshot_local_user_input; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; @@ -31,6 +30,7 @@ use codex_thread_store::MAX_QUEUE_ITEMS; use codex_thread_store::QueueStore; use codex_thread_store::QueuedUserSubmissionRecord; use codex_thread_store::ThreadStoreError; +use codex_user_input::snapshot_local_user_input; use thiserror::Error; use tokio::sync::Mutex; use tokio::sync::OwnedMutexGuard; diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index 095ed6a9e2..c4d2f9c11c 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -43,7 +43,6 @@ pub mod exec_output; pub mod hooks; pub mod items; mod legacy_events; -pub mod local_media; pub mod mcp; pub mod mcp_approval_meta; pub mod mcp_policy; diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 05273a0cac..1df16bc298 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -5,7 +5,6 @@ use std::path::Path; use std::sync::Arc; use codex_utils_image::PromptImageMode; -use codex_utils_image::data_url_from_bytes; use codex_utils_image::load_for_prompt_bytes; use serde::Deserialize; use serde::Deserializer; @@ -14,7 +13,6 @@ use serde::ser::Serializer; use serde_with::serde_as; use ts_rs::TS; -use crate::local_media::audio_mime_for_path; use crate::permissions::FileSystemAccessMode; use crate::permissions::FileSystemPath; use crate::permissions::FileSystemSandboxEntry; @@ -25,7 +23,6 @@ use crate::permissions::LegacySandboxPolicyProjection; use crate::permissions::NetworkSandboxPolicy; use crate::permissions::RawFileSystemSandboxEntry; use crate::protocol::SandboxPolicy; -use crate::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_image::ImageProcessingError; use schemars::JsonSchema; @@ -43,8 +40,6 @@ mod mcp_content; use mcp_content::convert_mcp_content_to_items; -pub use crate::local_media::MAX_PROMPT_AUDIO_INPUT_BYTES; -pub use crate::local_media::snapshot_local_user_input; pub use crate::permission_profile_snapshot::PermissionProfileSnapshot; pub use crate::permission_profile_snapshot::ProfileWorkspaceRoot; pub use configuration_update::ConfigurationReasoning; @@ -1900,33 +1895,21 @@ fn should_serialize_reasoning_content(content: &Option } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum LocalMediaKind { - Audio, - Image, -} - -impl LocalMediaKind { - fn name(self) -> &'static str { - match self { - Self::Audio => "audio", - Self::Image => "image", - } - } -} - -fn local_media_error_placeholder( +fn local_image_error_placeholder( path: &std::path::Path, error: impl std::fmt::Display, - media_kind: LocalMediaKind, ) -> ContentItem { - let media_name = media_kind.name(); let path = path.display(); ContentItem::InputText { - text: format!("Codex could not read the local {media_name} at `{path}`: {error}"), + text: format!("Codex could not read the local image at `{path}`: {error}"), } } +/// Maximum accepted decoded byte length for prompt audio inputs. +/// +/// This matches the Responses API audio input limit. +pub const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024; + pub const VIEW_IMAGE_TOOL_NAME: &str = "view_image"; const IMAGE_OPEN_TAG: &str = ""; @@ -1938,7 +1921,6 @@ const AUDIO_OPEN_TAG: &str = ""; const LOCAL_AUDIO_OPEN_TAG_PREFIX: &str = "