mirror of
https://github.com/openai/codex.git
synced 2026-10-31 19:28:49 +00:00
447 lines
17 KiB
Python
447 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
|
|
import run_bazel_with_buildbuddy
|
|
|
|
|
|
class RunBazelWithBuildBuddyTest(unittest.TestCase):
|
|
def run_bazel_ci(
|
|
self,
|
|
temp_dir: str,
|
|
env_overrides: dict[str, str],
|
|
*args: str,
|
|
) -> list[str]:
|
|
fake_bazel_impl = Path(temp_dir) / "fake-bazel.py"
|
|
fake_bazel_impl.write_text(
|
|
"#!/usr/bin/env python3\n"
|
|
"import json\n"
|
|
"import os\n"
|
|
"import sys\n"
|
|
"if os.environ.get('RUNNER_OS') == 'Windows':\n"
|
|
" forbidden = {'AR', 'ARFLAGS', 'AS', 'BAZEL_LLVM', "
|
|
"'BAZEL_LLVM_COV', 'BAZEL_LLVM_PROFDATA', 'BAZEL_VC', "
|
|
"'BAZEL_VC_FULL_VERSION', 'BAZEL_VS', 'BAZEL_WINSDK_FULL_VERSION', "
|
|
"'CC', 'CFLAGS', 'CL', 'CMAKE', 'CPPFLAGS', 'CXX', 'CXXFLAGS', "
|
|
"'DEVENVDIR', 'DLLTOOL', 'EXTENSIONSDKDIR', 'FRAMEWORKDIR', "
|
|
"'FRAMEWORKDIR32', 'FRAMEWORKVERSION', 'FRAMEWORKVERSION32', "
|
|
"'GO', 'INCLUDE', 'LD', 'LDFLAGS', 'LIB', 'LIBPATH', 'LINK', "
|
|
"'MAKE', 'MESON', 'NASM', 'NETFXSDKDIR', 'NINJA', 'NM', "
|
|
"'OBJCOPY', 'OBJDUMP', 'PERL', 'PERL5LIB', 'PKG_CONFIG', "
|
|
"'PROTOC', 'RANLIB', 'RC', 'RUSTC', 'RUSTDOC', 'RUSTFLAGS', "
|
|
"'STRIP', 'UCRTVERSION', 'UNIVERSALCRTSDKDIR', 'USE_CLANG_CL', "
|
|
"'VCIDEINSTALLDIR', "
|
|
"'USE_CLANG_CL', 'VCINSTALLDIR', 'VISUALSTUDIOVERSION', 'VSINSTALLDIR', "
|
|
"'WINDRES', 'WINDOWSLIBPATH', 'YASM', '_CL_', '_LINK_'}\n"
|
|
" tool_affixes = ('AR', 'AS', 'CC', 'CFLAGS', 'CXX', "
|
|
"'CXXFLAGS', 'LD')\n"
|
|
" present = []\n"
|
|
" for name in os.environ:\n"
|
|
" upper = name.upper()\n"
|
|
" if (upper in forbidden or upper.startswith(('CMAKE_', "
|
|
"'LLVM_', 'MINGW', 'MSVC', 'VCTOOLS', 'VSCMD_', 'WINDOWSSDK')) "
|
|
"or (upper.startswith('CARGO_TARGET_') and upper.endswith('_LINKER')) "
|
|
"or any(upper.startswith(f'{tool}_') or upper.endswith(f'_{tool}') "
|
|
"for tool in tool_affixes)):\n"
|
|
" present.append(name)\n"
|
|
" if present:\n"
|
|
" raise SystemExit(f'compiler environment was not cleared: {present}')\n"
|
|
" normalized_path = os.environ.get('PATH', '').lower().replace('\\\\', '/')\n"
|
|
" forbidden_paths = ('/microsoft visual studio/', '/windows kits/', "
|
|
"'/microsoft sdks/', '/program files/llvm/', '/program files (x86)/llvm/', "
|
|
"'/msys64/', '/mingw32/', '/mingw64/')\n"
|
|
" if any(path in normalized_path for path in forbidden_paths):\n"
|
|
" raise SystemExit(f'compiler path was not cleared: {normalized_path}')\n"
|
|
" expected_bazel_sh = 'C:/Program Files/Git/usr/bin/bash.exe'\n"
|
|
" if os.environ.get('BAZEL_SH') != expected_bazel_sh:\n"
|
|
" raise SystemExit(f'BAZEL_SH is not frozen: {os.environ.get(\"BAZEL_SH\")}')\n"
|
|
"print(json.dumps(sys.argv[1:]))\n",
|
|
encoding="utf-8",
|
|
)
|
|
if os.name == "nt":
|
|
fake_bazel = Path(temp_dir) / "fake-bazel.cmd"
|
|
fake_bazel.write_text(
|
|
f'@"{sys.executable}" "{fake_bazel_impl}" %*\n',
|
|
encoding="utf-8",
|
|
)
|
|
else:
|
|
fake_bazel = fake_bazel_impl
|
|
fake_bazel.chmod(0o755)
|
|
|
|
env = os.environ.copy()
|
|
for name in (
|
|
"BUILDBUDDY_API_KEY",
|
|
"GITHUB_ACTIONS",
|
|
"GITHUB_EVENT_NAME",
|
|
"GITHUB_EVENT_PATH",
|
|
"GITHUB_REPOSITORY",
|
|
):
|
|
env.pop(name, None)
|
|
env.update(env_overrides)
|
|
env["CODEX_BAZEL_BIN"] = str(fake_bazel)
|
|
|
|
bash = "bash"
|
|
if os.name == "nt":
|
|
bash = str(Path(os.environ["ProgramFiles"]) / "Git" / "bin" / "bash.exe")
|
|
self.assertTrue(Path(bash).is_file(), bash)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
bash,
|
|
str(Path(__file__).with_name("run-bazel-ci.sh")),
|
|
*args,
|
|
],
|
|
env=env,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
return next(
|
|
json.loads(line)
|
|
for line in result.stdout.splitlines()
|
|
if line.startswith("[")
|
|
)
|
|
|
|
def github_env(
|
|
self,
|
|
temp_dir: str,
|
|
*,
|
|
repository: str = "openai/codex",
|
|
fork: bool = False,
|
|
event_name: str = "pull_request",
|
|
) -> dict[str, str]:
|
|
event_path = Path(temp_dir) / "event.json"
|
|
event_path.write_text(
|
|
json.dumps({"pull_request": {"head": {"repo": {"fork": fork}}}}),
|
|
encoding="utf-8",
|
|
)
|
|
return {
|
|
"BUILDBUDDY_API_KEY": "token",
|
|
"GITHUB_ACTIONS": "true",
|
|
"GITHUB_EVENT_NAME": event_name,
|
|
"GITHUB_EVENT_PATH": str(event_path),
|
|
"GITHUB_REPOSITORY": repository,
|
|
}
|
|
|
|
def test_keyless_invocation_drops_remote_ci_configuration(self) -> None:
|
|
self.assertIsNone(
|
|
run_bazel_with_buildbuddy.remote_config(
|
|
["build", "--config=ci-linux", "//codex-rs/cli:codex"],
|
|
{},
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
|
["build", "--config=ci-linux", "--", "//codex-rs/cli:codex"],
|
|
{},
|
|
),
|
|
["build", "--", "//codex-rs/cli:codex"],
|
|
)
|
|
|
|
def test_program_arguments_after_separator_do_not_select_or_lose_rbe(self) -> None:
|
|
args = ["run", "//codex-rs/cli:codex", "--", "--config=remote"]
|
|
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_args_with_remote_config(args, {}),
|
|
args,
|
|
)
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.remote_config(
|
|
args, {"BUILDBUDDY_API_KEY": "fork-token"}
|
|
),
|
|
"buildbuddy-generic",
|
|
)
|
|
|
|
def test_upstream_push_selects_openai_rbe_before_target_separator(self) -> None:
|
|
with TemporaryDirectory() as temp_dir:
|
|
env = self.github_env(temp_dir, event_name="push")
|
|
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
|
["build", "--config=ci-linux", "--", "//codex-rs/cli:codex"],
|
|
env,
|
|
),
|
|
[
|
|
"build",
|
|
"--config=buildbuddy-openai-rbe",
|
|
"--remote_header=x-buildbuddy-api-key=token",
|
|
"--config=ci-linux",
|
|
"--",
|
|
"//codex-rs/cli:codex",
|
|
],
|
|
)
|
|
|
|
def test_windows_cross_ci_configuration_follows_remote_configuration(self) -> None:
|
|
env = {"BUILDBUDDY_API_KEY": "fork-token"}
|
|
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
|
["build", "--config=ci-windows-cross", "//codex-rs/cli:codex"],
|
|
env,
|
|
),
|
|
[
|
|
"build",
|
|
"--config=buildbuddy-generic-rbe",
|
|
"--remote_header=x-buildbuddy-api-key=fork-token",
|
|
"--config=ci-windows-cross",
|
|
"//codex-rs/cli:codex",
|
|
],
|
|
)
|
|
|
|
def test_windows_hybrid_configuration_uses_remote_execution(self) -> None:
|
|
env = {"BUILDBUDDY_API_KEY": "fork-token"}
|
|
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
|
[
|
|
"build",
|
|
"--config=ci-windows-hybrid",
|
|
"//codex-rs/cli:codex",
|
|
],
|
|
env,
|
|
),
|
|
[
|
|
"build",
|
|
"--config=buildbuddy-generic-rbe",
|
|
"--remote_header=x-buildbuddy-api-key=fork-token",
|
|
"--config=ci-windows-hybrid",
|
|
"//codex-rs/cli:codex",
|
|
],
|
|
)
|
|
|
|
def test_windows_hybrid_execution_separates_build_and_test_environments(
|
|
self,
|
|
) -> None:
|
|
with TemporaryDirectory() as temp_dir:
|
|
command = self.run_bazel_ci(
|
|
temp_dir,
|
|
{
|
|
"BUILDBUDDY_API_KEY": "token",
|
|
"CODEX_BAZEL_WINDOWS_EXECUTION_PATH": r"C:\substrate\bin",
|
|
"CODEX_BAZEL_WINDOWS_TEST_PATH": r"C:\runtime\bin",
|
|
"INCLUDE": r"C:\Visual Studio\include",
|
|
"ProgramFiles": "",
|
|
"RUNNER_OS": "Windows",
|
|
},
|
|
"--windows-hybrid-execution",
|
|
"--",
|
|
"build",
|
|
"--config=argument-comment-lint",
|
|
"--",
|
|
"//codex-rs/arg0:arg0",
|
|
)
|
|
self.assertIn("--config=ci-windows-hybrid", command)
|
|
self.assertIn("--shell_executable=/bin/bash", command)
|
|
self.assertIn("--action_env=PATH=/usr/bin:/bin", command)
|
|
self.assertIn("--host_action_env=PATH=/usr/bin:/bin", command)
|
|
self.assertIn(r"--test_env=PATH=C:\runtime\bin", command)
|
|
self.assertNotIn("--host_platform=//:rbe", command)
|
|
self.assertNotIn("--action_env=INCLUDE", command)
|
|
self.assertNotIn("--host_action_env=INCLUDE", command)
|
|
self.assertFalse(
|
|
any(
|
|
arg.startswith("--action_env=PATH=C:")
|
|
or arg.startswith("--host_action_env=PATH=C:")
|
|
for arg in command
|
|
)
|
|
)
|
|
|
|
def test_keyless_windows_cross_build_uses_gnullvm_substrate(self) -> None:
|
|
with TemporaryDirectory() as temp_dir:
|
|
command = self.run_bazel_ci(
|
|
temp_dir,
|
|
{
|
|
"CODEX_BAZEL_WINDOWS_EXECUTION_PATH": r"C:\substrate\bin",
|
|
"CODEX_BAZEL_WINDOWS_TEST_PATH": r"C:\runtime\bin",
|
|
"CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_LINKER": "system-linker",
|
|
"cc_x86_64_pc_windows_gnullvm": "system-cc",
|
|
"DevEnvDir": r"C:\Visual Studio\Common7\IDE",
|
|
"INCLUDE": r"C:\Visual Studio\include",
|
|
"PATH": os.environ["PATH"] + r":C:\Program Files\LLVM\bin",
|
|
"ProgramFiles": r"C:\Program Files",
|
|
"RUNNER_OS": "Windows",
|
|
},
|
|
"--windows-cross-compile",
|
|
"--",
|
|
"build",
|
|
"--",
|
|
"//codex-rs/arg0:arg0",
|
|
)
|
|
|
|
self.assertIn("--jobs=8", command)
|
|
self.assertIn(
|
|
"--shell_executable=C:/Program Files/Git/usr/bin/bash.exe", command
|
|
)
|
|
self.assertIn("--host_platform=//:local_windows", command)
|
|
self.assertIn("--platforms=//:windows_x86_64_gnullvm", command)
|
|
self.assertIn(
|
|
"--extra_execution_platforms=//:windows_x86_64_gnullvm", command
|
|
)
|
|
self.assertIn(
|
|
"--extra_toolchains=//:windows_gnullvm_tests_on_gnullvm_host_toolchain",
|
|
command,
|
|
)
|
|
self.assertIn(r"--action_env=PATH=C:\substrate\bin", command)
|
|
self.assertIn(r"--host_action_env=PATH=C:\substrate\bin", command)
|
|
self.assertIn(r"--test_env=PATH=C:\runtime\bin", command)
|
|
self.assertFalse(any("msvc" in arg.lower() for arg in command))
|
|
self.assertNotIn("--action_env=INCLUDE", command)
|
|
self.assertNotIn("--host_action_env=INCLUDE", command)
|
|
|
|
def test_query_remote_configuration_is_inserted_before_expression(self) -> None:
|
|
expression = 'kind("rust_library rule", //codex-rs/...)'
|
|
env = {"BUILDBUDDY_API_KEY": "fork-token"}
|
|
|
|
for command in ("query", "cquery", "aquery"):
|
|
with self.subTest(command=command):
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_args_with_remote_config(
|
|
[
|
|
command,
|
|
"--config=ci-windows-cross",
|
|
"--output=label",
|
|
expression,
|
|
],
|
|
env,
|
|
),
|
|
[
|
|
command,
|
|
"--config=buildbuddy-generic-rbe",
|
|
"--remote_header=x-buildbuddy-api-key=fork-token",
|
|
"--config=ci-windows-cross",
|
|
"--output=label",
|
|
expression,
|
|
],
|
|
)
|
|
|
|
def test_same_repository_pull_request_selects_openai_host(self) -> None:
|
|
with TemporaryDirectory() as temp_dir:
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.remote_config(
|
|
["build", "--config=ci-v8"], self.github_env(temp_dir)
|
|
),
|
|
"buildbuddy-openai-rbe",
|
|
)
|
|
|
|
def test_fork_pull_request_cannot_select_openai_host(self) -> None:
|
|
with TemporaryDirectory() as temp_dir:
|
|
env = self.github_env(temp_dir, fork=True)
|
|
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.remote_config(
|
|
["build", "--config=ci-v8"], env
|
|
),
|
|
"buildbuddy-generic-rbe",
|
|
)
|
|
|
|
def test_run_in_fork_repository_cannot_select_openai_host(self) -> None:
|
|
with TemporaryDirectory() as temp_dir:
|
|
env = self.github_env(temp_dir, repository="contributor/codex")
|
|
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.remote_config(
|
|
["build", "--config=ci-v8"], env
|
|
),
|
|
"buildbuddy-generic-rbe",
|
|
)
|
|
|
|
def test_pull_request_without_readable_event_payload_fails_closed(self) -> None:
|
|
for event_path in (None, "missing-event.json"):
|
|
env = {
|
|
"BUILDBUDDY_API_KEY": "token",
|
|
"GITHUB_ACTIONS": "true",
|
|
"GITHUB_EVENT_NAME": "pull_request",
|
|
"GITHUB_REPOSITORY": "openai/codex",
|
|
}
|
|
if event_path is not None:
|
|
env["GITHUB_EVENT_PATH"] = event_path
|
|
|
|
with self.subTest(event_path=event_path):
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.remote_config(["build"], env),
|
|
"buildbuddy-generic",
|
|
)
|
|
|
|
def test_bazel_command_uses_configured_binary_locally(self) -> None:
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_command(
|
|
"info",
|
|
"execution_root",
|
|
env={"CODEX_BAZEL_BIN": "fake-bazel"},
|
|
),
|
|
["fake-bazel", "info", "execution_root"],
|
|
)
|
|
|
|
def test_bazel_command_normalizes_github_actions_startup_options(self) -> None:
|
|
env = {
|
|
"BAZEL_OUTPUT_USER_ROOT": "/tmp/bazel-output",
|
|
"GITHUB_ACTIONS": "true",
|
|
}
|
|
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_command("build", "//codex-rs/...", env=env),
|
|
[
|
|
"bazel",
|
|
"--output_user_root=/tmp/bazel-output",
|
|
"--noexperimental_remote_repo_contents_cache",
|
|
"build",
|
|
"//codex-rs/...",
|
|
],
|
|
)
|
|
self.assertEqual(
|
|
run_bazel_with_buildbuddy.bazel_command(
|
|
"--experimental_remote_repo_contents_cache",
|
|
"build",
|
|
"//codex-rs/...",
|
|
env=env,
|
|
),
|
|
[
|
|
"bazel",
|
|
"--output_user_root=/tmp/bazel-output",
|
|
"--experimental_remote_repo_contents_cache",
|
|
"build",
|
|
"//codex-rs/...",
|
|
],
|
|
)
|
|
|
|
def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None:
|
|
spaced_arg = (
|
|
r"--test_env=PATH=C:\Program Files\PowerShell\7;C:\Program Files\Git\bin"
|
|
)
|
|
child_code = (
|
|
f"import sys; sys.exit(37 if sys.argv[1] == {spaced_arg!r} else 91)"
|
|
)
|
|
env = os.environ.copy()
|
|
env["CODEX_BAZEL_BIN"] = sys.executable
|
|
env.pop("BAZEL_OUTPUT_USER_ROOT", None)
|
|
env.pop("BUILDBUDDY_API_KEY", None)
|
|
env.pop("GITHUB_ACTIONS", None)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(Path(run_bazel_with_buildbuddy.__file__)),
|
|
"-c",
|
|
child_code,
|
|
spaced_arg,
|
|
],
|
|
env=env,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 37, result.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|