diff --git a/.github/actions/setup-rusty-v8/action.yml b/.github/actions/setup-rusty-v8/action.yml index d9c4484657c6..564792ce4164 100644 --- a/.github/actions/setup-rusty-v8/action.yml +++ b/.github/actions/setup-rusty-v8/action.yml @@ -15,8 +15,7 @@ runs: run: | set -euo pipefail - version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)" - release_tag="rusty-v8-v${version}" + release_tag="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" rusty-v8-release-tag)" base_url="https://github.com/openai/codex/releases/download/${release_tag}" binding_dir="${RUNNER_TEMP}/rusty_v8" archive_path="${binding_dir}/librusty_v8_release_${TARGET}.a.gz" diff --git a/.github/scripts/rusty_v8_bazel.py b/.github/scripts/rusty_v8_bazel.py index 329d3f6c54ad..f2eef61ada1e 100644 --- a/.github/scripts/rusty_v8_bazel.py +++ b/.github/scripts/rusty_v8_bazel.py @@ -27,6 +27,10 @@ RELEASE_ARTIFACT_PROFILE = "release" SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release" ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"] +V8_SOURCE_ARCHIVE_PATTERN = re.compile( + r"https://github\.com/v8/v8/archive/refs/tags/" + r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz" +) def bazel_execroot() -> Path: @@ -170,6 +174,26 @@ def resolved_v8_crate_version() -> str: return matches[0] +def resolved_v8_source_version(module_bazel: Path | None = None) -> str: + module_bazel = module_bazel or MODULE_BAZEL + matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text()))) + if len(matches) != 1: + raise SystemExit( + "expected exactly one pinned V8 source version in MODULE.bazel, " + f"found: {matches}" + ) + return matches[0] + + +def rusty_v8_release_tag( + crate_version: str | None = None, + source_version: str | None = None, +) -> str: + crate_version = crate_version or resolved_v8_crate_version() + source_version = source_version or resolved_v8_source_version() + return f"rusty-v8-v{crate_version}-v8-{source_version}" + + def rusty_v8_checksum_manifest_path(version: str) -> Path: return RUSTY_V8_CHECKSUMS_DIR / f"rusty_v8_{version.replace('.', '_')}.sha256" @@ -337,6 +361,8 @@ def parse_args() -> argparse.Namespace: stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true") subparsers.add_parser("resolved-v8-crate-version") + subparsers.add_parser("resolved-v8-source-version") + subparsers.add_parser("rusty-v8-release-tag") check_module_bazel_parser = subparsers.add_parser("check-module-bazel") check_module_bazel_parser.add_argument("--version") @@ -382,6 +408,12 @@ def main() -> int: if args.command == "resolved-v8-crate-version": print(resolved_v8_crate_version()) return 0 + if args.command == "resolved-v8-source-version": + print(resolved_v8_source_version()) + return 0 + if args.command == "rusty-v8-release-tag": + print(rusty_v8_release_tag()) + return 0 if args.command == "check-module-bazel": version = command_version(args.version) manifest_path = command_manifest_path(args.manifest, version) diff --git a/.github/scripts/test_rusty_v8_bazel.py b/.github/scripts/test_rusty_v8_bazel.py index 0b5c03f43664..f5006a7684da 100644 --- a/.github/scripts/test_rusty_v8_bazel.py +++ b/.github/scripts/test_rusty_v8_bazel.py @@ -54,6 +54,34 @@ def test_consumer_selectors_track_resolved_crate_version(self) -> None: build_bazel, ) + def test_release_tag_combines_crate_and_source_versions(self) -> None: + self.assertEqual( + "rusty-v8-v149.2.0-v8-14.9.207.35", + rusty_v8_bazel.rusty_v8_release_tag( + crate_version="149.2.0", + source_version="14.9.207.35", + ), + ) + + def test_resolved_v8_source_version_reads_archive_override(self) -> None: + with TemporaryDirectory() as temp_dir: + module_bazel = Path(temp_dir) / "MODULE.bazel" + module_bazel.write_text( + textwrap.dedent( + """\ + archive_override( + module_name = "v8", + urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"], + ) + """ + ) + ) + + self.assertEqual( + "14.9.207.35", + rusty_v8_bazel.resolved_v8_source_version(module_bazel), + ) + def test_command_version_tracks_remaining_http_file_assets(self) -> None: with TemporaryDirectory() as temp_dir: module_bazel = Path(temp_dir) / "MODULE.bazel" diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.yml index 92be3761ddbd..77b8d493c0b5 100644 --- a/.github/workflows/rusty-v8-release.yml +++ b/.github/workflows/rusty-v8-release.yml @@ -3,7 +3,7 @@ name: rusty-v8-release on: push: tags: - - "rusty-v8-v*.*.*" + - "rusty-v8-v*.*.*-v8-*.*.*.*" # Cargo's libgit2 transport has been flaky when fetching git dependencies with # nested submodules. Prefer the system git CLI for Cargo smoke tests. @@ -43,12 +43,11 @@ jobs: id: release_tag env: GITHUB_REF_NAME: ${{ github.ref_name }} - V8_VERSION: ${{ steps.v8_version.outputs.version }} shell: bash run: | set -euo pipefail - expected_release_tag="rusty-v8-v${V8_VERSION}" + expected_release_tag="$(python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag)" release_tag="${GITHUB_REF_NAME}" if [[ "${release_tag}" != "${expected_release_tag}" ]]; then echo "Tag ${release_tag} does not match expected release tag ${expected_release_tag}." >&2 diff --git a/MODULE.bazel b/MODULE.bazel index 20d80f396260..02ae0750d412 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -432,18 +432,18 @@ crate.annotation( inject_repo(crate, "alsa_lib") -bazel_dep(name = "v8", version = "14.9.207.2") +bazel_dep(name = "v8", version = "14.9.207.35") archive_override( module_name = "v8", - integrity = "sha256-tflbZE5srqal6leMxJjK/ZQtwpF96OMGJ6avd5lice4=", + integrity = "sha256-YI1fHXrwPWls/kfz/gDJ4kasc/TR2ng0n70PHUQAgsE=", patch_strip = 3, patches = [ "//patches:v8_module_deps.patch", "//patches:v8_bazel_rules.patch", "//patches:v8_source_portability.patch", ], - strip_prefix = "v8-14.9.207.2", - urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.2.tar.gz"], + strip_prefix = "v8-14.9.207.35", + urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"], ) http_archive( diff --git a/scripts/codex_package/test_v8.py b/scripts/codex_package/test_v8.py new file mode 100644 index 000000000000..bd397a74e170 --- /dev/null +++ b/scripts/codex_package/test_v8.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys +import tempfile +import textwrap +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from codex_package.v8 import resolved_v8_source_version +from codex_package.v8 import rusty_v8_release_tag + + +class RustyV8ReleaseTagTest(unittest.TestCase): + def test_release_tag_combines_crate_and_source_versions(self) -> None: + self.assertEqual( + rusty_v8_release_tag( + crate_version="149.2.0", + source_version="14.9.207.35", + ), + "rusty-v8-v149.2.0-v8-14.9.207.35", + ) + + def test_resolved_v8_source_version_reads_archive_override(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + module_bazel = Path(temp_dir) / "MODULE.bazel" + module_bazel.write_text( + textwrap.dedent( + """\ + archive_override( + module_name = "v8", + urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"], + ) + """ + ) + ) + + self.assertEqual( + resolved_v8_source_version(module_bazel), + "14.9.207.35", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/codex_package/v8.py b/scripts/codex_package/v8.py index 4033e6822fbf..c049c70f628e 100644 --- a/scripts/codex_package/v8.py +++ b/scripts/codex_package/v8.py @@ -4,6 +4,7 @@ import hashlib import os +import re import shutil import tempfile from collections.abc import Mapping @@ -16,6 +17,10 @@ DOWNLOAD_TIMEOUT_SECS = 120 +V8_SOURCE_ARCHIVE_PATTERN = re.compile( + r"https://github\.com/v8/v8/archive/refs/tags/" + r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz" +) @dataclass(frozen=True) @@ -56,7 +61,7 @@ def resolve_codex_v8_cargo_env( def fetch_codex_v8_artifacts( spec: TargetSpec, *, - version: str | None = None, + crate_version: str | None = None, cache_root: Path | None = None, ) -> RustyV8ArtifactPair: if spec.is_windows: @@ -64,12 +69,14 @@ def fetch_codex_v8_artifacts( f"No Codex-built V8 release artifacts for target: {spec.target}" ) - version = version or resolved_v8_crate_version() - release_url = ( - f"https://github.com/openai/codex/releases/download/rusty-v8-v{version}" + crate_version = crate_version or resolved_v8_crate_version() + release_tag = rusty_v8_release_tag( + crate_version=crate_version, + source_version=resolved_v8_source_version(), ) + release_url = f"https://github.com/openai/codex/releases/download/{release_tag}" target = spec.target - cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}" + cache_dir = (cache_root or default_cache_root()) / f"{release_tag}-{target}" archive = cache_dir / f"librusty_v8_release_{target}.a.gz" binding = cache_dir / f"src_binding_release_{target}.rs" checksums = cache_dir / f"rusty_v8_release_{target}.sha256" @@ -104,6 +111,27 @@ def resolved_v8_crate_version() -> str: return versions[0] +def resolved_v8_source_version(module_bazel: Path | None = None) -> str: + module_bazel = module_bazel or REPO_ROOT / "MODULE.bazel" + matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text()))) + if len(matches) != 1: + raise RuntimeError( + "Expected exactly one pinned V8 source version in MODULE.bazel, " + f"found: {matches}" + ) + return matches[0] + + +def rusty_v8_release_tag( + *, + crate_version: str | None = None, + source_version: str | None = None, +) -> str: + crate_version = crate_version or resolved_v8_crate_version() + source_version = source_version or resolved_v8_source_version() + return f"rusty-v8-v{crate_version}-v8-{source_version}" + + def default_cache_root() -> Path: return Path(tempfile.gettempdir()) / "codex-package" diff --git a/third_party/v8/README.md b/third_party/v8/README.md index 4e1c022b98c6..91961e7a7bb6 100644 --- a/third_party/v8/README.md +++ b/third_party/v8/README.md @@ -19,17 +19,22 @@ prebuilts. Current pinned versions: - Rust crate: `v8 = =149.2.0` -- Embedded upstream V8 source for Bazel-produced release builds: `14.9.207.2` +- Embedded upstream V8 source for Bazel-produced release builds: `14.9.207.35` + (the revision used by Chromium `149.0.7827.201`) +- Codex artifact release tag: + `rusty-v8-v149.2.0-v8-14.9.207.35` ## Updating to a new `v8` release Use this as the maintainer flow for a version bump: -1. Bump the `v8` crate version and refresh `codex-rs/Cargo.lock`. -2. Update the Bazel versioned inputs in `MODULE.bazel`, then refresh the - matching checksum manifest and generated checksums as described below. +1. Update the `v8` crate and `codex-rs/Cargo.lock` when the Rust binding + changes. Update the embedded V8 source in `MODULE.bazel` independently. +2. Refresh the matching checksum manifest and generated checksums when the + remaining prebuilt inputs change, as described below. 3. Publish a release-candidate PR and validate that `v8-canary` passes. -4. If the canary is green, publish the release tag and release build. +4. If the canary is green, publish the release tag returned by + `python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag`. 5. Once the release build completes, rerun the build on the candidate branch and verify that the final artifact builds and tests pass. @@ -52,7 +57,7 @@ The consumer-facing selectors are: Published release assets are expected at the tag: -- `rusty-v8-v` +- `rusty-v8-v-v8-` with these raw asset names: @@ -88,8 +93,10 @@ The same run also builds the matching sandbox pair targets: The workflow also builds sandbox-enabled `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc` archive/binding pairs -from upstream `rusty_v8` source. Those ABI-specific outputs cannot be produced -by Codex's Bazel Windows GNU toolchain. +from the upstream `rusty_v8` crate source. These outputs use that crate's own V8 +checkout, which may differ from the V8 source version in the Codex artifact tag. +This is a known temporary exception: those ABI-specific outputs cannot be +produced by Codex's Bazel Windows GNU toolchain. The Bazel graph pins the same libc++, libc++abi, and llvm-libc source revisions used by `rusty_v8 v149.2.0`, compiles published artifact targets with @@ -110,7 +117,10 @@ Release and CI Cargo builds for Darwin and Linux use `RUSTY_V8_ARCHIVE` plus a downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release assets directly. We do not use `RUSTY_V8_MIRROR` because the upstream `v8` crate hardcodes a `v` tag layout, while our artifacts are published -under `rusty-v8-v`. +under `rusty-v8-v-v8-`. -Do not mix artifacts across crate versions. The archive and binding must match -the exact resolved `v8` crate version in `codex-rs/Cargo.lock`. +For Codex-built artifacts, do not mix artifacts across crate or source versions. +The archive and binding must match both the exact resolved `v8` crate version in +`codex-rs/Cargo.lock` and the embedded V8 source version in `MODULE.bazel`. The +Windows MSVC exception above matches the crate version but does not yet +guarantee the tagged Codex source version.