Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/actions/setup-rusty-v8/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@ runs:
run: |
set -euo pipefail

version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)"
release_tag="rusty-v8-v${version}"
release_tag="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" rusty-v8-release-tag)"
base_url="https://github.com/openai/codex/releases/download/${release_tag}"
binding_dir="${RUNNER_TEMP}/rusty_v8"
archive_path="${binding_dir}/librusty_v8_release_${TARGET}.a.gz"
Expand Down
32 changes: 32 additions & 0 deletions .github/scripts/rusty_v8_bazel.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
RELEASE_ARTIFACT_PROFILE = "release"
SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release"
ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"]
V8_SOURCE_ARCHIVE_PATTERN = re.compile(
r"https://github\.com/v8/v8/archive/refs/tags/"
r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz"
)


def bazel_execroot() -> Path:
Expand Down Expand Up @@ -170,6 +174,26 @@ def resolved_v8_crate_version() -> str:
return matches[0]


def resolved_v8_source_version(module_bazel: Path | None = None) -> str:
module_bazel = module_bazel or MODULE_BAZEL
matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text())))
if len(matches) != 1:
raise SystemExit(
"expected exactly one pinned V8 source version in MODULE.bazel, "
f"found: {matches}"
)
return matches[0]


def rusty_v8_release_tag(
crate_version: str | None = None,
source_version: str | None = None,
) -> str:
crate_version = crate_version or resolved_v8_crate_version()
source_version = source_version or resolved_v8_source_version()
return f"rusty-v8-v{crate_version}-v8-{source_version}"


def rusty_v8_checksum_manifest_path(version: str) -> Path:
return RUSTY_V8_CHECKSUMS_DIR / f"rusty_v8_{version.replace('.', '_')}.sha256"

Expand Down Expand Up @@ -337,6 +361,8 @@ def parse_args() -> argparse.Namespace:
stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true")

subparsers.add_parser("resolved-v8-crate-version")
subparsers.add_parser("resolved-v8-source-version")
subparsers.add_parser("rusty-v8-release-tag")

check_module_bazel_parser = subparsers.add_parser("check-module-bazel")
check_module_bazel_parser.add_argument("--version")
Expand Down Expand Up @@ -382,6 +408,12 @@ def main() -> int:
if args.command == "resolved-v8-crate-version":
print(resolved_v8_crate_version())
return 0
if args.command == "resolved-v8-source-version":
print(resolved_v8_source_version())
return 0
if args.command == "rusty-v8-release-tag":
print(rusty_v8_release_tag())
return 0
if args.command == "check-module-bazel":
version = command_version(args.version)
manifest_path = command_manifest_path(args.manifest, version)
Expand Down
28 changes: 28 additions & 0 deletions .github/scripts/test_rusty_v8_bazel.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,34 @@ def test_consumer_selectors_track_resolved_crate_version(self) -> None:
build_bazel,
)

def test_release_tag_combines_crate_and_source_versions(self) -> None:
self.assertEqual(
"rusty-v8-v149.2.0-v8-14.9.207.35",
rusty_v8_bazel.rusty_v8_release_tag(
crate_version="149.2.0",
source_version="14.9.207.35",
),
)

def test_resolved_v8_source_version_reads_archive_override(self) -> None:
with TemporaryDirectory() as temp_dir:
module_bazel = Path(temp_dir) / "MODULE.bazel"
module_bazel.write_text(
textwrap.dedent(
"""\
archive_override(
module_name = "v8",
urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"],
)
"""
)
)

self.assertEqual(
"14.9.207.35",
rusty_v8_bazel.resolved_v8_source_version(module_bazel),
)

def test_command_version_tracks_remaining_http_file_assets(self) -> None:
with TemporaryDirectory() as temp_dir:
module_bazel = Path(temp_dir) / "MODULE.bazel"
Expand Down
5 changes: 2 additions & 3 deletions .github/workflows/rusty-v8-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: rusty-v8-release
on:
push:
tags:
- "rusty-v8-v*.*.*"
- "rusty-v8-v*.*.*-v8-*.*.*.*"

# Cargo's libgit2 transport has been flaky when fetching git dependencies with
# nested submodules. Prefer the system git CLI for Cargo smoke tests.
Expand Down Expand Up @@ -43,12 +43,11 @@ jobs:
id: release_tag
env:
GITHUB_REF_NAME: ${{ github.ref_name }}
V8_VERSION: ${{ steps.v8_version.outputs.version }}
shell: bash
run: |
set -euo pipefail

expected_release_tag="rusty-v8-v${V8_VERSION}"
expected_release_tag="$(python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag)"
release_tag="${GITHUB_REF_NAME}"
if [[ "${release_tag}" != "${expected_release_tag}" ]]; then
echo "Tag ${release_tag} does not match expected release tag ${expected_release_tag}." >&2
Expand Down
8 changes: 4 additions & 4 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -432,18 +432,18 @@ crate.annotation(

inject_repo(crate, "alsa_lib")

bazel_dep(name = "v8", version = "14.9.207.2")
bazel_dep(name = "v8", version = "14.9.207.35")
archive_override(
module_name = "v8",
integrity = "sha256-tflbZE5srqal6leMxJjK/ZQtwpF96OMGJ6avd5lice4=",
integrity = "sha256-YI1fHXrwPWls/kfz/gDJ4kasc/TR2ng0n70PHUQAgsE=",
patch_strip = 3,
patches = [
"//patches:v8_module_deps.patch",
"//patches:v8_bazel_rules.patch",
"//patches:v8_source_portability.patch",
],
strip_prefix = "v8-14.9.207.2",
urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.2.tar.gz"],
strip_prefix = "v8-14.9.207.35",
urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"],
)

http_archive(
Expand Down
46 changes: 46 additions & 0 deletions scripts/codex_package/test_v8.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/usr/bin/env python3

from pathlib import Path
import sys
import tempfile
import textwrap
import unittest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from codex_package.v8 import resolved_v8_source_version
from codex_package.v8 import rusty_v8_release_tag


class RustyV8ReleaseTagTest(unittest.TestCase):
def test_release_tag_combines_crate_and_source_versions(self) -> None:
self.assertEqual(
rusty_v8_release_tag(
crate_version="149.2.0",
source_version="14.9.207.35",
),
"rusty-v8-v149.2.0-v8-14.9.207.35",
)

def test_resolved_v8_source_version_reads_archive_override(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
module_bazel = Path(temp_dir) / "MODULE.bazel"
module_bazel.write_text(
textwrap.dedent(
"""\
archive_override(
module_name = "v8",
urls = ["https://github.com/v8/v8/archive/refs/tags/14.9.207.35.tar.gz"],
)
"""
)
)

self.assertEqual(
resolved_v8_source_version(module_bazel),
"14.9.207.35",
)


if __name__ == "__main__":
unittest.main()
38 changes: 33 additions & 5 deletions scripts/codex_package/v8.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import hashlib
import os
import re
import shutil
import tempfile
from collections.abc import Mapping
Expand All @@ -16,6 +17,10 @@


DOWNLOAD_TIMEOUT_SECS = 120
V8_SOURCE_ARCHIVE_PATTERN = re.compile(
r"https://github\.com/v8/v8/archive/refs/tags/"
r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz"
)


@dataclass(frozen=True)
Expand Down Expand Up @@ -56,20 +61,22 @@ def resolve_codex_v8_cargo_env(
def fetch_codex_v8_artifacts(
spec: TargetSpec,
*,
version: str | None = None,
crate_version: str | None = None,
cache_root: Path | None = None,
) -> RustyV8ArtifactPair:
if spec.is_windows:
raise RuntimeError(
f"No Codex-built V8 release artifacts for target: {spec.target}"
)

version = version or resolved_v8_crate_version()
release_url = (
f"https://github.com/openai/codex/releases/download/rusty-v8-v{version}"
crate_version = crate_version or resolved_v8_crate_version()
release_tag = rusty_v8_release_tag(
crate_version=crate_version,
source_version=resolved_v8_source_version(),
)
release_url = f"https://github.com/openai/codex/releases/download/{release_tag}"
target = spec.target
cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}"
cache_dir = (cache_root or default_cache_root()) / f"{release_tag}-{target}"
archive = cache_dir / f"librusty_v8_release_{target}.a.gz"
binding = cache_dir / f"src_binding_release_{target}.rs"
checksums = cache_dir / f"rusty_v8_release_{target}.sha256"
Expand Down Expand Up @@ -104,6 +111,27 @@ def resolved_v8_crate_version() -> str:
return versions[0]


def resolved_v8_source_version(module_bazel: Path | None = None) -> str:
module_bazel = module_bazel or REPO_ROOT / "MODULE.bazel"
matches = sorted(set(V8_SOURCE_ARCHIVE_PATTERN.findall(module_bazel.read_text())))
if len(matches) != 1:
raise RuntimeError(
"Expected exactly one pinned V8 source version in MODULE.bazel, "
f"found: {matches}"
)
return matches[0]


def rusty_v8_release_tag(
*,
crate_version: str | None = None,
source_version: str | None = None,
) -> str:
crate_version = crate_version or resolved_v8_crate_version()
source_version = source_version or resolved_v8_source_version()
return f"rusty-v8-v{crate_version}-v8-{source_version}"


def default_cache_root() -> Path:
return Path(tempfile.gettempdir()) / "codex-package"

Expand Down
32 changes: 21 additions & 11 deletions third_party/v8/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,22 @@ prebuilts.
Current pinned versions:

- Rust crate: `v8 = =149.2.0`
- Embedded upstream V8 source for Bazel-produced release builds: `14.9.207.2`
- Embedded upstream V8 source for Bazel-produced release builds: `14.9.207.35`
(the revision used by Chromium `149.0.7827.201`)
- Codex artifact release tag:
`rusty-v8-v149.2.0-v8-14.9.207.35`

## Updating to a new `v8` release

Use this as the maintainer flow for a version bump:

1. Bump the `v8` crate version and refresh `codex-rs/Cargo.lock`.
2. Update the Bazel versioned inputs in `MODULE.bazel`, then refresh the
matching checksum manifest and generated checksums as described below.
1. Update the `v8` crate and `codex-rs/Cargo.lock` when the Rust binding
changes. Update the embedded V8 source in `MODULE.bazel` independently.
2. Refresh the matching checksum manifest and generated checksums when the
remaining prebuilt inputs change, as described below.
3. Publish a release-candidate PR and validate that `v8-canary` passes.
4. If the canary is green, publish the release tag and release build.
4. If the canary is green, publish the release tag returned by
`python3 .github/scripts/rusty_v8_bazel.py rusty-v8-release-tag`.
5. Once the release build completes, rerun the build on the candidate branch
and verify that the final artifact builds and tests pass.

Expand All @@ -52,7 +57,7 @@ The consumer-facing selectors are:

Published release assets are expected at the tag:

- `rusty-v8-v<crate_version>`
- `rusty-v8-v<crate_version>-v8-<source_version>`

with these raw asset names:

Expand Down Expand Up @@ -88,8 +93,10 @@ The same run also builds the matching sandbox pair targets:

The workflow also builds sandbox-enabled
`x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc` archive/binding pairs
from upstream `rusty_v8` source. Those ABI-specific outputs cannot be produced
by Codex's Bazel Windows GNU toolchain.
from the upstream `rusty_v8` crate source. These outputs use that crate's own V8
checkout, which may differ from the V8 source version in the Codex artifact tag.
This is a known temporary exception: those ABI-specific outputs cannot be
produced by Codex's Bazel Windows GNU toolchain.

The Bazel graph pins the same libc++, libc++abi, and llvm-libc source revisions
used by `rusty_v8 v149.2.0`, compiles published artifact targets with
Expand All @@ -110,7 +117,10 @@ Release and CI Cargo builds for Darwin and Linux use `RUSTY_V8_ARCHIVE` plus a
downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release
assets directly. We do not use `RUSTY_V8_MIRROR` because the upstream `v8` crate
hardcodes a `v<crate_version>` tag layout, while our artifacts are published
under `rusty-v8-v<crate_version>`.
under `rusty-v8-v<crate_version>-v8-<source_version>`.

Do not mix artifacts across crate versions. The archive and binding must match
the exact resolved `v8` crate version in `codex-rs/Cargo.lock`.
For Codex-built artifacts, do not mix artifacts across crate or source versions.
The archive and binding must match both the exact resolved `v8` crate version in
`codex-rs/Cargo.lock` and the embedded V8 source version in `MODULE.bazel`. The
Windows MSVC exception above matches the crate version but does not yet
guarantee the tagged Codex source version.
Loading