From be7f6f739f5f16caf5ccddff8e5e3a422eac5f75 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 9 Apr 2026 21:27:23 -0700 Subject: [PATCH 1/6] fix: add migration script for flat-path firecracker binaries to amd64/ Before ARM64 support was added, firecracker binaries were uploaded to GCS at the flat path firecrackers/{version}/firecracker. The orchestrator's arch-aware path resolution expects firecrackers/{version}/amd64/firecracker, so old builds are not found via the arch-based lookup. Add migrate-gcs-arch.sh to dry-run/copy (and optionally delete) existing flat-path GCS objects into the amd64/ subdirectory. --- migrate-gcs-arch.sh | 104 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 migrate-gcs-arch.sh diff --git a/migrate-gcs-arch.sh b/migrate-gcs-arch.sh new file mode 100755 index 000000000..b119d87e3 --- /dev/null +++ b/migrate-gcs-arch.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# Copies flat-path firecracker binaries into amd64/ subdirectories in GCS. +# +# Before ARM64 support was added, builds were uploaded as: +# firecrackers/{version_name}/firecracker +# +# The arch-aware layout expects: +# firecrackers/{version_name}/amd64/firecracker +# +# This script finds flat-path binaries and copies them into the amd64/ subdir +# so the orchestrator's arch-based path resolution can find them. +# +# Usage: +# ./migrate-gcs-arch.sh # dry-run: show what would be copied +# ./migrate-gcs-arch.sh --apply # copy flat -> amd64/ +# ./migrate-gcs-arch.sh --delete-old # dry-run: show what old flat files would be deleted +# ./migrate-gcs-arch.sh --delete-old --apply # actually delete old flat files +# +# Recommended workflow: +# 1. ./migrate-gcs-arch.sh gs://my-bucket # review what will be copied +# 2. ./migrate-gcs-arch.sh gs://my-bucket --apply # copy to amd64/ +# 3. ... verify everything works ... +# 4. ./migrate-gcs-arch.sh gs://my-bucket --delete-old # review what will be deleted +# 5. ./migrate-gcs-arch.sh gs://my-bucket --delete-old --apply # clean up old flat files + +set -euo pipefail + +BUCKET="${1:?Usage: $0 [--apply] [--delete-old]}" +shift + +APPLY=false +DELETE_OLD=false +for arg in "$@"; do + case "$arg" in + --apply) APPLY=true ;; + --delete-old) DELETE_OLD=true ;; + *) echo "Unknown flag: $arg"; exit 1 ;; + esac +done + +# Strip gs:// prefix if provided, we add it back +BUCKET="${BUCKET#gs://}" + +GCS_PREFIX="firecrackers" + +echo "Scanning gs://${BUCKET}/${GCS_PREFIX} for flat-path firecracker binaries..." +echo "" + +# Match only flat-path binaries: firecrackers/{version}/firecracker +# The single * does NOT match path separators, so this excludes +# firecrackers/{version}/{arch}/firecracker. +objects=$(gsutil ls "gs://${BUCKET}/${GCS_PREFIX}/*/firecracker" 2>/dev/null || true) + +if [[ -z "$objects" ]]; then + echo "No flat-path firecracker binaries found in gs://${BUCKET}/${GCS_PREFIX}" + exit 0 +fi + +count=0 +if [[ "$DELETE_OLD" == true ]]; then + while IFS= read -r src; do + [[ -z "$src" ]] && continue + + if [[ "$APPLY" == true ]]; then + echo " DELETE $src" + gsutil rm "$src" + else + echo " [dry-run] would delete $src" + fi + ((count++)) || true + done <<< "$objects" + + echo "" + echo "Total: $count objects" + if [[ "$APPLY" != true ]]; then + echo "" + echo "This was a dry run. Add --apply to actually delete." + fi +else + while IFS= read -r src; do + [[ -z "$src" ]] && continue + + # Insert /amd64 before the filename: + # .../firecrackers/v1.10.1/firecracker -> .../firecrackers/v1.10.1/amd64/firecracker + dst="${src%/firecracker}/amd64/firecracker" + + if [[ "$APPLY" == true ]]; then + echo " COPY $src" + echo " -> $dst" + gsutil cp "$src" "$dst" + else + echo " [dry-run] $src" + echo " -> $dst" + fi + ((count++)) || true + done <<< "$objects" + + echo "" + echo "Total: $count objects" + if [[ "$APPLY" != true ]]; then + echo "" + echo "This was a dry run. Add --apply to actually copy." + fi +fi From f054bf1210d8a851f0f32531fa83eeb6813d5fa1 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 9 Apr 2026 21:31:47 -0700 Subject: [PATCH 2/6] move migrate-gcs-arch.sh to scripts/ --- migrate-gcs-arch.sh => scripts/migrate-gcs-arch.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename migrate-gcs-arch.sh => scripts/migrate-gcs-arch.sh (100%) diff --git a/migrate-gcs-arch.sh b/scripts/migrate-gcs-arch.sh similarity index 100% rename from migrate-gcs-arch.sh rename to scripts/migrate-gcs-arch.sh From 031372a79bdde17e648e0c81da78aef13d145db2 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 9 Apr 2026 21:32:30 -0700 Subject: [PATCH 3/6] simplify migration script: remove --delete-old --- scripts/migrate-gcs-arch.sh | 76 +++++++++++-------------------------- 1 file changed, 22 insertions(+), 54 deletions(-) diff --git a/scripts/migrate-gcs-arch.sh b/scripts/migrate-gcs-arch.sh index b119d87e3..26d09f1ba 100755 --- a/scripts/migrate-gcs-arch.sh +++ b/scripts/migrate-gcs-arch.sh @@ -11,29 +11,18 @@ # so the orchestrator's arch-based path resolution can find them. # # Usage: -# ./migrate-gcs-arch.sh # dry-run: show what would be copied -# ./migrate-gcs-arch.sh --apply # copy flat -> amd64/ -# ./migrate-gcs-arch.sh --delete-old # dry-run: show what old flat files would be deleted -# ./migrate-gcs-arch.sh --delete-old --apply # actually delete old flat files -# -# Recommended workflow: -# 1. ./migrate-gcs-arch.sh gs://my-bucket # review what will be copied -# 2. ./migrate-gcs-arch.sh gs://my-bucket --apply # copy to amd64/ -# 3. ... verify everything works ... -# 4. ./migrate-gcs-arch.sh gs://my-bucket --delete-old # review what will be deleted -# 5. ./migrate-gcs-arch.sh gs://my-bucket --delete-old --apply # clean up old flat files +# ./scripts/migrate-gcs-arch.sh # dry-run: show what would be copied +# ./scripts/migrate-gcs-arch.sh --apply # copy flat -> amd64/ set -euo pipefail -BUCKET="${1:?Usage: $0 [--apply] [--delete-old]}" +BUCKET="${1:?Usage: $0 [--apply]}" shift APPLY=false -DELETE_OLD=false for arg in "$@"; do case "$arg" in - --apply) APPLY=true ;; - --delete-old) DELETE_OLD=true ;; + --apply) APPLY=true ;; *) echo "Unknown flag: $arg"; exit 1 ;; esac done @@ -57,48 +46,27 @@ if [[ -z "$objects" ]]; then fi count=0 -if [[ "$DELETE_OLD" == true ]]; then - while IFS= read -r src; do - [[ -z "$src" ]] && continue +while IFS= read -r src; do + [[ -z "$src" ]] && continue - if [[ "$APPLY" == true ]]; then - echo " DELETE $src" - gsutil rm "$src" - else - echo " [dry-run] would delete $src" - fi - ((count++)) || true - done <<< "$objects" + # Insert /amd64 before the filename: + # .../firecrackers/v1.10.1/firecracker -> .../firecrackers/v1.10.1/amd64/firecracker + dst="${src%/firecracker}/amd64/firecracker" - echo "" - echo "Total: $count objects" - if [[ "$APPLY" != true ]]; then - echo "" - echo "This was a dry run. Add --apply to actually delete." + if [[ "$APPLY" == true ]]; then + echo " COPY $src" + echo " -> $dst" + gsutil cp "$src" "$dst" + else + echo " [dry-run] $src" + echo " -> $dst" fi -else - while IFS= read -r src; do - [[ -z "$src" ]] && continue - - # Insert /amd64 before the filename: - # .../firecrackers/v1.10.1/firecracker -> .../firecrackers/v1.10.1/amd64/firecracker - dst="${src%/firecracker}/amd64/firecracker" - - if [[ "$APPLY" == true ]]; then - echo " COPY $src" - echo " -> $dst" - gsutil cp "$src" "$dst" - else - echo " [dry-run] $src" - echo " -> $dst" - fi - ((count++)) || true - done <<< "$objects" + ((count++)) || true +done <<< "$objects" +echo "" +echo "Total: $count objects" +if [[ "$APPLY" != true ]]; then echo "" - echo "Total: $count objects" - if [[ "$APPLY" != true ]]; then - echo "" - echo "This was a dry run. Add --apply to actually copy." - fi + echo "This was a dry run. Add --apply to actually copy." fi From cbea1a20f763b58df3e3b8572af16b26f43aca1c Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 9 Apr 2026 21:34:23 -0700 Subject: [PATCH 4/6] fix: don't hardcode firecrackers/ prefix, accept full GCS path --- scripts/migrate-gcs-arch.sh | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/scripts/migrate-gcs-arch.sh b/scripts/migrate-gcs-arch.sh index 26d09f1ba..e220a046b 100755 --- a/scripts/migrate-gcs-arch.sh +++ b/scripts/migrate-gcs-arch.sh @@ -2,21 +2,25 @@ # Copies flat-path firecracker binaries into amd64/ subdirectories in GCS. # # Before ARM64 support was added, builds were uploaded as: -# firecrackers/{version_name}/firecracker +# {version_name}/firecracker # # The arch-aware layout expects: -# firecrackers/{version_name}/amd64/firecracker +# {version_name}/amd64/firecracker # # This script finds flat-path binaries and copies them into the amd64/ subdir # so the orchestrator's arch-based path resolution can find them. # # Usage: -# ./scripts/migrate-gcs-arch.sh # dry-run: show what would be copied -# ./scripts/migrate-gcs-arch.sh --apply # copy flat -> amd64/ +# ./scripts/migrate-gcs-arch.sh # dry-run: show what would be copied +# ./scripts/migrate-gcs-arch.sh --apply # copy flat -> amd64/ +# +# Examples: +# ./scripts/migrate-gcs-arch.sh gs://e2b-staging-fc-versions +# ./scripts/migrate-gcs-arch.sh gs://e2b-prod-public-builds/firecrackers --apply set -euo pipefail -BUCKET="${1:?Usage: $0 [--apply]}" +GCS_PATH="${1:?Usage: $0 [--apply]}" shift APPLY=false @@ -27,21 +31,19 @@ for arg in "$@"; do esac done -# Strip gs:// prefix if provided, we add it back -BUCKET="${BUCKET#gs://}" - -GCS_PREFIX="firecrackers" +# Normalize: strip trailing slash +GCS_PATH="${GCS_PATH%/}" -echo "Scanning gs://${BUCKET}/${GCS_PREFIX} for flat-path firecracker binaries..." +echo "Scanning ${GCS_PATH} for flat-path firecracker binaries..." echo "" -# Match only flat-path binaries: firecrackers/{version}/firecracker +# Match only flat-path binaries: {version}/firecracker # The single * does NOT match path separators, so this excludes -# firecrackers/{version}/{arch}/firecracker. -objects=$(gsutil ls "gs://${BUCKET}/${GCS_PREFIX}/*/firecracker" 2>/dev/null || true) +# {version}/{arch}/firecracker. +objects=$(gsutil ls "${GCS_PATH}/*/firecracker" 2>/dev/null || true) if [[ -z "$objects" ]]; then - echo "No flat-path firecracker binaries found in gs://${BUCKET}/${GCS_PREFIX}" + echo "No flat-path firecracker binaries found in ${GCS_PATH}" exit 0 fi @@ -50,7 +52,7 @@ while IFS= read -r src; do [[ -z "$src" ]] && continue # Insert /amd64 before the filename: - # .../firecrackers/v1.10.1/firecracker -> .../firecrackers/v1.10.1/amd64/firecracker + # .../v1.10.1/firecracker -> .../v1.10.1/amd64/firecracker dst="${src%/firecracker}/amd64/firecracker" if [[ "$APPLY" == true ]]; then From 12bc1ef4802cd13311fc17116a8ab78450cd71b1 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 9 Apr 2026 21:40:36 -0700 Subject: [PATCH 5/6] skip versions that already have amd64/firecracker --- scripts/migrate-gcs-arch.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/migrate-gcs-arch.sh b/scripts/migrate-gcs-arch.sh index e220a046b..533d33953 100755 --- a/scripts/migrate-gcs-arch.sh +++ b/scripts/migrate-gcs-arch.sh @@ -47,7 +47,8 @@ if [[ -z "$objects" ]]; then exit 0 fi -count=0 +copied=0 +skipped=0 while IFS= read -r src; do [[ -z "$src" ]] && continue @@ -55,6 +56,13 @@ while IFS= read -r src; do # .../v1.10.1/firecracker -> .../v1.10.1/amd64/firecracker dst="${src%/firecracker}/amd64/firecracker" + # Skip if destination already exists + if gsutil -q stat "$dst" 2>/dev/null; then + echo " SKIP $dst (already exists)" + ((skipped++)) || true + continue + fi + if [[ "$APPLY" == true ]]; then echo " COPY $src" echo " -> $dst" @@ -63,12 +71,12 @@ while IFS= read -r src; do echo " [dry-run] $src" echo " -> $dst" fi - ((count++)) || true + ((copied++)) || true done <<< "$objects" echo "" -echo "Total: $count objects" -if [[ "$APPLY" != true ]]; then +echo "Total: $copied to copy, $skipped skipped (already exist)" +if [[ "$APPLY" != true && "$copied" -gt 0 ]]; then echo "" echo "This was a dry run. Add --apply to actually copy." fi From 0cd7a296d6eb8e58cdc61e37ad53be3ca839c72f Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 12 May 2026 22:56:34 -0700 Subject: [PATCH 6/6] add upload-release-to-gcs.sh for uploading release assets to GCS --- scripts/upload-release-to-gcs.sh | 112 +++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100755 scripts/upload-release-to-gcs.sh diff --git a/scripts/upload-release-to-gcs.sh b/scripts/upload-release-to-gcs.sh new file mode 100755 index 000000000..0c7c8d5f5 --- /dev/null +++ b/scripts/upload-release-to-gcs.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Uploads firecracker-{amd64,arm64} assets from a fc-versions GitHub release +# to GCS at: +# ///firecracker +# //firecracker (legacy amd64 copy) +# +# Existing objects are never overwritten. +# +# Usage: +# ./scripts/upload-release-to-gcs.sh --tag --bucket [--dry-run] [--repo ] +# +# Options: +# --tag Release tag / version name (e.g. v1.14.1_af9c995). +# --bucket Target bucket (with optional path prefix), e.g. +# my-bucket +# my-bucket/firecrackers +# gs://my-bucket/firecrackers +# --repo GitHub repo (default: e2b-dev/fc-versions). +# --dry-run Print what would be uploaded without writing. +# -h, --help Show this help. + +set -euo pipefail + +REPO="e2b-dev/fc-versions" +TAG="" +BUCKET="" +DRY_RUN=false + +usage() { sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --tag) TAG="${2:?--tag needs a value}"; shift 2 ;; + --bucket) BUCKET="${2:?--bucket needs a value}"; shift 2 ;; + --repo) REPO="${2:?--repo needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage 0 ;; + *) echo "Unknown argument: $1" >&2; usage 1 ;; + esac +done + +[[ -n "$TAG" ]] || { echo "ERROR: --tag is required" >&2; usage 1; } +[[ -n "$BUCKET" ]] || { echo "ERROR: --bucket is required" >&2; usage 1; } + +command -v gh >/dev/null || { echo "ERROR: gh CLI not found" >&2; exit 1; } +command -v gcloud >/dev/null || { echo "ERROR: gcloud CLI not found" >&2; exit 1; } + +BUCKET="${BUCKET#gs://}" +BUCKET="${BUCKET%/}" +BUCKET_URI="gs://${BUCKET}" + +if ! gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then + echo "ERROR: release '$TAG' not found in $REPO" >&2 + exit 1 +fi + +echo "Release: $TAG" +echo "Target: ${BUCKET_URI}" +$DRY_RUN && echo "Mode: dry-run" + +ASSETS=() +while IFS= read -r line || [[ -n "$line" ]]; do + [[ -n "$line" ]] && ASSETS+=("$line") +done < <(gh release view "$TAG" --repo "$REPO" --json assets \ + --jq '.assets[].name') + +if [[ "${#ASSETS[@]}" -eq 0 ]]; then + echo "ERROR: release $TAG has no assets" >&2 + exit 1 +fi + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +uploaded=0 +skipped=0 +for asset in "${ASSETS[@]}"; do + if [[ "$asset" =~ ^firecracker-(amd64|arm64)$ ]]; then + arch="${BASH_REMATCH[1]}" + dst="${BUCKET_URI}/${TAG}/${arch}/firecracker" + elif [[ "$asset" == "firecracker" ]]; then + dst="${BUCKET_URI}/${TAG}/firecracker" + else + continue + fi + + if gcloud storage ls "$dst" >/dev/null 2>&1; then + echo " EXISTS $dst" + skipped=$((skipped + 1)) + continue + fi + + if $DRY_RUN; then + echo " WOULD $asset -> $dst" + uploaded=$((uploaded + 1)) + continue + fi + + echo " UPLOAD $asset -> $dst" + gh release download "$TAG" --repo "$REPO" \ + --pattern "$asset" --dir "$TMP_DIR" --clobber >/dev/null + gcloud storage cp "$TMP_DIR/$asset" "$dst" + rm -f "$TMP_DIR/$asset" + uploaded=$((uploaded + 1)) +done + +echo "" +if $DRY_RUN; then + echo "Dry run complete. Would upload: $uploaded, already in GCS: $skipped." +else + echo "Done. Uploaded: $uploaded, already in GCS: $skipped." +fi