fix: paginate and diagnose GitHub release discovery

.github/workflows/smoke.yml

@@ -241,6 +241,9 @@ jobs:
           bun test diagnostics.test.ts
           bunx playwright install-deps chromium
 
+      - name: Release discovery tests
+        run: ./tests/release.sh
+
       # Fail loudly if we've drifted off "latest": a committed lock file, a moq
       # package that stopped being requested at latest, or a stale playwright pin.
       - name: Freshness

freshness.sh

@@ -145,14 +145,16 @@ else
     fail=1
 fi
 # C: smoke.sh resolves the newest libmoq-v* release, never a fixed version.
-if grep -q "grep '\^libmoq-v' | head -1" smoke.sh; then
+# shellcheck disable=SC2016
+if grep -qF '"$SMOKE_DIR/release.sh" libmoq-v' smoke.sh; then
     note ok "libmoq -> latest release"
 else
     note FAIL "smoke.sh no longer resolves the latest libmoq-v* release"
     fail=1
 fi
 # GStreamer: smoke.sh resolves the newest moq-gst-v* release, never a fixed version.
-if grep -q "grep '\^moq-gst-v' | head -1" smoke.sh; then
+# shellcheck disable=SC2016
+if grep -qF '"$SMOKE_DIR/release.sh" moq-gst-v' smoke.sh; then
     note ok "moq-gst -> latest release"
 else
     note FAIL "smoke.sh no longer resolves the latest moq-gst-v* release"

justfile

@@ -85,7 +85,7 @@ negative *args:
 freshness:
     ./freshness.sh
 
-# Lint the harness. shfmt discovers every shell script (incl. the extensionless
+# Lint the harness and test release discovery. shfmt discovers every shell script (incl. the extensionless
 # clients/docker/* wrappers) and reads .editorconfig for style; shellcheck lints
 # the same set. Each tool is guarded so it skips when not on PATH (nix develop
 # provides them all).
@@ -94,6 +94,7 @@ check:
     @if command -v shellcheck >/dev/null 2>&1 && command -v shfmt >/dev/null 2>&1; then shellcheck $(shfmt -f .); else echo "shellcheck: skipped"; fi
     @command -v actionlint >/dev/null && actionlint || echo "actionlint: skipped"
     just freshness
+    ./tests/release.sh
 
 # Compile-check and unit-test the local integrity client against the current
 # cloudflare/moq-rs Git dependencies. The end-to-end relay check is `just cloudflare`.

release.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Print the newest stable MoQ release with the requested tag prefix as JSON.
+set -euo pipefail
+
+case "${1:-}" in
+    libmoq-v | moq-gst-v) prefix=$1 ;;
+    *)
+        echo "usage: $0 {libmoq-v|moq-gst-v}" >&2
+        exit 2
+        ;;
+esac
+[[ $# -eq 1 ]] || {
+    echo "expected one release prefix" >&2
+    exit 2
+}
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+hdr=()
+[[ -z "${GITHUB_TOKEN:-}" ]] || hdr=(-H "Authorization: Bearer $GITHUB_TOKEN")
+page=1
+while :; do
+    : >"$work/headers"
+    : >"$work/body"
+    # Keep HTTP failures distinct from a valid list with no matching release.
+    # The empty-array expansion also works with macOS's bash 3.2 and set -u.
+    if status=$(curl -sS -L --fail-with-body --retry 3 --retry-all-errors \
+        ${hdr[@]+"${hdr[@]}"} -D "$work/headers" -o "$work/body" -w '%{http_code}' \
+        "https://api.github.com/repos/moq-dev/moq/releases?per_page=100&page=$page"); then
+        fetched=1
+    else
+        fetched=0
+    fi
+    echo "GitHub releases page $page: HTTP $status" >&2
+    # Response headers contain no credentials. Retain the request ID and rate
+    # limit evidence so a failed lookup can be investigated from its CI log.
+    grep -iE '^(x-github-request-id|x-ratelimit-[^:]+|link):' "$work/headers" >&2 || true
+    if [[ $fetched -ne 1 || "$status" != 200 ]]; then
+        echo "failed to query $prefix releases" >&2
+        head -c 4096 "$work/body" >&2
+        exit 1
+    fi
+    if ! jq -es 'length == 1 and (.[0] | type == "array" and all(.[];
+        type == "object" and (.tag_name | type == "string") and
+        (.draft | type == "boolean") and (.prerelease | type == "boolean")))' \
+        "$work/body" >/dev/null; then
+        echo "invalid GitHub releases response for $prefix" >&2
+        head -c 4096 "$work/body" >&2
+        exit 1
+    fi
+    release=$(jq -c --arg prefix "$prefix" \
+        'first(.[] | select(.draft == false and .prerelease == false and
+        (.tag_name | startswith($prefix)))) // empty' "$work/body")
+    if [[ -n "$release" ]]; then
+        printf '%s\n' "$release"
+        exit 0
+    fi
+    echo "No $prefix match on page $page; received tags:" >&2
+    jq -c '[.[].tag_name]' "$work/body" >&2
+    # A short page can still have a next link. Follow GitHub's pagination
+    # signal instead of assuming every response contains exactly 100 releases.
+    if ! grep -qi '^link:.*rel="next"' "$work/headers"; then
+        echo "no $prefix* release found after $page page(s)" >&2
+        exit 1
+    fi
+    page=$((page + 1))
+done

smoke.sh

@@ -233,7 +233,7 @@ run_swift() {
 # ships include/moq.h, lib/libmoq.a, lib/pkgconfig/moq.pc and lib/cmake/moq/.
 libmoq_fetch() {
     [[ -n "$LIBMOQ_ROOT" && -d "$LIBMOQ_ROOT" ]] && return 0
-    local target tag ver url hdr=()
+    local target tag ver url release
     have curl || {
         echo "curl required" >&2
         return 1
@@ -258,17 +258,8 @@ libmoq_fetch() {
         # libva symbols and a libva-less host just falls back to another encoder.
         *) LIBMOQ_OS_LIBS="-lpthread -ldl -lm -lstdc++" ;;
     esac
-    # Latest libmoq-v* release, never pinned. Authenticated if a token is around
-    # (CI) to dodge the 60/hr anonymous GitHub API limit.
-    [[ -n "${GITHUB_TOKEN:-}" ]] && hdr=(-H "Authorization: Bearer $GITHUB_TOKEN")
-    # macOS ships bash 3.2, where "${hdr[@]}" on an empty array trips `set -u`;
-    # the ${arr[@]+...} guard expands to nothing when the array is unset/empty.
-    tag=$(curl -sf --retry 3 --retry-all-errors ${hdr[@]+"${hdr[@]}"} "https://api.github.com/repos/moq-dev/moq/releases?per_page=100" |
-        jq -r '.[].tag_name' | grep '^libmoq-v' | head -1)
-    [[ -n "$tag" ]] || {
-        echo "no libmoq-v* release found" >&2
-        return 1
-    }
+    release=$("$SMOKE_DIR/release.sh" libmoq-v) || return 1
+    tag=$(jq -er '.tag_name' <<<"$release") || return 1
     ver=${tag#libmoq-v}
     url="https://github.com/moq-dev/moq/releases/download/$tag/moq-$ver-$target.tar.gz"
     echo "libmoq $tag ($target)"
@@ -349,7 +340,7 @@ c_build_cmake() {
 # needs. (Local nix shells have no GStreamer, and a prebuilt plugin wouldn't link
 # against nixpkgs' gst anyway, so this cell wants a real system gst install.)
 gst_prepare() {
-    local target tag asset_name url root releases hdr=()
+    local target tag asset_name url root release
     have gst-launch-1.0 || {
         echo "gst-launch-1.0 not on PATH (apt install gstreamer1.0-tools / brew install gstreamer)" >&2
         return 1
@@ -395,34 +386,23 @@ gst_prepare() {
             return 1
             ;;
     esac
-    # Latest moq-gst-v* release, never pinned. Authenticated if a token is around
-    # (CI) to dodge the 60/hr anonymous GitHub API limit.
-    [[ -n "${GITHUB_TOKEN:-}" ]] && hdr=(-H "Authorization: Bearer $GITHUB_TOKEN")
-    releases="$TMP/moq-gst-releases.json"
-    curl -sf --retry 3 --retry-all-errors ${hdr[@]+"${hdr[@]}"} \
-        "https://api.github.com/repos/moq-dev/moq/releases?per_page=100" -o "$releases" || {
-        echo "failed to query moq-gst releases" >&2
-        return 1
-    }
-    tag=$(jq -r '.[].tag_name' "$releases" | grep '^moq-gst-v' | head -1)
-    [[ -n "$tag" ]] || {
-        echo "no moq-gst-v* release found" >&2
-        return 1
-    }
+    release="$TMP/moq-gst-release.json"
+    "$SMOKE_DIR/release.sh" moq-gst-v >"$release" || return 1
+    tag=$(jq -er '.tag_name' "$release") || return 1
     # Release asset names are part of the release metadata, not a derivation of
     # the crate version. In particular, 0.3.7 added the tag's `v` to the archive
     # and extracted-directory name. Select the platform asset that was actually
     # published so future naming changes fail as a missing asset, not a guessed URL.
-    asset_name=$(jq -r --arg tag "$tag" --arg suffix "-$target.tar.gz" \
-        '.[] | select(.tag_name == $tag) | .assets[].name |
-        select(startswith("moq-gst-") and endswith($suffix))' "$releases" | head -1)
+    asset_name=$(jq -r --arg suffix "-$target.tar.gz" \
+        '.assets[].name |
+        select(startswith("moq-gst-") and endswith($suffix))' "$release" | head -1)
     [[ -n "$asset_name" ]] || {
         echo "no moq-gst asset for $target in release $tag" >&2
         return 1
     }
-    url=$(jq -r --arg tag "$tag" --arg name "$asset_name" \
-        '.[] | select(.tag_name == $tag) | .assets[] |
-        select(.name == $name) | .browser_download_url' "$releases" | head -1)
+    url=$(jq -r --arg name "$asset_name" \
+        '.assets[] |
+        select(.name == $name) | .browser_download_url' "$release" | head -1)
     [[ -n "$url" ]] || {
         echo "moq-gst asset has no download URL: $asset_name" >&2
         return 1

tests/release.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Exercise release discovery without registry access or client toolchains.
+set -euo pipefail
+script=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/release.sh
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+export RELEASE_FIXTURES="$work/responses"
+mkdir -p "$work/bin" "$RELEASE_FIXTURES"
+cat >"$work/bin/curl" <<'CURL'
+#!/usr/bin/env bash
+set -euo pipefail
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -D) headers=$2; shift ;;
+        -o) body=$2; shift ;;
+        https://api.github.com/repos/moq-dev/moq/releases\?*) page=${1##*&page=} ;;
+    esac
+    shift
+done
+cp "$RELEASE_FIXTURES/$page.headers" "$headers"
+cp "$RELEASE_FIXTURES/$page.json" "$body"
+printf '%s' "${RELEASE_STATUS:-200}"
+exit "${RELEASE_CURL_EXIT:-0}"
+CURL
+chmod +x "$work/bin/curl"
+export PATH="$work/bin:$PATH"
+# Do not send a developer's token to the fixture command.
+unset GITHUB_TOKEN RELEASE_STATUS RELEASE_CURL_EXIT
+
+reset_pages() {
+    rm -f "$RELEASE_FIXTURES"/*
+    printf 'X-GitHub-Request-Id: fixture-id\r\n' >"$RELEASE_FIXTURES/1.headers"
+}
+expect_tag() {
+    local prefix=$1 expected=$2
+    "$script" "$prefix" >"$work/out" 2>"$work/err" || {
+        cat "$work/err" >&2
+        exit 1
+    }
+    [[ $(jq -r '.tag_name' "$work/out") == "$expected" ]]
+    jq -e '.assets[0].browser_download_url == "https://example.com/asset.tar.gz"' "$work/out" >/dev/null
+}
+expect_error() {
+    if "$script" libmoq-v >"$work/out" 2>"$work/err"; then
+        echo "expected release lookup to fail: $1" >&2
+        exit 1
+    fi
+    [[ ! -s "$work/out" ]]
+    grep -qF "$1" "$work/err"
+    grep -qF 'fixture-id' "$work/err"
+}
+
+reset_pages
+jq -n '[{tag_name:"libmoq-v9",draft:true,prerelease:false},
+    {tag_name:"libmoq-v8-rc1",draft:false,prerelease:true}] +
+    ["libmoq-v7", "moq-gst-v6", "libmoq-v5" | {tag_name:.,draft:false,prerelease:false,
+    assets:[{browser_download_url:"https://example.com/asset.tar.gz"}]}]' >"$RELEASE_FIXTURES/1.json"
+expect_tag libmoq-v libmoq-v7
+expect_tag moq-gst-v moq-gst-v6
+echo 'ok: selects the first stable matching release and preserves its assets'
+
+cp "$RELEASE_FIXTURES/1.json" "$RELEASE_FIXTURES/2.json"
+cp "$RELEASE_FIXTURES/1.headers" "$RELEASE_FIXTURES/2.headers"
+printf 'link: <https://api.github.com/repositories/508763694/releases?per_page=100&page=2>; rel="next"\r\n' >>"$RELEASE_FIXTURES/1.headers"
+jq -n '[range(100) | {tag_name:("other-v" + tostring),draft:false,prerelease:false}]' >"$RELEASE_FIXTURES/1.json"
+expect_tag libmoq-v libmoq-v7
+expect_tag moq-gst-v moq-gst-v6
+echo 'ok: finds both clients beyond the first 100 releases'
+printf '[]' >"$RELEASE_FIXTURES/1.json"
+expect_tag libmoq-v libmoq-v7
+echo 'ok: follows a next link even on a short page'
+printf '[]' >"$RELEASE_FIXTURES/2.json"
+expect_error 'no libmoq-v* release found after 2 page(s)'
+echo 'ok: reports absence only after exhausting the pages'
+
+reset_pages
+: >"$RELEASE_FIXTURES/1.json"
+expect_error 'invalid GitHub releases response'
+printf '[][]' >"$RELEASE_FIXTURES/1.json"
+expect_error 'invalid GitHub releases response'
+printf '{' >"$RELEASE_FIXTURES/1.json"
+expect_error 'invalid GitHub releases response'
+printf '{"message":"not an array"}' >"$RELEASE_FIXTURES/1.json"
+expect_error 'invalid GitHub releases response'
+printf '[{"tag_name":123}]' >"$RELEASE_FIXTURES/1.json"
+expect_error 'invalid GitHub releases response'
+echo 'ok: rejects empty, malformed, and unsupported responses'
+
+printf '{"message":"API rate limit exceeded"}' >"$RELEASE_FIXTURES/1.json"
+export RELEASE_STATUS=403 RELEASE_CURL_EXIT=22
+expect_error 'failed to query libmoq-v releases'
+grep -qF 'HTTP 403' "$work/err"
+grep -qF 'API rate limit exceeded' "$work/err"
+export RELEASE_STATUS=000 RELEASE_CURL_EXIT=6
+expect_error 'failed to query libmoq-v releases'
+export RELEASE_STATUS=302 RELEASE_CURL_EXIT=0
+expect_error 'failed to query libmoq-v releases'
+echo 'ok: distinguishes HTTP and transport failures from absent releases'