diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cac604d..6f9c3993 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -389,12 +389,13 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # --------------------------------------------------------------------------- - # Docs — deploy to Vercel (only on push, not PRs) + # Docs — deploy to Vercel (always, so e2e_cdn can test the live CDN) # --------------------------------------------------------------------------- docs: name: Deploy Docs - if: ${{ github.event_name == 'push' }} runs-on: ubuntu-24.04-arm + outputs: + deployment_url: ${{ steps.deploy.outputs.url }} steps: - uses: actions/checkout@v5 @@ -403,14 +404,18 @@ jobs: run: npm install -g vercel - name: Pull Vercel config - run: vercel pull --yes --environment=production --token=${{ secrets.VERCEL_TOKEN }} + run: | + ENV=${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + vercel pull --yes --environment=$ENV --token=${{ secrets.VERCEL_TOKEN }} working-directory: docs env: VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - name: Build - run: vercel build --prod --token=${{ secrets.VERCEL_TOKEN }} + run: | + PROD=${{ github.ref == 'refs/heads/main' && '--prod' || '' }} + vercel build $PROD --token=${{ secrets.VERCEL_TOKEN }} working-directory: docs env: VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} @@ -428,13 +433,55 @@ jobs: working-directory: docs/slides - name: Deploy - run: vercel deploy --prebuilt --prod --token=${{ secrets.VERCEL_TOKEN }} + id: deploy + run: | + PROD=${{ github.ref == 'refs/heads/main' && '--prod' || '' }} + BRANCH_SLUG="${GITHUB_REF_NAME//\//-}" + url=$(vercel deploy --prebuilt $PROD \ + --meta gitBranch="$GITHUB_REF_NAME" \ + --meta gitCommit="$GITHUB_SHA" \ + --token=${{ secrets.VERCEL_TOKEN }}) + echo "url=$url" >> "$GITHUB_OUTPUT" + echo "### Vercel deployment" >> "$GITHUB_STEP_SUMMARY" + echo "**Branch:** \`$GITHUB_REF_NAME\`" >> "$GITHUB_STEP_SUMMARY" + echo "**URL:** $url" >> "$GITHUB_STEP_SUMMARY" + echo "**CDN:** $url/stripe-api-specs/manifest.json" >> "$GITHUB_STEP_SUMMARY" + echo "" + echo "Deployed: $url (branch: $GITHUB_REF_NAME)" working-directory: docs env: VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - # Deployed to: https://stripe-sync-engine.dev/ + # main → https://stripe-sync.dev/ | branch → unique preview URL + + # --------------------------------------------------------------------------- + # E2E CDN — verify stripe-sync.dev/stripe-api-specs after Vercel deploy + # --------------------------------------------------------------------------- + e2e_cdn: + name: E2E CDN + needs: [docs] + runs-on: ubuntu-24.04-arm + + steps: + - uses: actions/checkout@v5 + + - name: Install pnpm + uses: pnpm/action-setup@v5 + + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: ./.nvmrc + cache: pnpm + + - name: Install dependencies & build + run: pnpm install --frozen-lockfile && pnpm build + + - name: CDN e2e tests + run: pnpm --filter @stripe/sync-e2e run test -- openapi-cdn.test.ts + env: + STRIPE_SPEC_CDN_BASE_URL: ${{ needs.docs.outputs.deployment_url }}/stripe-api-specs # --------------------------------------------------------------------------- # Docker Hub — promote the built GHCR image to Docker Hub tags diff --git a/docs/build.mjs b/docs/build.mjs index 64fef2b2..49ec4219 100644 --- a/docs/build.mjs +++ b/docs/build.mjs @@ -1,6 +1,7 @@ import Markdoc from '@markdoc/markdoc' import fs from 'node:fs' import path from 'node:path' +import { spawnSync } from 'node:child_process' const ROOT = path.dirname(new URL(import.meta.url).pathname) const PAGES_DIR = ROOT @@ -26,21 +27,34 @@ fs.mkdirSync(OUT_DIR, { recursive: true }) // Copy public assets if (fs.existsSync(PUBLIC_DIR)) copyDir(PUBLIC_DIR, OUT_DIR) -// Copy OpenAPI specs +// Copy Sync Engine OpenAPI specs (engine/service/webhook) → /openapi/ if (fs.existsSync(OPENAPI_DIR)) copyDir(OPENAPI_DIR, path.join(OUT_DIR, 'openapi')) -// Collect all .md files recursively, skipping non-content dirs -const SKIP_DIRS = new Set(['node_modules', 'out', 'openapi', 'public', 'slides']) +// Generate official Stripe API specs (from stripe/openapi) → /stripe-api-specs/ +// These are the upstream Stripe REST API specs, NOT the Sync Engine API. +// Served as a CDN mirror so consumers avoid GitHub rate limits. +// Skipped when SKIP_STRIPE_SPECS=1 (e.g. quick local builds). +if (process.env.SKIP_STRIPE_SPECS !== '1') { + console.log('Generating Stripe API specs...') + const stripeSpecDir = path.join(OUT_DIR, 'stripe-api-specs') + const result = spawnSync( + process.execPath, + [path.join(ROOT, 'scripts', 'generate-stripe-specs.mjs'), stripeSpecDir], + { stdio: 'inherit' } + ) + if (result.status !== 0) { + console.error('Warning: Stripe API spec generation failed — CDN specs will be unavailable') + } +} +// Collect all .md files recursively under PAGES_DIR function collectPages(dir, base = '') { const entries = fs.readdirSync(dir, { withFileTypes: true }) const pages = [] for (const entry of entries) { const rel = base ? `${base}/${entry.name}` : entry.name if (entry.isDirectory()) { - if (!SKIP_DIRS.has(entry.name)) { - pages.push(...collectPages(path.join(dir, entry.name), rel)) - } + pages.push(...collectPages(path.join(dir, entry.name), rel)) } else if (entry.name.endsWith('.md')) { pages.push(rel) } diff --git a/docs/index.md b/docs/index.md index 65dac38b..0c7164ca 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,6 +36,7 @@ npx @stripe/sync-engine sync \ ## API Reference - [Engine API](/engine.html) — HTTP API for running syncs +- [Stripe API Specs](/stripe-api-specs/) — CDN mirror of official Stripe OpenAPI specs (all versions) ## Slides diff --git a/docs/scripts/generate-stripe-specs.mjs b/docs/scripts/generate-stripe-specs.mjs new file mode 100644 index 00000000..569506d8 --- /dev/null +++ b/docs/scripts/generate-stripe-specs.mjs @@ -0,0 +1,142 @@ +#!/usr/bin/env node +/** + * Fetches all published Stripe REST API spec versions from + * github.com/stripe/openapi and writes .json + manifest.json to . + * + * Usage: + * node generate-stripe-specs.mjs + * + * Clones stripe/openapi (single-branch) then walks the full history, collecting + * every unique API version (deduplicated by blob SHA, then by version string). + * Set STRIPE_OPENAPI_REPO to a pre-cloned path to skip the clone (e.g. CI cache). + * + * These are the official Stripe REST API specs (github.com/stripe/openapi), NOT + * the Sync Engine's own OpenAPI spec (which lives at /openapi/engine.json etc.). + * + * No npm dependencies. + */ +import { writeFileSync, mkdirSync, existsSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { execFileSync } from 'node:child_process' + +const [outputDir] = process.argv.slice(2) +if (!outputDir) { + console.error('Usage: node generate-stripe-specs.mjs ') + process.exit(1) +} + +const REPO_URL = 'https://github.com/stripe/openapi' +// stripe/openapi uses 'latest/openapi.spec3.sdk.json' for recent specs and +// 'openapi/spec3.json' for historic ones. +const SPEC_PATHS = ['latest/openapi.spec3.sdk.json', 'openapi/spec3.json'] + +function git(...args) { + // maxBuffer: Stripe specs are ~10 MB each; default 1 MB would silently truncate/throw. + return execFileSync('git', ['-C', repoDir, ...args], { + encoding: 'utf8', + maxBuffer: 50 * 1024 * 1024, + }) +} + +// Clone or use pre-cloned repo +const repoDir = process.env.STRIPE_OPENAPI_REPO ?? join(tmpdir(), 'stripe-openapi') +if (!existsSync(join(repoDir, '.git'))) { + console.error(`Cloning ${REPO_URL}...`) + execFileSync('git', ['clone', '--single-branch', REPO_URL, repoDir], { stdio: 'inherit' }) +} else { + console.error(`Using pre-cloned repo at ${repoDir}`) +} + +// Walk commits newest→oldest, collect all unique API versions. +console.error('Collecting all spec versions...') +const commits = git('log', '--format=%H', '--', ...SPEC_PATHS) + .trim() + .split('\n') + .filter(Boolean) + +mkdirSync(outputDir, { recursive: true }) + +const seen = new Map() // version -> filename +const seenBlobs = new Set() + +for (const commit of commits) { + let blobSha + for (const specPath of SPEC_PATHS) { + let ls + try { + ls = git('ls-tree', commit, specPath).trim() + } catch { + continue + } + if (!ls) continue + blobSha = ls.split(/\s+/)[2] + break + } + if (!blobSha || seenBlobs.has(blobSha)) continue + seenBlobs.add(blobSha) + + let raw + try { + raw = git('cat-file', 'blob', blobSha) + } catch { + continue + } + + let version + try { + version = JSON.parse(raw).info?.version + } catch { + continue + } + if (!version || seen.has(version)) continue + + writeFileSync(join(outputDir, `${version}.json`), raw) + seen.set(version, `${version}.json`) + console.error(` ${version}`) +} + +writeFileSync( + join(outputDir, 'manifest.json'), + JSON.stringify(Object.fromEntries(seen), null, 2) + '\n' +) + +// Generate an index page so https://stripe-sync.dev/stripe-api-specs/ is browsable +const versions = [...seen.keys()].sort().reverse() +const rows = versions.map((v) => `
  • ${v}
    • `).join('\n') +writeFileSync( + join(outputDir, 'index.html'), + ` + + + + Stripe REST API Specs — stripe-sync.dev CDN + + + +

    Stripe REST API OpenAPI Specs

    +

    + These are the official Stripe REST API specs from + github.com/stripe/openapi, + mirrored here to avoid GitHub API rate limits. + This is not the Sync Engine's own OpenAPI spec + (see engine.json for that). +

    +

    Machine-readable index: manifest.json — ${versions.length} versions available.

    +
      +${rows} +
    + + +` +) + +console.error(`\nDone: ${seen.size} spec versions`) diff --git a/docs/vercel.json b/docs/vercel.json index 57494a21..e0b26ad0 100644 --- a/docs/vercel.json +++ b/docs/vercel.json @@ -1,4 +1,5 @@ { "buildCommand": "npm run build", - "outputDirectory": "out" + "outputDirectory": "out", + "trailingSlash": true } diff --git a/e2e/openapi-cdn.test.ts b/e2e/openapi-cdn.test.ts new file mode 100644 index 00000000..6d86bffb --- /dev/null +++ b/e2e/openapi-cdn.test.ts @@ -0,0 +1,79 @@ +/** + * E2E test: Stripe spec CDN at stripe-sync.dev/openapi/stripe + * + * Verifies that: + * 1. manifest.json is accessible and lists known spec versions + * 2. resolveOpenApiSpec returns source:'cdn' for a non-bundled version + * + * No env vars required — hits the live CDN directly. + * Tests are skipped automatically when the CDN isn't deployed yet (404). + * Once deployed, any failure here means the CDN is broken. + */ +import os from 'node:os' +import path from 'node:path' +import { beforeAll, describe, it, expect } from 'vitest' +import { resolveOpenApiSpec, BUNDLED_API_VERSION } from '@stripe/sync-openapi' + +const CDN_BASE = process.env.STRIPE_SPEC_CDN_BASE_URL ?? 'https://stripe-sync.dev/stripe-api-specs' + +describe('Stripe spec CDN', () => { + let manifest: Record | null = null + + beforeAll(async () => { + const res = await fetch(`${CDN_BASE}/manifest.json`).catch(() => null) + if (!res || !res.ok) { + console.warn( + `Skipping CDN tests — ${CDN_BASE}/manifest.json returned ${res?.status ?? 'no response'} (CDN not deployed yet)` + ) + return + } + manifest = (await res.json()) as Record + }) + + it('manifest.json is reachable and lists spec versions', () => { + if (!manifest) return // CDN not deployed yet + + const versions = Object.keys(manifest) + expect(versions.length, 'manifest should list at least one version').toBeGreaterThan(0) + + for (const [version, filename] of Object.entries(manifest)) { + expect(filename, `manifest entry for ${version}`).toMatch(/^.+\.json$/) + } + }) + + it('each manifest entry resolves to a valid OpenAPI spec', async () => { + if (!manifest) return // CDN not deployed yet + + // Spot-check the first 3 entries to keep the test fast + const entries = Object.entries(manifest).slice(0, 3) + for (const [version, filename] of entries) { + const specRes = await fetch(`${CDN_BASE}/${filename}`) + expect(specRes.status, `GET ${CDN_BASE}/${filename}`).toBe(200) + + const spec = (await specRes.json()) as Record + expect(typeof spec.openapi, `spec ${version} missing openapi field`).toBe('string') + expect(spec.components, `spec ${version} missing components`).toBeTruthy() + } + }) + + it('resolveOpenApiSpec uses cdn source for non-bundled version', async () => { + if (!manifest) return // CDN not deployed yet + + // Find any version that isn't the bundled one + const nonBundled = Object.keys(manifest).find( + (v) => !v.startsWith(BUNDLED_API_VERSION.slice(0, 10)) + ) + if (!nonBundled) { + console.warn('Only the bundled version is in the CDN manifest — skipping cdn source check') + return + } + + // Use an isolated cache dir so we don't hit a pre-existing cache entry + const cacheDir = path.join(os.tmpdir(), `openapi-cdn-test-${Date.now()}`) + const result = await resolveOpenApiSpec({ apiVersion: nonBundled, cacheDir }, fetch) + + expect(result.source).toBe('cdn') + expect(result.apiVersion).toBe(nonBundled) + expect(typeof result.spec.openapi).toBe('string') + }) +}) diff --git a/e2e/package.json b/e2e/package.json index 188ad3e5..3beb1145 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -7,6 +7,7 @@ }, "devDependencies": { "@hono/node-server": "^1", + "@stripe/sync-openapi": "workspace:*", "@stripe/sync-protocol": "workspace:*", "@stripe/sync-engine": "workspace:*", "@stripe/sync-service": "workspace:*", diff --git a/packages/openapi/__tests__/specFetchHelper.test.ts b/packages/openapi/__tests__/specFetchHelper.test.ts index a6bb5a78..8cddea6a 100644 --- a/packages/openapi/__tests__/specFetchHelper.test.ts +++ b/packages/openapi/__tests__/specFetchHelper.test.ts @@ -9,6 +9,20 @@ async function createTempDir(prefix: string): Promise { return fs.mkdtemp(path.join(os.tmpdir(), `${prefix}-`)) } +// Mock fetch that returns 404 for CDN URLs (not deployed in tests) and +// handles GitHub API calls normally. +function makeFetchMock( + githubHandler: (url: string) => Response | Promise +): typeof globalThis.fetch { + return vi.fn(async (input: URL | string) => { + const url = String(input) + if (url.includes('stripe-sync.dev')) { + return new Response('not found', { status: 404 }) + } + return githubHandler(url) + }) as unknown as typeof globalThis.fetch +} + describe('resolveOpenApiSpec', () => { it('prefers explicit local spec path over cache and network', async () => { const tempDir = await createTempDir('openapi-explicit') @@ -52,8 +66,7 @@ describe('resolveOpenApiSpec', () => { it('fetches from GitHub when cache misses and persists cache', async () => { const tempDir = await createTempDir('openapi-fetch') - const fetchMock = vi.fn(async (input: URL | string) => { - const url = String(input) + const fetchMock = makeFetchMock((url) => { if (url.includes('/commits')) { return new Response(JSON.stringify([{ sha: 'abc123def456' }]), { status: 200 }) } @@ -73,14 +86,14 @@ describe('resolveOpenApiSpec', () => { const cached = await fs.readFile(path.join(tempDir, '2020-08-27.spec3.sdk.json'), 'utf8') expect(JSON.parse(cached)).toMatchObject({ openapi: '3.0.0' }) - expect(fetchMock).toHaveBeenCalledTimes(2) + // 1 CDN manifest (404) + 1 GitHub commits + 1 GitHub spec + expect(fetchMock).toHaveBeenCalledTimes(3) await fs.rm(tempDir, { recursive: true, force: true }) }) it('uses the injected fetch for GitHub fetches', async () => { const tempDir = await createTempDir('openapi-fetch-proxy') - const fetchMock = vi.fn(async (input: URL | string) => { - const url = String(input) + const fetchMock = makeFetchMock((url) => { if (url.includes('/commits')) { return new Response(JSON.stringify([{ sha: 'abc123def456' }]), { status: 200 }) } @@ -92,7 +105,8 @@ describe('resolveOpenApiSpec', () => { fetchMock ) expect(result.source).toBe('github') - expect(fetchMock).toHaveBeenCalledTimes(2) + // 1 CDN manifest (404) + 1 GitHub commits + 1 GitHub spec + expect(fetchMock).toHaveBeenCalledTimes(3) } finally { await fs.rm(tempDir, { recursive: true, force: true }) } @@ -117,6 +131,7 @@ describe('resolveOpenApiSpec', () => { it('fails fast when GitHub resolution fails and no explicit spec path is set', async () => { const tempDir = await createTempDir('openapi-fail-fast') + // CDN returns 500 → tryFetchFromCdn returns null → falls through to GitHub (500) → throws const fetchMock = vi.fn(async () => new Response('boom', { status: 500 })) await expect( diff --git a/packages/openapi/index.ts b/packages/openapi/index.ts index 8976e358..4fad34cf 100644 --- a/packages/openapi/index.ts +++ b/packages/openapi/index.ts @@ -2,7 +2,7 @@ export type * from './types.js' export { SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES } from './specParser.js' export { OPENAPI_COMPATIBILITY_COLUMNS } from './runtimeMappings.js' -export { resolveOpenApiSpec } from './specFetchHelper.js' +export { resolveOpenApiSpec, BUNDLED_API_VERSION } from './specFetchHelper.js' export { discoverListEndpoints, discoverNestedEndpoints, diff --git a/packages/openapi/specFetchHelper.ts b/packages/openapi/specFetchHelper.ts index e576ed2b..d6caaccf 100644 --- a/packages/openapi/specFetchHelper.ts +++ b/packages/openapi/specFetchHelper.ts @@ -6,6 +6,13 @@ import type { OpenApiSpec, ResolveSpecConfig, ResolvedOpenApiSpec } from './type const DEFAULT_CACHE_DIR = path.join(os.tmpdir(), 'stripe-sync-openapi-cache') +// CDN mirror of the official Stripe REST API specs (from github.com/stripe/openapi). +// Served from stripe-sync.dev — no auth, no GitHub rate limits. +// These are the upstream Stripe API specs, NOT the Sync Engine's own OpenAPI spec. +// Override with STRIPE_SPEC_CDN_BASE_URL env var (e.g. in tests or self-hosting). +const STRIPE_SPEC_CDN_BASE_URL = + process.env.STRIPE_SPEC_CDN_BASE_URL ?? 'https://stripe-sync.dev/stripe-api-specs' + // The spec bundled into this package at build time. // Update this constant and bundled-spec.json together when bumping. export const BUNDLED_API_VERSION = '2026-03-25.dahlia' @@ -56,6 +63,19 @@ export async function resolveOpenApiSpec( } } + // Try the Vercel CDN mirror before falling back to the GitHub API. + // The CDN serves spec versions without auth or rate limits. + const cdnSpec = await tryFetchFromCdn(apiVersion, fetch) + if (cdnSpec) { + await tryWriteCache(cachePath, cdnSpec) + return { + apiVersion, + spec: cdnSpec, + source: 'cdn', + cachePath, + } + } + let commitSha = await resolveCommitShaForApiVersion(apiVersion, fetch) if (!commitSha) { commitSha = await resolveLatestCommitSha(fetch) @@ -113,6 +133,35 @@ async function tryWriteCache(cachePath: string, spec: OpenApiSpec): Promise { + // The CDN manifest maps "YYYY-MM-DD.codename" → "YYYY-MM-DD.codename.json". + // We match by date part so "2026-03-25" resolves to "2026-03-25.dahlia.json". + if (!STRIPE_SPEC_CDN_BASE_URL) return null + try { + const manifestUrl = `${STRIPE_SPEC_CDN_BASE_URL}/manifest.json` + const manifestRes = await fetch(manifestUrl) + if (!manifestRes.ok) return null + + const manifest = (await manifestRes.json()) as Record + const datePart = extractDatePart(apiVersion) + const filename = Object.keys(manifest).find((v) => extractDatePart(v) === datePart) + if (!filename) return null + + const specUrl = `${STRIPE_SPEC_CDN_BASE_URL}/${manifest[filename]}` + const specRes = await fetch(specUrl) + if (!specRes.ok) return null + + const spec = (await specRes.json()) as unknown + validateOpenApiSpec(spec) + return spec + } catch { + return null + } +} + function getCachePath(cacheDir: string, apiVersion: string): string { const safeVersion = apiVersion.replace(/[^0-9a-zA-Z_-]/g, '_') return path.join(cacheDir, `${safeVersion}.spec3.sdk.json`) diff --git a/packages/openapi/types.ts b/packages/openapi/types.ts index b9564440..8c72e471 100644 --- a/packages/openapi/types.ts +++ b/packages/openapi/types.ts @@ -114,7 +114,7 @@ export type ResolveSpecConfig = { export type ResolvedOpenApiSpec = { apiVersion: string spec: OpenApiSpec - source: 'explicit_path' | 'cache' | 'github' | 'bundled' + source: 'explicit_path' | 'cache' | 'cdn' | 'github' | 'bundled' cachePath?: string commitSha?: string } diff --git a/packages/source-stripe/src/index.ts b/packages/source-stripe/src/index.ts index 20d8d26b..5ed48501 100644 --- a/packages/source-stripe/src/index.ts +++ b/packages/source-stripe/src/index.ts @@ -10,6 +10,7 @@ import { buildResourceRegistry } from './resourceRegistry.js' import { catalogFromRegistry, catalogFromOpenApi } from './catalog.js' import { resolveOpenApiSpec, + BUNDLED_API_VERSION, SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES, } from '@stripe/sync-openapi' @@ -153,7 +154,7 @@ export function createStripeSource( async discover({ config }) { const resolved = await resolveOpenApiSpec( - { apiVersion: config.api_version ?? '2020-08-27' }, + { apiVersion: config.api_version ?? BUNDLED_API_VERSION }, apiFetch ) const registry = buildResourceRegistry( @@ -216,7 +217,7 @@ export function createStripeSource( externalRateLimiter ?? createInMemoryRateLimiter(config.rate_limit ?? DEFAULT_MAX_RPS) const stripe = makeClient(config) const resolved = await resolveOpenApiSpec( - { apiVersion: config.api_version ?? '2020-08-27' }, + { apiVersion: config.api_version ?? BUNDLED_API_VERSION }, apiFetch ) const registry = buildResourceRegistry( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12206b52..203fb376 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -291,6 +291,9 @@ importers: '@stripe/sync-engine': specifier: workspace:* version: link:../apps/engine + '@stripe/sync-openapi': + specifier: workspace:* + version: link:../packages/openapi '@stripe/sync-protocol': specifier: workspace:* version: link:../packages/protocol