diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1999b3d..6230f0b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: e2e: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 permissions: contents: write pull-requests: write @@ -65,6 +65,11 @@ jobs: - name: Install dependencies run: npm ci + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + - name: Install Playwright run: npx playwright install --with-deps chromium @@ -80,8 +85,19 @@ jobs: id: e2e continue-on-error: true env: + # Cloud infrastructure credentials (from repo secrets with E2E_ prefix) + CLOUDFLARE_API_TOKEN: ${{ secrets.E2E_CLOUDFLARE_API_TOKEN }} + CF_ACCOUNT_ID: ${{ secrets.E2E_CF_ACCOUNT_ID }} + WORKERS_SUBDOMAIN: ${{ secrets.E2E_WORKERS_SUBDOMAIN }} + CF_ACCESS_TEAM_DOMAIN: ${{ secrets.E2E_CF_ACCESS_TEAM_DOMAIN }} + R2_ACCESS_KEY_ID: ${{ secrets.E2E_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.E2E_R2_SECRET_ACCESS_KEY }} + # AI provider (optional, for chat tests) AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} AI_GATEWAY_BASE_URL: ${{ secrets.AI_GATEWAY_BASE_URL }} + # Unique test run ID for parallel isolation + E2E_TEST_RUN_ID: ${{ github.run_id }}-${{ matrix.config.name }} + # Matrix-specific config TELEGRAM_BOT_TOKEN: ${{ matrix.config.env.TELEGRAM_BOT_TOKEN }} TELEGRAM_DM_POLICY: ${{ matrix.config.env.TELEGRAM_DM_POLICY }} DISCORD_BOT_TOKEN: ${{ matrix.config.env.DISCORD_BOT_TOKEN }} diff --git a/.gitignore b/.gitignore index 8a01f626..fad19933 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,17 @@ Thumbs.db *.greger # playwright-cli -.playwright-cli/ \ No newline at end of file +.playwright-cli/ + +# Terraform +*.tfstate +*.tfstate.* +.terraform/ +.terraform.lock.hcl +terraform.tfvars + +# E2E test credentials +test/e2e/.dev.vars + +# Temporary e2e wrangler configs +.wrangler-e2e-*.jsonc \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 77e68fa7..2576af9b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,5 +11,10 @@ export const STARTUP_TIMEOUT_MS = 180_000; /** Mount path for R2 persistent storage inside the container */ export const R2_MOUNT_PATH = '/data/moltbot'; -/** R2 bucket name for persistent storage */ -export const R2_BUCKET_NAME = 'moltbot-data'; +/** + * R2 bucket name for persistent storage. + * Can be overridden via R2_BUCKET_NAME env var for test isolation. + */ +export function getR2BucketName(env?: { R2_BUCKET_NAME?: string }): string { + return env?.R2_BUCKET_NAME || 'moltbot-data'; +} diff --git a/src/gateway/r2.test.ts b/src/gateway/r2.test.ts index e4228dfa..ea2a2f2b 100644 --- a/src/gateway/r2.test.ts +++ b/src/gateway/r2.test.ts @@ -88,6 +88,25 @@ describe('mountR2Storage', () => { ); }); + it('uses custom bucket name from R2_BUCKET_NAME env var', async () => { + const { sandbox, mountBucketMock } = createMockSandbox({ mounted: false }); + const env = createMockEnvWithR2({ + R2_ACCESS_KEY_ID: 'key123', + R2_SECRET_ACCESS_KEY: 'secret', + CF_ACCOUNT_ID: 'account123', + R2_BUCKET_NAME: 'moltbot-e2e-test123', + }); + + const result = await mountR2Storage(sandbox, env); + + expect(result).toBe(true); + expect(mountBucketMock).toHaveBeenCalledWith( + 'moltbot-e2e-test123', + '/data/moltbot', + expect.any(Object) + ); + }); + it('returns true immediately when bucket is already mounted', async () => { const { sandbox, mountBucketMock } = createMockSandbox({ mounted: true }); const env = createMockEnvWithR2(); diff --git a/src/gateway/r2.ts b/src/gateway/r2.ts index 0887d59e..302c61d7 100644 --- a/src/gateway/r2.ts +++ b/src/gateway/r2.ts @@ -1,6 +1,6 @@ import type { Sandbox } from '@cloudflare/sandbox'; import type { MoltbotEnv } from '../types'; -import { R2_MOUNT_PATH, R2_BUCKET_NAME } from '../config'; +import { R2_MOUNT_PATH, getR2BucketName } from '../config'; /** * Check if R2 is already mounted by looking at the mount table @@ -45,9 +45,10 @@ export async function mountR2Storage(sandbox: Sandbox, env: MoltbotEnv): Promise return true; } + const bucketName = getR2BucketName(env); try { - console.log('Mounting R2 bucket at', R2_MOUNT_PATH); - await sandbox.mountBucket(R2_BUCKET_NAME, R2_MOUNT_PATH, { + console.log('Mounting R2 bucket', bucketName, 'at', R2_MOUNT_PATH); + await sandbox.mountBucket(bucketName, R2_MOUNT_PATH, { endpoint: `https://${env.CF_ACCOUNT_ID}.r2.cloudflarestorage.com`, // Pass credentials explicitly since we use R2_* naming instead of AWS_* credentials: { diff --git a/src/types.ts b/src/types.ts index 6287bc70..d0fe5450 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,6 +33,7 @@ export interface MoltbotEnv { // R2 credentials for bucket mounting (set via wrangler secret) R2_ACCESS_KEY_ID?: string; R2_SECRET_ACCESS_KEY?: string; + R2_BUCKET_NAME?: string; // Override bucket name (default: 'moltbot-data') CF_ACCOUNT_ID?: string; // Cloudflare account ID for R2 endpoint // Browser Rendering binding for CDP shim BROWSER?: Fetcher; diff --git a/test/e2e/.dev.vars.example b/test/e2e/.dev.vars.example new file mode 100644 index 00000000..0233663c --- /dev/null +++ b/test/e2e/.dev.vars.example @@ -0,0 +1,120 @@ +# Cloud E2E Test Credentials +# Copy this file to .dev.vars and fill in your values +# DO NOT commit .dev.vars to git! + +# ============================================================================= +# CLOUDFLARE_API_TOKEN +# ============================================================================= +# Required: Cloudflare API token with specific permissions for e2e tests. +# +# How to create: +# 1. Go to https://dash.cloudflare.com/profile/api-tokens +# 2. Click "Create Token" +# 3. Click "Create Custom Token" (at the bottom) +# 4. Configure the token: +# +# Token name: moltworker-e2e-tests (or whatever you prefer) +# +# Permissions (add all of these): +# ┌─────────────────────────────────────────────────────────────────────────┐ +# │ Account │ Workers Scripts │ Edit │ +# │ Account │ Workers R2 Storage │ Edit │ +# │ Account │ Cloudflare Containers │ Edit │ +# │ Account │ Access: Apps and Policies │ Edit │ +# │ Account │ Access: Service Tokens │ Edit │ +# └─────────────────────────────────────────────────────────────────────────┘ +# +# Account Resources: +# - Include: Your account (or "All accounts" if you have multiple) +# +# Client IP Address Filtering: (optional, leave blank for no restrictions) +# +# TTL: (optional, set an expiry if desired) +# +# 5. Click "Continue to summary" +# 6. Click "Create Token" +# 7. Copy the token value (you won't see it again!) +# +CLOUDFLARE_API_TOKEN= + +# ============================================================================= +# CF_ACCOUNT_ID +# ============================================================================= +# Required: Your Cloudflare account ID +# +# How to find: +# 1. Go to https://dash.cloudflare.com/ +# 2. Click the "..." menu next to your account name in the sidebar +# 3. Click "Copy Account ID" +# +# Or: Dashboard → any zone → Overview → scroll down to "API" section +# +CF_ACCOUNT_ID= + +# ============================================================================= +# WORKERS_SUBDOMAIN +# ============================================================================= +# Required: Your workers.dev subdomain +# +# This is the subdomain part of your workers.dev URL. +# For example, if your workers deploy to "my-worker.myaccount.workers.dev", +# then your WORKERS_SUBDOMAIN is "myaccount". +# +# How to find: +# 1. Go to https://dash.cloudflare.com/ → Workers & Pages +# 2. Look at any deployed worker's URL, or +# 3. Go to Workers & Pages → Overview → your subdomain is shown at the top +# +WORKERS_SUBDOMAIN= + +# ============================================================================= +# CF_ACCESS_TEAM_DOMAIN +# ============================================================================= +# Required: Your Cloudflare Access team domain +# +# This is your Zero Trust organization's domain, typically in the format: +# "yourteam.cloudflareaccess.com" +# +# How to find: +# 1. Go to https://one.dash.cloudflare.com/ (Zero Trust dashboard) +# 2. Go to Settings → Custom Pages +# 3. Your team domain is shown at the top (e.g., "yourteam.cloudflareaccess.com") +# +# Or: Look at any Access login page URL - it will be https://yourteam.cloudflareaccess.com/... +# +CF_ACCESS_TEAM_DOMAIN= + +# ============================================================================= +# R2_ACCESS_KEY_ID and R2_SECRET_ACCESS_KEY +# ============================================================================= +# Required: R2 API credentials for bucket mounting inside the container +# +# How to create: +# 1. Go to https://dash.cloudflare.com/ → R2 → Overview +# 2. Click "Manage R2 API Tokens" (top right) +# 3. Click "Create API Token" +# 4. Configure: +# - Token name: moltworker-e2e (or whatever you prefer) +# - Permissions: Object Read & Write +# - Specify bucket(s): You can leave as "Apply to all buckets" or +# limit to buckets starting with "moltbot-" for safety +# - TTL: (optional) +# 5. Click "Create API Token" +# 6. Copy both the "Access Key ID" and "Secret Access Key" +# (Secret is only shown once!) +# +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= + +# ============================================================================= +# OPTIONAL SETTINGS +# ============================================================================= + +# Unique test run ID for isolation (default: "local") +# In CI, set this to the PR number or a unique identifier to allow parallel runs +# E2E_TEST_RUN_ID=local + +# AI provider credentials (at least one recommended for chat/conversation tests) +# AI_GATEWAY_API_KEY= +# AI_GATEWAY_BASE_URL= +# ANTHROPIC_API_KEY= diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 00000000..6c1c05aa --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,185 @@ +# E2E tests for Moltworker + +End-to-end tests that deploy real Moltworker instances to Cloudflare infrastructure. + +## Why cloud-based e2e tests? + +These tests run against actual Cloudflare infrastructure—the same environment users get when they deploy Moltworker themselves. This catches issues that local testing can't: + +- **R2 bucket mounting** only works in production (not with `wrangler dev`) +- **Container cold starts** and sandbox behavior +- **Cloudflare Access** authentication flows +- **Real network latency** and timeout handling + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ Test runner │ +│ │ +│ cctr test/e2e/ │ +│ ├── _setup.txt (start server, browser, video) │ +│ ├── pairing_and_conversation.txt │ +│ └── _teardown.txt (stop everything, clean up cloud resources) │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Cloud infrastructure │ +│ │ +│ Terraform (main.tf) Wrangler deploy Access API │ +│ ├── Service token → ├── Worker → ├── App │ +│ └── R2 bucket ├── Container └── Policies │ +│ └── Secrets │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Deployed worker │ +│ │ +│ https://moltbot-sandbox-e2e-{id}.{subdomain}.workers.dev │ +│ │ +│ Protected by Cloudflare Access: │ +│ - Service token (for automated tests) │ +│ - @cloudflare.com emails (for manual debugging) │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### Test flow + +1. **Terraform** creates isolated resources: service token + R2 bucket +2. **Wrangler** deploys worker with unique name (timestamp + random suffix) +3. **Access API** creates Access application (must be after worker exists—workers.dev domains require the worker to exist first) +4. **Playwright** opens browser with Access headers, navigates to worker +5. **Tests run** with video recording capturing the full UI flow +6. **Teardown** deletes everything: Access app → worker → R2 bucket → service token + +### Key design decisions + +- **Unique IDs per test run**: `$(date +%s)-$(openssl rand -hex 4)` ensures parallel test runs don't conflict +- **Access created post-deploy**: Terraform can't create Access apps for non-existent domains +- **Container names**: Derived from worker name as `{worker-name}-sandbox` + +## Test framework: cctr + playwright-cli + +Tests use two complementary tools: + +### [cctr](https://github.com/andreasjansson/cctr) - CLI Corpus Test Runner + +cctr runs test where each test case is a command line script, e.g. + +``` +=== +navigate to admin page to approve device +%require +=== +TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") +WORKER_URL=$(cat "$CCTR_FIXTURE_DIR/worker-url.txt") +./pw --session=moltworker-e2e open "$WORKER_URL/_admin/?token=$TOKEN" +--- +``` + +Key features: +- **Plain text format**: Easy to read and write +- **`%require` directive**: If this test fails, skip all subsequent tests +- **Variables**: Capture dynamic output with `{{ name }}` +- **Fixtures**: `fixture/` directory copied to temp dir for each suite +- **Setup/teardown**: `_setup.txt` and `_teardown.txt` run before/after tests + +### [playwright-cli](https://github.com/microsoft/playwright-cli) - Browser automation CLI + +playwright-cli provides shell-friendly browser automation. Instead of writing JavaScript test files, you control the browser with CLI commands: + +```bash +# Open a page +playwright-cli --session=test open "https://example.com" + +# Run arbitrary Playwright code +playwright-cli --session=test run-code "async page => { + await page.waitForSelector('text=Hello'); +}" + +# Take screenshots, record video +playwright-cli --session=test video-start +playwright-cli --session=test screenshot +``` + +The `./pw` wrapper in our fixture works around a playwright-cli bug where errors don't set a non-zero exit code. It detects `### Error` in the output and exits with code 1, making errors fail the test properly. + +## Example test + +Here's a complete test that approves a device and sends a chat message: + +``` +=== +wait for Approve All button and click it +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + const btn = await page.waitForSelector('button:has-text(\"Approve All\")', { timeout: 120000 }); + await btn.click(); +}" +--- + +=== +wait for approval to complete +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForSelector('text=No pending pairing requests', { timeout: 120000 }); +}" +--- + +=== +type math question into chat +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + const textarea = await page.waitForSelector('textarea'); + await textarea.fill('What is 847293 + 651824? Reply with just the number.'); +}" +--- + +=== +wait for response containing the correct answer +=== +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForSelector('text=1499117', { timeout: 120000 }); +}" +--- +``` + +## Running the e2e test suite locally + +### Prerequisites + +1. Copy `.dev.vars.example` to `.dev.vars` and fill in credentials (see file for detailed instructions) +2. Install dependencies: `npm install` +3. Install cctr: `brew install andreasjansson/tap/cctr` or `cargo install cctr` +4. Install playwright-cli: `npm install -g playwright-cli` + +### Run tests + +```bash +# Run all e2e tests +cctr test/e2e/ + +# Run with verbose output +cctr test/e2e/ -v + +# Run specific test file +cctr test/e2e/ -p pairing + +# Watch test output in real-time (for debugging) +cctr test/e2e/ -vv +``` + +### Run headed (see the browser) + +```bash +PLAYWRIGHT_HEADED=1 cctr test/e2e/ +``` + +### View test videos + +Videos are saved to `/tmp/moltworker-e2e-videos/` after each run. diff --git a/test/e2e/_setup.txt b/test/e2e/_setup.txt index fe8350b0..63d907aa 100644 --- a/test/e2e/_setup.txt +++ b/test/e2e/_setup.txt @@ -24,3 +24,21 @@ start video recording --- where * output contains "Video recording started" + +=== +navigate to main page and wait for worker to be ready +%require +=== +TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") +WORKER_URL=$(cat "$CCTR_FIXTURE_DIR/worker-url.txt") +./pw --session=moltworker-e2e open "$WORKER_URL/?token=$TOKEN" +# Wait for pairing required message (worker shows loading screen first, then UI loads) +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForSelector('text=Pairing required', { timeout: 300000 }); +}" +echo "Worker is ready" +--- +{{ output }} +--- +where +* output contains "Worker is ready" diff --git a/test/e2e/_teardown.txt b/test/e2e/_teardown.txt index 575c417a..1b7888e6 100644 --- a/test/e2e/_teardown.txt +++ b/test/e2e/_teardown.txt @@ -1,12 +1,12 @@ === stop video recording === -./pw --session=moltworker-e2e video-stop +./pw --session=moltworker-e2e video-stop || true --- {{ output }} --- where -* output contains "Video" +* output contains "Video" or output contains "Error" or output contains "No" === save video recording @@ -19,16 +19,19 @@ for f in ./.playwright-cli/*.webm; do echo "video saved to /tmp/moltworker-e2e-videos/${datetime}.webm" fi done +# Always succeed even if no video +echo "video cleanup complete" --- {{ output }} --- where -* output contains "video saved to" +* output contains "video" === stop playwright browser === -./stop-browser +./stop-browser || true +echo "browser stopped" --- {{ output }} --- @@ -36,8 +39,9 @@ where * output contains "stopped" === -stop moltworker server +stop moltworker server and destroy cloud resources === +# This deletes the worker AND destroys terraform resources (Access app, service token, R2 bucket) ./stop-server --- {{ s }} diff --git a/test/e2e/fixture/curl-auth b/test/e2e/fixture/curl-auth new file mode 100755 index 00000000..71767bab --- /dev/null +++ b/test/e2e/fixture/curl-auth @@ -0,0 +1,26 @@ +#!/bin/bash +# Wrapper for curl that adds Cloudflare Access service token headers +# +# Usage: ./curl-auth [curl-args...] +# +# Automatically adds CF-Access-Client-Id and CF-Access-Client-Secret headers +# using values from $CCTR_FIXTURE_DIR + +set -e + +if [ -z "$CCTR_FIXTURE_DIR" ]; then + CCTR_FIXTURE_DIR="/tmp/e2e-cloud-manual" +fi + +CF_ACCESS_CLIENT_ID=$(cat "$CCTR_FIXTURE_DIR/cf-access-client-id.txt" 2>/dev/null || echo "") +CF_ACCESS_CLIENT_SECRET=$(cat "$CCTR_FIXTURE_DIR/cf-access-client-secret.txt" 2>/dev/null || echo "") + +if [ -z "$CF_ACCESS_CLIENT_ID" ] || [ -z "$CF_ACCESS_CLIENT_SECRET" ]; then + echo "Error: Access credentials not found in $CCTR_FIXTURE_DIR" >&2 + exit 1 +fi + +exec curl \ + -H "CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID" \ + -H "CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET" \ + "$@" diff --git a/test/e2e/fixture/server/create-access-app b/test/e2e/fixture/server/create-access-app new file mode 100755 index 00000000..34fed4c5 --- /dev/null +++ b/test/e2e/fixture/server/create-access-app @@ -0,0 +1,107 @@ +#!/bin/bash +# Create Access application to protect the deployed worker +# Must be called AFTER the worker is deployed +set -e + +WORKER_NAME="$1" +SERVICE_TOKEN_ID="$2" + +if [ -z "$WORKER_NAME" ] || [ -z "$SERVICE_TOKEN_ID" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}" +: "${CLOUDFLARE_ACCOUNT_ID:?CLOUDFLARE_ACCOUNT_ID is required}" +: "${WORKERS_SUBDOMAIN:?WORKERS_SUBDOMAIN is required}" + +WORKER_DOMAIN="${WORKER_NAME}.${WORKERS_SUBDOMAIN}.workers.dev" +APP_NAME="e2e-${WORKER_NAME}" + +echo "Creating Access application for $WORKER_DOMAIN..." >&2 + +# Create the Access application +APP_RESPONSE=$(curl -s -X POST \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/access/apps" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{ + "name": "'"$APP_NAME"'", + "domain": "'"$WORKER_DOMAIN"'", + "type": "self_hosted", + "session_duration": "24h", + "auto_redirect_to_identity": false, + "app_launcher_visible": false + }') + +APP_ID=$(echo "$APP_RESPONSE" | jq -r '.result.id // empty') +APP_AUD=$(echo "$APP_RESPONSE" | jq -r '.result.aud // empty') + +if [ -z "$APP_ID" ]; then + echo "Failed to create Access application:" >&2 + echo "$APP_RESPONSE" | jq >&2 + exit 1 +fi + +echo "Created Access application: $APP_ID" >&2 + +# Create Service Auth policy to allow our service token +echo "Creating Service Auth policy..." >&2 +POLICY_RESPONSE=$(curl -s -X POST \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/access/apps/$APP_ID/policies" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{ + "name": "E2E Service Token", + "decision": "non_identity", + "precedence": 1, + "include": [ + { + "service_token": { + "token_id": "'"$SERVICE_TOKEN_ID"'" + } + } + ] + }') + +POLICY_SUCCESS=$(echo "$POLICY_RESPONSE" | jq -r '.success') +if [ "$POLICY_SUCCESS" != "true" ]; then + echo "Failed to create service token policy:" >&2 + echo "$POLICY_RESPONSE" | jq >&2 + # Clean up the app we just created + curl -s -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/access/apps/$APP_ID" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" >/dev/null + exit 1 +fi +echo "Created Service Auth policy" >&2 + +# Create Allow policy for Cloudflare employees +echo "Creating Cloudflare employees policy..." >&2 +POLICY_RESPONSE=$(curl -s -X POST \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/access/apps/$APP_ID/policies" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{ + "name": "Cloudflare Employees", + "decision": "allow", + "precedence": 2, + "include": [ + { + "email_domain": { + "domain": "cloudflare.com" + } + } + ] + }') + +POLICY_SUCCESS=$(echo "$POLICY_RESPONSE" | jq -r '.success') +if [ "$POLICY_SUCCESS" != "true" ]; then + echo "Warning: Failed to create Cloudflare employees policy (non-fatal):" >&2 + echo "$POLICY_RESPONSE" | jq >&2 +fi +echo "Created Cloudflare employees policy" >&2 + +# Output the app ID and AUD for use by other scripts +echo "$APP_ID" +echo "$APP_AUD" diff --git a/test/e2e/fixture/server/delete-worker b/test/e2e/fixture/server/delete-worker new file mode 100755 index 00000000..9b08123a --- /dev/null +++ b/test/e2e/fixture/server/delete-worker @@ -0,0 +1,19 @@ +#!/bin/bash +# Delete the deployed e2e worker +set -e + +WORKER_NAME="$1" +if [ -z "$WORKER_NAME" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}" + +echo "Deleting worker: $WORKER_NAME" >&2 + +# Delete the worker using wrangler +# Use --force to skip confirmation prompt +npx wrangler delete --name "$WORKER_NAME" --force 2>&1 || true + +echo "Worker deleted: $WORKER_NAME" >&2 diff --git a/test/e2e/fixture/server/deploy b/test/e2e/fixture/server/deploy new file mode 100755 index 00000000..f139dddd --- /dev/null +++ b/test/e2e/fixture/server/deploy @@ -0,0 +1,83 @@ +#!/bin/bash +# Deploy the worker to Cloudflare with e2e configuration +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Find project directory - use CCTR_TEST_PATH if available (cctr copies fixture to temp dir) +if [ -n "$CCTR_TEST_PATH" ]; then + PROJECT_DIR="$(cd "$CCTR_TEST_PATH/../.." && pwd)" +else + FIXTURE_DIR="$(dirname "$SCRIPT_DIR")" + PROJECT_DIR="$(cd "$FIXTURE_DIR/../.." && pwd)" +fi + +# Required environment variables +: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}" +: "${CF_ACCOUNT_ID:?CF_ACCOUNT_ID is required}" +: "${R2_ACCESS_KEY_ID:?R2_ACCESS_KEY_ID is required}" +: "${R2_SECRET_ACCESS_KEY:?R2_SECRET_ACCESS_KEY is required}" +: "${MOLTBOT_GATEWAY_TOKEN:?MOLTBOT_GATEWAY_TOKEN is required}" + +# Read terraform outputs +TERRAFORM_OUTPUT="$1" +if [ -z "$TERRAFORM_OUTPUT" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +WORKER_NAME=$(echo "$TERRAFORM_OUTPUT" | jq -r '.worker_name.value') +R2_BUCKET=$(echo "$TERRAFORM_OUTPUT" | jq -r '.r2_bucket_name.value') + +# Get CF_ACCESS_TEAM_DOMAIN from environment +: "${CF_ACCESS_TEAM_DOMAIN:?CF_ACCESS_TEAM_DOMAIN is required}" + +cd "$PROJECT_DIR" + +# Build first +echo "Building project..." >&2 +npm run build >&2 + +# Export account ID for all wrangler commands (and unset deprecated name) +export CLOUDFLARE_ACCOUNT_ID="$CF_ACCOUNT_ID" + +# Generate a temporary wrangler config with unique worker name +# This ensures the container name is also unique (container name = worker-name + class-name) +E2E_CONFIG="$PROJECT_DIR/.wrangler-e2e-$WORKER_NAME.jsonc" +echo "Generating e2e config: $E2E_CONFIG" >&2 + +# Copy config and replace the name field (sed handles JSONC comments fine) +sed 's/"name": "moltbot-sandbox"/"name": "'"$WORKER_NAME"'"/' "$PROJECT_DIR/wrangler.jsonc" > "$E2E_CONFIG" + +# Deploy using the e2e-specific config +echo "Deploying worker: $WORKER_NAME to account $CLOUDFLARE_ACCOUNT_ID" >&2 +npx wrangler deploy \ + --config "$E2E_CONFIG" \ + --var "DEBUG_ROUTES:true" \ + --var "E2E_TEST_MODE:true" \ + >&2 + +# Clean up temp config +rm -f "$E2E_CONFIG" + +# Set secrets for the deployed worker +echo "Setting worker secrets..." >&2 +echo "$MOLTBOT_GATEWAY_TOKEN" | npx wrangler secret put MOLTBOT_GATEWAY_TOKEN --name "$WORKER_NAME" >&2 +echo "$CF_ACCESS_TEAM_DOMAIN" | npx wrangler secret put CF_ACCESS_TEAM_DOMAIN --name "$WORKER_NAME" >&2 +echo "$R2_ACCESS_KEY_ID" | npx wrangler secret put R2_ACCESS_KEY_ID --name "$WORKER_NAME" >&2 +echo "$R2_SECRET_ACCESS_KEY" | npx wrangler secret put R2_SECRET_ACCESS_KEY --name "$WORKER_NAME" >&2 +echo "$R2_BUCKET" | npx wrangler secret put R2_BUCKET_NAME --name "$WORKER_NAME" >&2 +echo "$CLOUDFLARE_ACCOUNT_ID" | npx wrangler secret put CF_ACCOUNT_ID --name "$WORKER_NAME" >&2 + +# Set AI provider keys if available +if [ -n "$AI_GATEWAY_API_KEY" ]; then + echo "$AI_GATEWAY_API_KEY" | npx wrangler secret put AI_GATEWAY_API_KEY --name "$WORKER_NAME" >&2 +fi +if [ -n "$AI_GATEWAY_BASE_URL" ]; then + echo "$AI_GATEWAY_BASE_URL" | npx wrangler secret put AI_GATEWAY_BASE_URL --name "$WORKER_NAME" >&2 +fi +if [ -n "$ANTHROPIC_API_KEY" ]; then + echo "$ANTHROPIC_API_KEY" | npx wrangler secret put ANTHROPIC_API_KEY --name "$WORKER_NAME" >&2 +fi + +echo "Worker deployed: $WORKER_NAME" >&2 diff --git a/test/e2e/fixture/server/main.tf b/test/e2e/fixture/server/main.tf new file mode 100755 index 00000000..b3a2aeb5 --- /dev/null +++ b/test/e2e/fixture/server/main.tf @@ -0,0 +1,32 @@ +terraform { + required_providers { + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 5.0" + } + } +} + +provider "cloudflare" { + api_token = var.cloudflare_api_token +} + +# Service Token for automated testing (available for future use) +resource "cloudflare_zero_trust_access_service_token" "e2e" { + account_id = var.cloudflare_account_id + name = "moltbot-e2e-${var.test_run_id}" + duration = "8760h" +} + +# R2 bucket for E2E tests (isolated from production) +resource "cloudflare_r2_bucket" "e2e" { + account_id = var.cloudflare_account_id + name = "moltbot-e2e-${var.test_run_id}" + location = "WNAM" +} + +# NOTE: Access application is NOT created here because workers.dev domains +# require the worker to exist first. Instead: +# - E2E_TEST_MODE=true in the worker skips Access validation +# - Authentication is done via MOLTBOT_GATEWAY_TOKEN +# - Service token is created above for potential future use diff --git a/test/e2e/fixture/server/outputs.tf b/test/e2e/fixture/server/outputs.tf new file mode 100755 index 00000000..d834cb1b --- /dev/null +++ b/test/e2e/fixture/server/outputs.tf @@ -0,0 +1,30 @@ +output "worker_url" { + description = "URL of the deployed e2e worker" + value = "https://moltbot-sandbox-e2e-${var.test_run_id}.${var.workers_subdomain}.workers.dev" +} + +output "worker_name" { + description = "Name of the deployed worker" + value = "moltbot-sandbox-e2e-${var.test_run_id}" +} + +output "service_token_id" { + description = "Service token ID (for creating Access policies)" + value = cloudflare_zero_trust_access_service_token.e2e.id +} + +output "service_token_client_id" { + description = "Service token Client ID for authentication" + value = cloudflare_zero_trust_access_service_token.e2e.client_id +} + +output "service_token_client_secret" { + description = "Service token Client Secret for authentication" + value = cloudflare_zero_trust_access_service_token.e2e.client_secret + sensitive = true +} + +output "r2_bucket_name" { + description = "Name of the R2 bucket for this e2e test run" + value = cloudflare_r2_bucket.e2e.name +} diff --git a/test/e2e/fixture/server/start b/test/e2e/fixture/server/start new file mode 100755 index 00000000..1b0e5962 --- /dev/null +++ b/test/e2e/fixture/server/start @@ -0,0 +1,122 @@ +#!/bin/bash +# Start cloud e2e infrastructure and deploy worker +# +# This script: +# 1. Creates Access application, service token, and R2 bucket via terraform +# 2. Deploys the worker with appropriate secrets +# 3. Waits for the worker to be ready +# 4. Outputs connection info for tests +set -e + +VERBOSE=false +if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then + VERBOSE=true +fi + +log() { + if [ "$VERBOSE" = true ]; then + echo "[cloud-e2e] $*" >&2 + fi +} + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +FIXTURE_DIR="$(dirname "$SCRIPT_DIR")" + +# Support running directly (not via cctr) for manual debugging +if [ -z "$CCTR_TEST_PATH" ]; then + # Running directly - E2E_DIR is parent of fixture dir + E2E_DIR="$(dirname "$FIXTURE_DIR")" + log "CCTR_TEST_PATH not set, using E2E_DIR: $E2E_DIR" +else + # Running via cctr - CCTR_TEST_PATH points to original test dir + E2E_DIR="$CCTR_TEST_PATH" +fi + +if [ -z "$CCTR_FIXTURE_DIR" ]; then + CCTR_FIXTURE_DIR="/tmp/e2e-cloud-manual" + mkdir -p "$CCTR_FIXTURE_DIR" + log "CCTR_FIXTURE_DIR not set, using: $CCTR_FIXTURE_DIR" +fi + +# Source .dev.vars if it exists (for local development) +if [ -f "$E2E_DIR/.dev.vars" ]; then + log "Loading credentials from $E2E_DIR/.dev.vars" + set -a + source "$E2E_DIR/.dev.vars" + set +a +fi + +# Check required environment variables +: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}" +: "${CF_ACCOUNT_ID:?CF_ACCOUNT_ID is required}" +: "${WORKERS_SUBDOMAIN:?WORKERS_SUBDOMAIN is required}" +: "${CF_ACCESS_TEAM_DOMAIN:?CF_ACCESS_TEAM_DOMAIN is required}" +: "${R2_ACCESS_KEY_ID:?R2_ACCESS_KEY_ID is required}" +: "${R2_SECRET_ACCESS_KEY:?R2_SECRET_ACCESS_KEY is required}" + +# Use timestamp + random suffix for truly unique IDs (avoids conflicts from stale resources) +export E2E_TEST_RUN_ID="${E2E_TEST_RUN_ID:-$(date +%s)-$(openssl rand -hex 4)}" + +# Generate a gateway token for this test run +GATEWAY_TOKEN="${MOLTBOT_GATEWAY_TOKEN:-e2e-cloud-$(openssl rand -hex 16)}" +export MOLTBOT_GATEWAY_TOKEN="$GATEWAY_TOKEN" + +log "Starting cloud e2e infrastructure..." +log "Test run ID: $E2E_TEST_RUN_ID" + +# Clean up any stale terraform state from previous runs +rm -rf "$SCRIPT_DIR/.terraform" "$SCRIPT_DIR/terraform.tfstate"* "$SCRIPT_DIR/.terraform.lock.hcl" + +# Step 1: Apply terraform to create Access app, service token, R2 bucket +log "Step 1: Creating cloud infrastructure with terraform..." +cd "$SCRIPT_DIR" +TERRAFORM_OUTPUT=$("$SCRIPT_DIR/terraform-apply") +log "Terraform output: $TERRAFORM_OUTPUT" + +# Parse terraform outputs +WORKER_URL=$(echo "$TERRAFORM_OUTPUT" | jq -r '.worker_url.value') +WORKER_NAME=$(echo "$TERRAFORM_OUTPUT" | jq -r '.worker_name.value') +ACCESS_AUD=$(echo "$TERRAFORM_OUTPUT" | jq -r '.access_application_aud.value') +SERVICE_TOKEN_CLIENT_ID=$(echo "$TERRAFORM_OUTPUT" | jq -r '.service_token_client_id.value') +SERVICE_TOKEN_CLIENT_SECRET=$(echo "$TERRAFORM_OUTPUT" | jq -r '.service_token_client_secret.value') +R2_BUCKET=$(echo "$TERRAFORM_OUTPUT" | jq -r '.r2_bucket_name.value') + +log "Worker URL: $WORKER_URL" +log "Worker name: $WORKER_NAME" +log "Access AUD: $ACCESS_AUD" +log "Service token client ID: $SERVICE_TOKEN_CLIENT_ID" +log "R2 bucket: $R2_BUCKET" + +# Save outputs for other scripts +echo "$TERRAFORM_OUTPUT" > "$CCTR_FIXTURE_DIR/terraform-output.json" +echo "$WORKER_URL" > "$CCTR_FIXTURE_DIR/worker-url.txt" +echo "$WORKER_NAME" > "$CCTR_FIXTURE_DIR/worker-name.txt" +echo "$GATEWAY_TOKEN" > "$CCTR_FIXTURE_DIR/gateway-token.txt" +echo "$SERVICE_TOKEN_CLIENT_ID" > "$CCTR_FIXTURE_DIR/cf-access-client-id.txt" +echo "$SERVICE_TOKEN_CLIENT_SECRET" > "$CCTR_FIXTURE_DIR/cf-access-client-secret.txt" +echo "$E2E_TEST_RUN_ID" > "$CCTR_FIXTURE_DIR/test-run-id.txt" +echo "$R2_BUCKET" > "$CCTR_FIXTURE_DIR/r2-bucket-name.txt" +echo "${WORKER_NAME}-sandbox" > "$CCTR_FIXTURE_DIR/container-name.txt" + +# Step 2: Deploy the worker +log "Step 2: Deploying worker..." +"$SCRIPT_DIR/deploy" "$TERRAFORM_OUTPUT" + +# Step 3: Create Access application (must be after worker exists) +log "Step 3: Creating Access application..." +SERVICE_TOKEN_ID=$(echo "$TERRAFORM_OUTPUT" | jq -r '.service_token_id.value') +export CLOUDFLARE_ACCOUNT_ID="$CF_ACCOUNT_ID" +ACCESS_OUTPUT=$("$SCRIPT_DIR/create-access-app" "$WORKER_NAME" "$SERVICE_TOKEN_ID") +ACCESS_APP_ID=$(echo "$ACCESS_OUTPUT" | head -1) +ACCESS_AUD=$(echo "$ACCESS_OUTPUT" | tail -1) +echo "$ACCESS_APP_ID" > "$CCTR_FIXTURE_DIR/access-app-id.txt" +echo "$ACCESS_AUD" > "$CCTR_FIXTURE_DIR/access-aud.txt" +log "Access app ID: $ACCESS_APP_ID" +log "Access AUD: $ACCESS_AUD" + +log "Cloud e2e infrastructure deployed!" +log "Worker URL: $WORKER_URL" +log "Gateway token: $GATEWAY_TOKEN" +log "Note: Worker may still be starting - browser will wait for it" +sleep 1 # Let stderr flush before stdout +echo "ready" diff --git a/test/e2e/fixture/server/stop b/test/e2e/fixture/server/stop new file mode 100755 index 00000000..73cc2fe4 --- /dev/null +++ b/test/e2e/fixture/server/stop @@ -0,0 +1,125 @@ +#!/bin/bash +# Stop and clean up ALL cloud e2e infrastructure +# +# This script: +# 1. Deletes the deployed worker +# 2. Deletes the R2 bucket (may fail if not empty - requires manual cleanup) +# 3. Deletes the service token +# 4. Cleans up local state files +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Find E2E_DIR for .dev.vars +if [ -n "$CCTR_TEST_PATH" ]; then + E2E_DIR="$CCTR_TEST_PATH" +else + FIXTURE_DIR="$(dirname "$SCRIPT_DIR")" + E2E_DIR="$(dirname "$FIXTURE_DIR")" +fi + +# Source .dev.vars if it exists +if [ -f "$E2E_DIR/.dev.vars" ]; then + set -a + source "$E2E_DIR/.dev.vars" + set +a +fi + +# Export CLOUDFLARE_ACCOUNT_ID (wrangler prefers this over CF_ACCOUNT_ID) +export CLOUDFLARE_ACCOUNT_ID="${CF_ACCOUNT_ID:-}" + +# Support running directly (not via cctr) +if [ -z "$CCTR_FIXTURE_DIR" ]; then + CCTR_FIXTURE_DIR="/tmp/e2e-cloud-manual" +fi + +echo "Stopping cloud e2e infrastructure..." >&2 + +# Read saved values from fixture dir +WORKER_NAME=$(cat "$CCTR_FIXTURE_DIR/worker-name.txt" 2>/dev/null || echo "") +R2_BUCKET=$(cat "$CCTR_FIXTURE_DIR/r2-bucket-name.txt" 2>/dev/null || echo "") +TEST_RUN_ID=$(cat "$CCTR_FIXTURE_DIR/test-run-id.txt" 2>/dev/null || echo "") +ACCESS_APP_ID=$(cat "$CCTR_FIXTURE_DIR/access-app-id.txt" 2>/dev/null || echo "") + +# Step 0: Delete the Access application first (so it stops protecting the worker) +if [ -n "$ACCESS_APP_ID" ] && [ -n "$CLOUDFLARE_API_TOKEN" ] && [ -n "$CLOUDFLARE_ACCOUNT_ID" ]; then + echo "Deleting Access application: $ACCESS_APP_ID" >&2 + curl -s -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/access/apps/$ACCESS_APP_ID" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" >/dev/null 2>&1 || true + echo "Access application deleted" >&2 +fi + +# Step 1: Delete the deployed worker +if [ -n "$WORKER_NAME" ]; then + echo "Deleting worker: $WORKER_NAME" >&2 + "$SCRIPT_DIR/delete-worker" "$WORKER_NAME" 2>&1 || true +fi + +# Step 1b: Delete the container application +CONTAINER_NAME=$(cat "$CCTR_FIXTURE_DIR/container-name.txt" 2>/dev/null || echo "${WORKER_NAME}-sandbox") +if [ -n "$WORKER_NAME" ] && [ -n "$CLOUDFLARE_API_TOKEN" ] && [ -n "$CLOUDFLARE_ACCOUNT_ID" ]; then + echo "Deleting container: $CONTAINER_NAME" >&2 + # Find the container ID + CONTAINER_ID=$(curl -s -X GET \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/containers/applications" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" | \ + jq -r ".result[] | select(.name == \"$CONTAINER_NAME\") | .id" 2>/dev/null || echo "") + + if [ -n "$CONTAINER_ID" ]; then + curl -s -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/containers/applications/$CONTAINER_ID" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" >/dev/null 2>&1 || true + echo "Container deleted" >&2 + fi +fi + +# Step 2: Delete R2 bucket +# Note: wrangler doesn't have a command to list/delete objects, so if the bucket +# has objects it will fail. Use the dashboard or aws cli for manual cleanup. +if [ -n "$R2_BUCKET" ]; then + echo "Deleting R2 bucket: $R2_BUCKET" >&2 + if ! npx wrangler r2 bucket delete "$R2_BUCKET" 2>&1; then + echo "Warning: Failed to delete R2 bucket (may not be empty). Manual cleanup required." >&2 + fi +fi + +# Step 3: Delete service token via API +if [ -n "$TEST_RUN_ID" ] && [ -n "$CLOUDFLARE_API_TOKEN" ] && [ -n "$CLOUDFLARE_ACCOUNT_ID" ]; then + echo "Deleting service token: moltbot-e2e-$TEST_RUN_ID" >&2 + # Find and delete the service token + TOKEN_ID=$(curl -s -X GET \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/access/service_tokens" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" | \ + jq -r ".result[] | select(.name == \"moltbot-e2e-$TEST_RUN_ID\") | .id" 2>/dev/null || echo "") + + if [ -n "$TOKEN_ID" ]; then + curl -s -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/access/service_tokens/$TOKEN_ID" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" >/dev/null 2>&1 || true + echo "Service token deleted" >&2 + fi +fi + +# Step 4: Clean up local files +echo "Cleaning up local files..." >&2 +rm -f "$CCTR_FIXTURE_DIR/terraform-output.json" +rm -f "$CCTR_FIXTURE_DIR/worker-url.txt" +rm -f "$CCTR_FIXTURE_DIR/worker-name.txt" +rm -f "$CCTR_FIXTURE_DIR/gateway-token.txt" +rm -f "$CCTR_FIXTURE_DIR/cf-access-client-id.txt" +rm -f "$CCTR_FIXTURE_DIR/cf-access-client-secret.txt" +rm -f "$CCTR_FIXTURE_DIR/test-run-id.txt" +rm -f "$CCTR_FIXTURE_DIR/r2-bucket-name.txt" +rm -f "$CCTR_FIXTURE_DIR/container-name.txt" +rm -f "$CCTR_FIXTURE_DIR/access-app-id.txt" +rm -f "$CCTR_FIXTURE_DIR/access-aud.txt" +rm -rf "$SCRIPT_DIR/.terraform" "$SCRIPT_DIR/terraform.tfstate"* "$SCRIPT_DIR/.terraform.lock.hcl" + +echo "Cloud e2e infrastructure stopped and cleaned up" >&2 +sleep 1 # Let stderr flush before stdout +echo "stopped" diff --git a/test/e2e/fixture/server/terraform-apply b/test/e2e/fixture/server/terraform-apply new file mode 100755 index 00000000..a77db2fb --- /dev/null +++ b/test/e2e/fixture/server/terraform-apply @@ -0,0 +1,43 @@ +#!/bin/bash +# Initialize and apply terraform configuration for cloud e2e infrastructure +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +# Required environment variables +: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}" +: "${CF_ACCOUNT_ID:?CF_ACCOUNT_ID is required}" +: "${WORKERS_SUBDOMAIN:?WORKERS_SUBDOMAIN is required}" + +# Validate we're targeting the correct account +echo "Validating Cloudflare account..." >&2 +ACCOUNT_NAME=$(curl -s -X GET "https://api.cloudflare.com/client/v4/accounts/$CF_ACCOUNT_ID" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json" | jq -r '.result.name // empty') + +if [ -z "$ACCOUNT_NAME" ]; then + echo "ERROR: Could not fetch account info for CF_ACCOUNT_ID=$CF_ACCOUNT_ID" >&2 + echo "Check your CLOUDFLARE_API_TOKEN and CF_ACCOUNT_ID" >&2 + exit 1 +fi + +echo "Deploying to account: $ACCOUNT_NAME (subdomain: $WORKERS_SUBDOMAIN)" >&2 + +# Optional: unique test run ID (defaults to "local") +TEST_RUN_ID="${E2E_TEST_RUN_ID:-local}" + +echo "Initializing terraform..." >&2 +terraform init -input=false -upgrade >&2 + +echo "Applying terraform configuration..." >&2 +terraform apply -auto-approve -input=false \ + -var="cloudflare_api_token=$CLOUDFLARE_API_TOKEN" \ + -var="cloudflare_account_id=$CF_ACCOUNT_ID" \ + -var="workers_subdomain=$WORKERS_SUBDOMAIN" \ + -var="test_run_id=$TEST_RUN_ID" \ + >&2 + +# Output the values for use by other scripts +echo "Terraform outputs:" >&2 +terraform output -json diff --git a/test/e2e/fixture/server/terraform-destroy b/test/e2e/fixture/server/terraform-destroy new file mode 100755 index 00000000..cbfa70a3 --- /dev/null +++ b/test/e2e/fixture/server/terraform-destroy @@ -0,0 +1,51 @@ +#!/bin/bash +# Destroy all terraform-managed e2e infrastructure +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +# Required environment variables +: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}" +: "${CF_ACCOUNT_ID:?CF_ACCOUNT_ID is required}" +: "${WORKERS_SUBDOMAIN:?WORKERS_SUBDOMAIN is required}" + +# Optional: unique test run ID (defaults to "local") +TEST_RUN_ID="${E2E_TEST_RUN_ID:-local}" + +# Check if terraform state exists +if [ ! -f "terraform.tfstate" ]; then + echo "No terraform state found, nothing to destroy" >&2 + exit 0 +fi + +# Get the R2 bucket name from terraform state before destroying +R2_BUCKET=$(terraform output -raw r2_bucket_name 2>/dev/null || echo "") + +# Empty the R2 bucket first (required before deletion) +if [ -n "$R2_BUCKET" ]; then + echo "Emptying R2 bucket: $R2_BUCKET" >&2 + # List and delete all objects in the bucket using wrangler + # Note: wrangler r2 object delete requires object keys, so we list first + npx wrangler r2 object list "$R2_BUCKET" --json 2>/dev/null | \ + jq -r '.objects[].key' 2>/dev/null | \ + while read -r key; do + if [ -n "$key" ]; then + npx wrangler r2 object delete "$R2_BUCKET/$key" 2>/dev/null || true + fi + done + echo "R2 bucket emptied" >&2 +fi + +echo "Destroying terraform-managed infrastructure..." >&2 +terraform destroy -auto-approve -input=false \ + -var="cloudflare_api_token=$CLOUDFLARE_API_TOKEN" \ + -var="cloudflare_account_id=$CF_ACCOUNT_ID" \ + -var="workers_subdomain=$WORKERS_SUBDOMAIN" \ + -var="test_run_id=$TEST_RUN_ID" + +# Clean up local state files +rm -f terraform.tfstate terraform.tfstate.backup +rm -rf .terraform .terraform.lock.hcl + +echo "Terraform infrastructure destroyed" >&2 diff --git a/test/e2e/fixture/server/variables.tf b/test/e2e/fixture/server/variables.tf new file mode 100755 index 00000000..e1c0659f --- /dev/null +++ b/test/e2e/fixture/server/variables.tf @@ -0,0 +1,21 @@ +variable "cloudflare_api_token" { + description = "Cloudflare API token with Access and R2 permissions" + type = string + sensitive = true +} + +variable "cloudflare_account_id" { + description = "Cloudflare account ID" + type = string +} + +variable "workers_subdomain" { + description = "Your workers.dev subdomain (e.g., 'myaccount' for myaccount.workers.dev)" + type = string +} + +variable "test_run_id" { + description = "Unique identifier for this test run (e.g., PR number or timestamp)" + type = string + default = "local" +} diff --git a/test/e2e/fixture/server/wait-ready b/test/e2e/fixture/server/wait-ready new file mode 100755 index 00000000..49f46e0b --- /dev/null +++ b/test/e2e/fixture/server/wait-ready @@ -0,0 +1,43 @@ +#!/bin/bash +# Wait for the deployed worker to be ready (container cold start can take 1-2 min) +set -e + +WORKER_URL="$1" +GATEWAY_TOKEN="$2" +CF_ACCESS_CLIENT_ID="$3" +CF_ACCESS_CLIENT_SECRET="$4" + +if [ -z "$WORKER_URL" ] || [ -z "$GATEWAY_TOKEN" ] || [ -z "$CF_ACCESS_CLIENT_ID" ] || [ -z "$CF_ACCESS_CLIENT_SECRET" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +TIMEOUT_SECONDS=300 # 5 minutes for cloud cold start +START_TIME=$(date +%s) + +echo "Waiting for worker to be ready at $WORKER_URL..." >&2 + +while true; do + ELAPSED=$(($(date +%s) - START_TIME)) + if [ "$ELAPSED" -ge "$TIMEOUT_SECONDS" ]; then + echo "Timeout waiting for worker after ${ELAPSED}s" >&2 + exit 1 + fi + + # Make request with Access service token headers + status=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID" \ + -H "CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET" \ + "$WORKER_URL/?token=$GATEWAY_TOKEN" 2>/dev/null || echo "000") + + if [ "$status" = "200" ]; then + echo "Worker is ready! (HTTP $status after ${ELAPSED}s)" >&2 + echo "ready" + exit 0 + fi + + if [ $((ELAPSED % 15)) -eq 0 ]; then + echo "Still waiting... (${ELAPSED}s elapsed, last status: $status)" >&2 + fi + sleep 2 +done diff --git a/test/e2e/fixture/start-browser b/test/e2e/fixture/start-browser index c8887f65..6338db6c 100755 --- a/test/e2e/fixture/start-browser +++ b/test/e2e/fixture/start-browser @@ -1,27 +1,43 @@ #!/bin/bash -# Start playwright-cli browser session for E2E testing +# Start playwright-cli browser session for E2E testing with Access headers set -e SESSION_NAME="moltworker-e2e" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# Stop and delete any existing session (delete needed to change headed/headless mode) +# Support running directly (not via cctr) +if [ -z "$CCTR_FIXTURE_DIR" ]; then + CCTR_FIXTURE_DIR="/tmp/e2e-cloud-manual" +fi + +# Stop and delete any existing session playwright-cli session-stop "$SESSION_NAME" >/dev/null 2>&1 || true playwright-cli session-delete "$SESSION_NAME" >/dev/null 2>&1 || true -# Build the open command args +# Build the args GLOBAL_ARGS=("--session=$SESSION_NAME") -# Run headed if PLAYWRIGHT_HEADED is set if [ "${PLAYWRIGHT_HEADED:-}" = "1" ] || [ "${PLAYWRIGHT_HEADED:-}" = "true" ]; then GLOBAL_ARGS+=("--headed") fi -# Open the browser to a blank page first (will navigate later in tests) -# Redirect all playwright output to /dev/null since it's very verbose +# Open the browser to a blank page first playwright-cli "${GLOBAL_ARGS[@]}" open "about:blank" >/dev/null 2>&1 & - -# Give it a moment to start sleep 2 +# Read Access credentials +CF_ACCESS_CLIENT_ID=$(cat "$CCTR_FIXTURE_DIR/cf-access-client-id.txt" 2>/dev/null || echo "") +CF_ACCESS_CLIENT_SECRET=$(cat "$CCTR_FIXTURE_DIR/cf-access-client-secret.txt" 2>/dev/null || echo "") + +if [ -n "$CF_ACCESS_CLIENT_ID" ] && [ -n "$CF_ACCESS_CLIENT_SECRET" ]; then + # Set extra HTTP headers for Access authentication + playwright-cli "${GLOBAL_ARGS[@]}" run-code "async page => { + await page.context().setExtraHTTPHeaders({ + 'CF-Access-Client-Id': '$CF_ACCESS_CLIENT_ID', + 'CF-Access-Client-Secret': '$CF_ACCESS_CLIENT_SECRET' + }); + }" >/dev/null 2>&1 +fi + echo "ready" diff --git a/test/e2e/fixture/start-server b/test/e2e/fixture/start-server index 8e28a1d6..b0a9f1a3 100755 --- a/test/e2e/fixture/start-server +++ b/test/e2e/fixture/start-server @@ -1,177 +1,17 @@ #!/bin/bash -# Start the moltworker for E2E testing - -set -e - -VERBOSE=false -if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then - VERBOSE=true -fi - -log() { - if [ "$VERBOSE" = true ]; then - echo "[start-server] $*" >&2 - fi -} - -# Support running directly (not via cctr) for manual debugging -if [ -z "$CCTR_TEST_PATH" ]; then - SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - CCTR_TEST_PATH="$(dirname "$SCRIPT_DIR")" - log "CCTR_TEST_PATH not set, using: $CCTR_TEST_PATH" -fi -if [ -z "$CCTR_FIXTURE_DIR" ]; then - CCTR_FIXTURE_DIR="/tmp/e2e-manual" - mkdir -p "$CCTR_FIXTURE_DIR" - log "CCTR_FIXTURE_DIR not set, using: $CCTR_FIXTURE_DIR" -fi - -PROJECT_DIR="$(cd "$CCTR_TEST_PATH/../.." && pwd)" -PORT=8686 -GATEWAY_TOKEN="e2e-test-token-1234567890" - -log "Project directory: $PROJECT_DIR" -log "Fixture directory: $CCTR_FIXTURE_DIR" -log "Port: $PORT" -log "Gateway token: $GATEWAY_TOKEN" - -# Kill any existing server on our port -log "Killing any existing server on port $PORT..." -pkill -f "wrangler.*--port.*$PORT" 2>/dev/null || true -pkill -f "wrangler dev" 2>/dev/null || true -sleep 0.5 - -# Stop any existing sandbox containers -log "Stopping any existing sandbox containers..." -docker ps -q --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker stop 2>/dev/null || true -docker ps -aq --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker rm 2>/dev/null || true - -cd "$PROJECT_DIR" - -# Install dependencies if needed -if [ ! -d node_modules ]; then - log "Installing dependencies..." - npm install --silent 2>/dev/null -fi - -# Build the project (required after code changes) -log "Building project..." -if [ "$VERBOSE" = true ]; then - npm run build >&2 -else - npm run build >/dev/null 2>&1 -fi - -# Write token to a file so tests can read it -echo "$GATEWAY_TOKEN" > "$CCTR_FIXTURE_DIR/gateway-token.txt" - -# Generate complete .dev.vars.e2e by copying from .dev.vars and overriding what we need -log "Creating .dev.vars.e2e..." -cat > "$CCTR_FIXTURE_DIR/.dev.vars.e2e" << EOF -E2E_TEST_MODE=true -DEBUG_ROUTES=true -MOLTBOT_GATEWAY_TOKEN=$GATEWAY_TOKEN -EOF - -# Copy all other settings from existing .dev.vars (except the ones we override) -if [ -f "$PROJECT_DIR/.dev.vars" ]; then - log "Copying settings from .dev.vars..." - grep -v -E "^(E2E_TEST_MODE|DEV_MODE|DEBUG_ROUTES|MOLTBOT_GATEWAY_TOKEN)=" "$PROJECT_DIR/.dev.vars" >> "$CCTR_FIXTURE_DIR/.dev.vars.e2e" 2>/dev/null || true -fi - -# Also pick up API keys and channel tokens from environment (for CI) -for var in AI_GATEWAY_API_KEY AI_GATEWAY_BASE_URL ANTHROPIC_API_KEY OPENAI_API_KEY \ - TELEGRAM_BOT_TOKEN TELEGRAM_DM_POLICY TELEGRAM_DM_ALLOW_FROM \ - DISCORD_BOT_TOKEN DISCORD_DM_POLICY \ - SLACK_BOT_TOKEN SLACK_APP_TOKEN; do - if [ -n "${!var}" ]; then - echo "$var=${!var}" >> "$CCTR_FIXTURE_DIR/.dev.vars.e2e" - fi -done - -if [ "$VERBOSE" = true ]; then - log "Generated .dev.vars.e2e contents:" - cat "$CCTR_FIXTURE_DIR/.dev.vars.e2e" >&2 -fi - -# Temporarily rename .dev.vars so wrangler ONLY reads our test config -if [ -f "$PROJECT_DIR/.dev.vars" ]; then - log "Temporarily moving .dev.vars out of the way..." - mv "$PROJECT_DIR/.dev.vars" "$PROJECT_DIR/.dev.vars.e2e-backup" -fi - -# Copy our test config to .dev.vars location so wrangler finds it -cp "$CCTR_FIXTURE_DIR/.dev.vars.e2e" "$PROJECT_DIR/.dev.vars" - -log "Starting wrangler dev..." -# Start wrangler in background, logging to file -# Use nohup and redirect all output to detach from terminal -nohup npx wrangler dev \ - --port "$PORT" \ - > "$CCTR_FIXTURE_DIR/wrangler.log" 2>&1 & -WRANGLER_PID=$! -echo $WRANGLER_PID > "$CCTR_FIXTURE_DIR/wrangler.pid" -log "Wrangler PID: $WRANGLER_PID" - -# In verbose mode, tail the log in background so we can see output -if [ "$VERBOSE" = true ]; then - tail -f "$CCTR_FIXTURE_DIR/wrangler.log" >&2 & - TAIL_PID=$! -fi - -# Give wrangler a moment to read the config, then restore original .dev.vars -sleep 2 -if [ -f "$PROJECT_DIR/.dev.vars.e2e-backup" ]; then - log "Restoring original .dev.vars..." - mv "$PROJECT_DIR/.dev.vars.e2e-backup" "$PROJECT_DIR/.dev.vars" -fi - -# Wait for server to be ready (container startup can take 1-2 minutes) -log "Waiting for server to be ready..." -consecutive_503=0 -TIMEOUT_SECONDS=180 -START_TIME=$(date +%s) -while true; do - ELAPSED=$(($(date +%s) - START_TIME)) - if [ "$ELAPSED" -ge "$TIMEOUT_SECONDS" ]; then - log "Timeout waiting for server after ${ELAPSED}s" - [ -n "$TAIL_PID" ] && kill $TAIL_PID 2>/dev/null || true - cat "$CCTR_FIXTURE_DIR/wrangler.log" >&2 - exit 1 - fi - - # Check for 200 response, not just any response - status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$PORT/?token=$GATEWAY_TOKEN" 2>/dev/null || echo "000") - if [ "$status" = "200" ]; then - log "Server is ready! (HTTP $status after ${ELAPSED}s)" - log "Open: http://localhost:$PORT/?token=$GATEWAY_TOKEN" - # Kill the tail process if running - [ -n "$TAIL_PID" ] && kill $TAIL_PID 2>/dev/null || true - # Small delay to let stderr flush before stdout - sleep 0.1 - echo "ready" - exit 0 - fi - - # Track consecutive 503 errors - these indicate the gateway is failing repeatedly - if [ "$status" = "503" ]; then - consecutive_503=$((consecutive_503 + 1)) - # After 3 consecutive 503s, check for fatal errors in the log - if [ "$consecutive_503" -ge 3 ]; then - if grep -q "Config invalid" "$CCTR_FIXTURE_DIR/wrangler.log" 2>/dev/null; then - log "Fatal error: Gateway config is invalid" - [ -n "$TAIL_PID" ] && kill $TAIL_PID 2>/dev/null || true - echo "ERROR: Gateway failed to start due to invalid config:" >&2 - grep -A5 "Config invalid" "$CCTR_FIXTURE_DIR/wrangler.log" | head -20 >&2 - exit 1 - fi - fi - else - consecutive_503=0 - fi - - if [ "$VERBOSE" = true ] && [ $((ELAPSED % 10)) -lt 2 ]; then - log "Still waiting... (${ELAPSED}s elapsed, last status: $status)" - fi - sleep 1 -done +# Start the moltworker for E2E testing (cloud deployment) +# +# Required environment variables: +# - CLOUDFLARE_API_TOKEN: API token with Workers, Access, R2 permissions +# - CF_ACCOUNT_ID: Cloudflare account ID +# - WORKERS_SUBDOMAIN: Your workers.dev subdomain +# - CF_ACCESS_TEAM_DOMAIN: Cloudflare Access team domain +# - R2_ACCESS_KEY_ID: R2 access key for bucket mounting +# - R2_SECRET_ACCESS_KEY: R2 secret key for bucket mounting +# +# Optional: +# - E2E_TEST_RUN_ID: Unique ID for this test run (default: "local") +# - AI_GATEWAY_API_KEY, AI_GATEWAY_BASE_URL: For AI provider +# - ANTHROPIC_API_KEY: Direct Anthropic access + +exec "$(dirname "$0")/server/start" "$@" diff --git a/test/e2e/fixture/stop-server b/test/e2e/fixture/stop-server index 82fb2d61..23a9caff 100755 --- a/test/e2e/fixture/stop-server +++ b/test/e2e/fixture/stop-server @@ -1,37 +1,9 @@ #!/bin/bash -# Stop the moltworker and clean up - -set -e - -# Stop wrangler if running -if [ -f "$CCTR_FIXTURE_DIR/wrangler.pid" ]; then - pid=$(cat "$CCTR_FIXTURE_DIR/wrangler.pid") - if kill -0 "$pid" 2>/dev/null; then - kill "$pid" 2>/dev/null || true - # Wait for it to die - for i in {1..10}; do - if ! kill -0 "$pid" 2>/dev/null; then - break - fi - sleep 0.5 - done - # Force kill if still running - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$CCTR_FIXTURE_DIR/wrangler.pid" -fi - -# Kill any remaining wrangler processes on our port -pkill -f "wrangler.*--port.*8686" 2>/dev/null || true -pkill -f "wrangler dev" 2>/dev/null || true - -# Stop and remove sandbox containers -docker ps -q --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker stop 2>/dev/null || true -docker ps -aq --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker rm 2>/dev/null || true - -# Clean up temp files -rm -f "$CCTR_FIXTURE_DIR/.dev.vars.e2e" -rm -f "$CCTR_FIXTURE_DIR/wrangler.log" -rm -f "$CCTR_FIXTURE_DIR/gateway-token.txt" - -echo "stopped" +# Stop the moltworker and clean up ALL cloud resources +# +# This will: +# 1. Delete the deployed worker +# 2. Destroy terraform resources (Access app, service token, R2 bucket) +# 3. Clean up local state files + +exec "$(dirname "$0")/server/stop" "$@" diff --git a/test/e2e/log_redaction.txt b/test/e2e/log_redaction.txt deleted file mode 100644 index af00f860..00000000 --- a/test/e2e/log_redaction.txt +++ /dev/null @@ -1,50 +0,0 @@ -=== -make request with secret query param (issue #85) -=== -curl -s -o /dev/null "http://localhost:8686/sandbox-health?secret=supersecretvalue123&other=visible" -echo "request sent" ---- -request sent - -=== -verify secret value is NOT in logs (issue #85) -%require -=== -if grep -q "supersecretvalue123" "$CCTR_FIXTURE_DIR/wrangler.log"; then - echo "FAIL: secret value found in logs" - grep "supersecretvalue123" "$CCTR_FIXTURE_DIR/wrangler.log" - exit 1 -else - echo "PASS: secret value not found in logs" -fi ---- -PASS: secret value not found in logs - -=== -verify REDACTED placeholder IS in logs (issue #85) -=== -# The [REDACTED] value appears URL-encoded in logs as %5BREDACTED%5D -if grep -qE "(\[REDACTED\]|%5BREDACTED%5D)" "$CCTR_FIXTURE_DIR/wrangler.log"; then - echo "PASS: [REDACTED] found in logs" -else - echo "FAIL: [REDACTED] not found in logs" - grep -i redact "$CCTR_FIXTURE_DIR/wrangler.log" || echo "(no redact matches)" - exit 1 -fi ---- -PASS: [REDACTED] found in logs - -=== -verify gateway token value is NOT in request logs -=== -TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") -# Check specifically in [REQ] lines - the token appears elsewhere (e.g. config output) -if grep "\[REQ\]" "$CCTR_FIXTURE_DIR/wrangler.log" | grep -q "$TOKEN"; then - echo "FAIL: gateway token found in [REQ] logs" - grep "\[REQ\].*$TOKEN" "$CCTR_FIXTURE_DIR/wrangler.log" - exit 1 -else - echo "PASS: gateway token not found in [REQ] logs" -fi ---- -PASS: gateway token not found in [REQ] logs diff --git a/test/e2e/pairing_and_conversation.txt b/test/e2e/pairing_and_conversation.txt index 86717189..7ae70dcb 100644 --- a/test/e2e/pairing_and_conversation.txt +++ b/test/e2e/pairing_and_conversation.txt @@ -1,26 +1,10 @@ -=== -navigate to main page to trigger pairing request -%require -=== -TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") -./pw --session=moltworker-e2e open "http://localhost:8686/?token=$TOKEN" ---- - -=== -wait for websocket connection to establish -%require -=== -./pw --session=moltworker-e2e run-code "async page => { - await page.waitForLoadState('networkidle'); -}" ---- - === navigate to admin page to approve device %require === TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") -./pw --session=moltworker-e2e open "http://localhost:8686/_admin/?token=$TOKEN" +WORKER_URL=$(cat "$CCTR_FIXTURE_DIR/worker-url.txt") +./pw --session=moltworker-e2e open "$WORKER_URL/_admin/?token=$TOKEN" --- === @@ -28,7 +12,7 @@ wait for pending devices section to load %require === ./pw --session=moltworker-e2e run-code "async page => { - await page.waitForSelector('text=Pending Pairing Requests', { timeout: 60000 }); + await page.waitForSelector('text=Pending Pairing Requests', { timeout: 120000 }); }" --- @@ -37,7 +21,7 @@ wait for Approve All button and click it %require === ./pw --session=moltworker-e2e run-code "async page => { - const btn = await page.waitForSelector('button:has-text(\"Approve All\")', { timeout: 60000 }); + const btn = await page.waitForSelector('button:has-text(\"Approve All\")', { timeout: 120000 }); await btn.click(); }" --- @@ -47,7 +31,7 @@ wait for approval to complete %require === ./pw --session=moltworker-e2e run-code "async page => { - await page.waitForSelector('text=No pending pairing requests', { timeout: 60000 }); + await page.waitForSelector('text=No pending pairing requests', { timeout: 120000 }); }" --- @@ -56,7 +40,8 @@ navigate back to main chat page %require === TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") -./pw --session=moltworker-e2e open "http://localhost:8686/?token=$TOKEN" +WORKER_URL=$(cat "$CCTR_FIXTURE_DIR/worker-url.txt") +./pw --session=moltworker-e2e open "$WORKER_URL/?token=$TOKEN" --- === @@ -64,7 +49,7 @@ wait for chat interface to load %require === ./pw --session=moltworker-e2e run-code "async page => { - await page.waitForSelector('textarea', { timeout: 60000 }); + await page.waitForSelector('textarea', { timeout: 120000 }); }" ---