Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
.git
bazel-*
bin
web/**/.next
web/**/coverage
web/**/dist
web/**/node_modules
web/**/playwright-report
web/**/test-results
web/**/*.tsbuildinfo
6 changes: 4 additions & 2 deletions .github/actions/upload-testlogs/action.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: Upload Bazel test logs
description: Upload bazel-testlogs on job failure so truncated CI output is not the only diagnostic.
description: Upload bazel-testlogs and test outputs (for example Playwright traces and compose logs) on job failure so truncated CI output is not the only diagnostic.

runs:
using: composite
Expand All @@ -8,6 +8,8 @@ runs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: testlogs-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
path: bazel-testlogs/**/test.log
path: |
bazel-testlogs/**/test.log
bazel-testlogs/**/test.outputs/**
if-no-files-found: warn
retention-days: 7
34 changes: 34 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,22 @@ jobs:
if: ${{ failure() }}
uses: ./.github/actions/upload-testlogs

web:
name: Web
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
persist-credentials: false
- uses: ./.github/actions/setup
- name: Run web checks
run: make web-check

- name: Upload Bazel failure logs
if: ${{ failure() }}
uses: ./.github/actions/upload-testlogs

# ---------------------------------------------------------------------------
# INTEGRATION TESTS (e2e, gateway, orchestrator)
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -135,6 +151,22 @@ jobs:
if: ${{ failure() }}
uses: ./.github/actions/upload-testlogs

web-e2e:
name: Web E2E
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
persist-credentials: false
- uses: ./.github/actions/setup
- name: Run real-stack web E2E
run: make web-e2e-test

- name: Upload Bazel failure logs
if: ${{ failure() }}
uses: ./.github/actions/upload-testlogs

gateway-integration-test:
name: Gateway Integration Test
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
Expand Down Expand Up @@ -354,6 +386,8 @@ jobs:
- merger-integration-test
- storage-integration-test
- consumer-integration-test
- web
- web-e2e
- workflow-security
steps:
- name: Fail if any required check did not succeed
Expand Down
10 changes: 10 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,13 @@ bin/
# Documentation site build output
/tool/docsite/build/
/tool/docsite/hooks/__pycache__/

# Web workspace build output
/web/node_modules/
/web/**/node_modules/
/web/**/dist/
/web/**/.next/
/web/**/coverage/
/web/**/playwright-report/
/web/**/test-results/
/web/**/*.tsbuildinfo
1 change: 1 addition & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ load("@gazelle//:def.bzl", "gazelle")
# Exclude nested worktrees (created under .claude/worktrees) so gazelle does not
# index them as duplicate rule definitions and corrupt the canonical BUILD files.
# gazelle:exclude .claude
# gazelle:exclude web

# Disable Gazelle proto rule generation globally. We rely on the checked-in
# .pb.go files in protopb/ dirs (plain go_library) rather than Bazel proto
Expand Down
54 changes: 54 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,15 @@ bazel_dep(name = "git", version = "2.55.0")
bazel_dep(name = "rules_go", version = "0.57.0")
bazel_dep(name = "gazelle", version = "0.45.0")
bazel_dep(name = "rules_proto", version = "7.1.0")
bazel_dep(name = "aspect_rules_js", version = "3.5.0")
bazel_dep(name = "aspect_rules_ts", version = "3.10.1")
bazel_dep(name = "rules_nodejs", version = "6.7.3")
bazel_dep(name = "rules_oci", version = "2.3.0")
bazel_dep(name = "rules_pkg", version = "1.0.1")
bazel_dep(name = "rules_playwright", version = "0.5.3")
bazel_dep(name = "rules_python", version = "2.2.0")
bazel_dep(name = "bazel_lib", version = "3.7.2")
bazel_dep(name = "platforms", version = "1.1.0")

single_version_override(
module_name = "git",
Expand All @@ -29,6 +37,52 @@ use_repo(protoc, "toolchains_protoc_hub")

register_toolchains("@toolchains_protoc_hub//:all")

node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
node.toolchain(node_version = "22.20.0")

typescript = use_extension("@aspect_rules_ts//ts:extensions.bzl", "typescript")
typescript.deps(version_from = "//web:package.json")
use_repo(typescript, "npm_typescript")

npm = use_extension("@aspect_rules_js//npm:extensions.bzl", "npm")
npm.npm_translate_lock(
name = "npm",
pnpm_lock = "//web:pnpm-lock.yaml",
)
use_repo(npm, "npm")

playwright = use_extension("@rules_playwright//playwright:extensions.bzl", "playwright")
playwright.repo(
name = "playwright",
integrity_path_map = {
"builds/chromium/1193/chromium-linux.zip": "sha256-AzVW84unLl+wfBqcmm4P9WtYI8LMQF4DS6V2OmLirRI=",
"builds/chromium/1193/chromium-mac-arm64.zip": "sha256-v7p0571A21zCoO9gPsm6kXNs9ragg2wnpZabV7gES2E=",
},
playwright_version = "1.55.1",
)
use_repo(playwright, "playwright")

oci = use_extension("@rules_oci//oci:extensions.bzl", "oci")
oci.pull(
name = "node_22_20_0_bookworm_slim",
digest = "sha256:b21fe589dfbe5cc39365d0544b9be3f1f33f55f3c86c87a76ff65a02f8f5848e",
image = "node",
platforms = ["linux/amd64"],
)
oci.pull(
name = "mysql_8_0",
digest = "sha256:7dcddc01f13bab2f15cde676d44d01f61fc9f99fe7785e86196dfc07d358ae2b",
image = "mysql",
platforms = ["linux/amd64"],
)
use_repo(
oci,
"mysql_8_0",
"mysql_8_0_linux_amd64",
"node_22_20_0_bookworm_slim",
"node_22_20_0_bookworm_slim_linux_amd64",
)

GO_VERSION = "1.25.0"

go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
Expand Down
50 changes: 41 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ COMPOSE = docker-compose

# SubmitQueue compose files
COMPOSE_FILE = service/submitqueue/docker-compose.yml
WEB_COMPOSE_FILE = service/submitqueue/docker-compose.web.yml
GATEWAY_COMPOSE_FILE = service/submitqueue/gateway/server/docker-compose.yml
ORCHESTRATOR_COMPOSE_FILE = service/submitqueue/orchestrator/server/docker-compose.yml

Expand Down Expand Up @@ -41,6 +42,11 @@ PROTO_PACKAGES = api/base/change api/base/hook api/base/mergestrategy api/base/m
# Set REPO_ROOT for docker-compose
export REPO_ROOT := $(shell pwd)

# Local-only credential for the reference web host. Override it for manual
# testing; production authentication is deliberately outside this demo.
export SUBMITQUEUE_WEB_TOKEN ?= test
export SUBMITQUEUE_WEB_CURSOR_SECRET ?= local-demo-cursor-secret

# Which provider the demo stack targets, and the only difference between a free
# local run and a live one. Selects a configuration directory rather than a code
# path, so adding a provider is mostly adding a directory — see
Expand Down Expand Up @@ -148,7 +154,7 @@ define assert_clean
fi
endef

.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps docs-build docs-serve e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go help
.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps docs-build docs-serve e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go web-build web-check web-e2e-test web-image-load web-install web-proto help


build: ## Build all services and examples
Expand Down Expand Up @@ -371,7 +377,7 @@ local-submitqueue-clean: ## Stop the stack and remove its volumes, images, and P
@echo "Cleaning all services and data..."
@# The overlay is named so that volumes it declares — Runway's checkouts —
@# are removed too, rather than surviving as an orphan.
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down -v --rmi local
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down -v --rmi local
@rm -rf "$(SQ_GIT_SANDBOX_DIR)"
@echo "All services, volumes, images, and $(SQ_GIT_SANDBOX_DIR) removed."

Expand Down Expand Up @@ -460,7 +466,7 @@ local-runway-stop: ## Stop Runway service
@echo "Runway services stopped."

local-submitqueue-logs: ## View logs from all running services
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) logs -f
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) logs -f

local-submitqueue-orchestrator-start: build-submitqueue-orchestrator-linux ## Start Orchestrator service locally (Orchestrator + 2 MySQL databases)
@echo "Starting Orchestrator with docker-compose..."
Expand All @@ -484,11 +490,12 @@ local-submitqueue-orchestrator-stop: ## Stop Orchestrator service
local-submitqueue-ps: ## Show running containers and their ports
@echo "Running containers and ports:"
@echo ""
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) ps
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) ps
@echo ""
@echo "📡 Service Endpoints:"
@echo " Gateway gRPC: localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'not running')"
@echo " Orchestrator gRPC: localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'not running')"
@echo " Web UX: http://localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-web-service-1 3000 2>/dev/null | cut -d: -f2 || echo 'not running')"
@echo ""
@echo "🗄️ Database Endpoints:"
@echo " MySQL App: localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'not running')"
Expand All @@ -506,11 +513,11 @@ local-submitqueue-ps: ## Show running containers and their ports

local-submitqueue-restart: build-all-linux ## Restart all services (rebuild and restart)
@echo "Restarting all services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) restart
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) restart
@echo "Services restarted!"
@make local-submitqueue-ps

local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git|github; github needs GITHUB_TOKEN)
local-submitqueue-start: build-all-linux web-image-load ## Start full stack (PROVIDER=fake|git|github; github needs GITHUB_TOKEN)
@echo "Starting full stack against provider '$(PROVIDER)' ($(SQ_PROVIDER_CONFIG_DIR))..."
@test -f "$(SQ_PROVIDER_CONFIG_DIR)/merge.yaml" \
|| { echo "No such provider '$(PROVIDER)': $(SQ_PROVIDER_CONFIG_DIR)/merge.yaml not found"; exit 2; }
Expand All @@ -529,13 +536,16 @@ local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git|
else \
export SQ_CONTAINER_USER=$$(id -u):$$(id -g); \
fi; \
$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) up -d --build --wait
$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) up -d --build --wait
@echo "Applying database schemas..."
@$(MAKE) -s local-init-submitqueue-schemas
@echo ""
@echo "✅ Stack is running against provider '$(PROVIDER)'."
@echo ""
@echo "Gateway gRPC port: $$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
@echo "Web UX: http://localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-web-service-1 3000 2>/dev/null | cut -d: -f2 || echo 'unknown')"
@echo "Web username: test"
@echo "Web token: $(SUBMITQUEUE_WEB_TOKEN)"
@if [ "$(PROVIDER)" = "git" ]; then \
echo "Land target: $(SQ_GIT_SANDBOX_DIR)/sandbox.git"; \
fi
Expand All @@ -545,7 +555,7 @@ local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git|

local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sandbox; the databases do not survive)
@echo "Stopping SubmitQueue services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@# Both MySQL services mount anonymous volumes, so `down` detaches them and
@# the next `up` creates fresh ones. Saying "data preserved" here would be
@# read as "your requests are still there", which they are not.
Expand All @@ -557,7 +567,7 @@ local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sand

local-stop: ## Stop every local stack — SubmitQueue, Stovepipe, and Runway
@echo "Stopping all services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(RUNWAY_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down
@echo "Services stopped. Anonymous database volumes are not reused on the next start."
Expand Down Expand Up @@ -687,6 +697,28 @@ tidy-go: ## Run go mod tidy
@echo "Running go mod tidy..."
@$(BAZEL) run @rules_go//go -- mod tidy -e

web-build: ## Build the web packages and deployable Next.js image with Bazel
@$(BAZEL) build //web/submitqueue:pkg //web/service/submitqueue:image

web-check: ## Run web generation, packaging, lint, type, unit, and image checks with Bazel
@$(BAZEL) test //web:check --test_output=errors
@$(BAZEL) build //web/service/submitqueue:image

web-e2e-test: ## Run the Bazel-managed real-stack Playwright and axe web test
@$(BAZEL) test //web/test/e2e:web_test --test_output=errors

web-image-load: ## Build and load the deployable web image into Docker with Bazel (also tagged with SQ_DOCKER_IMAGE_PREFIX)
@$(BAZEL) run //web/service/submitqueue:image_load
@if [ -n "$$SQ_DOCKER_IMAGE_PREFIX" ] && [ "$$SQ_DOCKER_IMAGE_PREFIX" != "submitqueue" ]; then \
docker tag submitqueue-web-service:latest "$$SQ_DOCKER_IMAGE_PREFIX-web-service:latest"; \
fi

web-install: ## Install the pinned web workspace dependencies
@corepack pnpm@10.17.1 --dir web install --frozen-lockfile

web-proto: ## Regenerate the committed TypeScript protobuf API with Bazel
@$(BAZEL) run //web/api:generate

help: ## Show this help message
@echo "Available targets:"
@echo ""
Expand Down
1 change: 1 addition & 0 deletions REPO.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ignore_directories(["**/node_modules"])
5 changes: 4 additions & 1 deletion api/base/change/proto/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
exports_files(
["change.proto"],
visibility = ["//tool/proto:__pkg__"],
visibility = [
"//tool/proto:__pkg__",
"//web/api:__pkg__",
],
)
5 changes: 4 additions & 1 deletion api/base/mergestrategy/proto/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
exports_files(
["mergestrategy.proto"],
visibility = ["//tool/proto:__pkg__"],
visibility = [
"//tool/proto:__pkg__",
"//web/api:__pkg__",
],
)
6 changes: 6 additions & 0 deletions doc/howto/DEVELOPMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,10 @@ CI runs `make lint`, `make check-tidy`, and `make check-gazelle`. `make lint` in
| `make gazelle` | Update `BUILD.bazel` files |
| `make mocks` | Regenerate mockgen files |
| `make check-mocks` | Fail if generated mocks are stale |
| `make web-build` | Build the web packages and deployable OCI image with Bazel |
| `make web-check` | Run web generation drift, package, lint, type, unit, and production image checks with Bazel |
| `make web-e2e-test` | Run the Bazel-managed real-stack Playwright and axe test |
| `make web-proto` | Regenerate the committed TypeScript protobuf API with Bazel |
| `make proto` | Regenerate protobuf files |
| `make clean` | Remove the Bazel cache and `bin/` |
| `make clean-proto` | Remove generated protobuf Go files |
Expand All @@ -118,6 +122,8 @@ CI runs `make lint`, `make check-tidy`, and `make check-gazelle`. `make lint` in
| `make local-stop` | Stop SubmitQueue, Stovepipe, and Runway |
| `make help` | List every target |

The web build does not require a host Node.js or pnpm installation: Bazel supplies Node, npm packages, Buf, TypeScript, Next.js, Playwright, Chromium, and the container image toolchain. `make web-install` is an optional convenience for editors and direct pnpm development only.

## Running Specific Tests

```bash
Expand Down
Loading
Loading