diff --git a/.env.example b/.env.example index 593c7577..7b5929b2 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,9 @@ API_HOST=127.0.0.1 API_PORT=8787 +WEB_HOST=0.0.0.0 +WEB_PORT=3000 +# LLM_* values are optional server-side defaults. Models can also be created in the Web UI after deploy. LLM_PROVIDER=openai-compatible LLM_MODEL=qwen-plus LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 @@ -76,7 +79,11 @@ SECRET_MASTER_KEY=replace-with-a-long-random-local-key DATAFOUNDRY_AUTH_MODE=password AUTH_SESSION_SECRET=replace-with-at-least-32-random-characters AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000 -# test = log verification/reset links to console; smtp = send real email. +# Required in password mode (open|closed). open = self-register; closed = REGISTRATION_CLOSED. +# Exposed via GET /api/v1/auth/status (no secrets). Do not use AUTH_EMAIL_DELIVERY=test +# with a non-loopback AUTH_PUBLIC_BASE_URL. +AUTH_REGISTRATION_MODE=open +# Required when set: only smtp|test. test = log links to console; smtp = send real email. AUTH_EMAIL_DELIVERY=test AUTH_EMAIL_FROM=DataFoundry AUTH_SMTP_HOST=smtp.example.com diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9da75ed2..ab80f885 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,12 @@ jobs: name: Core Smoke Tests runs-on: ubuntu-latest timeout-minutes: 25 + env: + DATAFOUNDRY_AUTH_MODE: password + AUTH_PUBLIC_BASE_URL: http://127.0.0.1:3000 + AUTH_REGISTRATION_MODE: open + AUTH_EMAIL_DELIVERY: test + AUTH_SESSION_SECRET: ci-auth-session-secret-with-32-bytes!! steps: - name: Checkout uses: actions/checkout@v5 @@ -65,6 +71,9 @@ jobs: - name: Build TypeScript workspaces run: npm run build + - name: Run formal auth foundation tests + run: npm run test:auth-foundation + - name: Generate built-in demo fixture env: SKIP_DTC_GROWTH_REGISTER: "1" @@ -97,6 +106,37 @@ jobs: - name: Run built-in DTC Growth regression run: node --test scripts/test-builtin-dtc-growth-datasource.mjs + native-deploy-smoke: + name: Native Deploy Smoke + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: "22" + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Run deployment unit tests + run: npm run test:deploy + + - name: Run native deployment smoke + run: npm run smoke:native-deploy + + - name: Upload redacted deployment logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: native-deploy-logs + path: storage/logs/ + if-no-files-found: ignore + docs: name: Docs runs-on: ubuntu-latest diff --git a/README.md b/README.md index 3cd32dd4..bc0e7d02 100644 --- a/README.md +++ b/README.md @@ -71,66 +71,54 @@ See the [v0.2.0 release notes](docs/en/releases/v0.2.0.md) for the complete capa ## 🚀 Formal deploy -Default path is the **formal** stack: `password` auth + `build` / `start` (do **not** run `npm run dev`). No business database required — built-in demo data sources, including the DTC Growth Review case, work out of the box. +Formal mode has two paths (do **not** run `npm run dev`). Docker / Compose is not provided in this release. -Two formal environments share the **same startup commands**: +### Recommended: Ubuntu / Debian one-click -| Environment | Use for | Email | Public URL | -| --- | --- | --- | --- | -| **Formal test** | Local / private acceptance | `AUTH_EMAIL_DELIVERY=test` (links in API console) | `http://127.0.0.1:3000` | -| **Real production** | Public service | `AUTH_EMAIL_DELIVERY=smtp` | Public HTTPS + reverse proxy | +`./deploy.sh` generates configuration, installs dependencies, builds (Web, API, and TUI), and starts Web + API as a detached background process — closing the terminal does not stop the stack. The TUI is built and ready but is a foreground client: start it in another terminal with `./deploy.sh tui` (or `npm run start:tui`); it does **not** stay running with the stack. No model key is required during deploy — create and enable a model profile in the Web UI after login. Does **not** support native Windows / macOS. ```bash git clone https://github.com/datagallery-lab/datafoundry.git cd datafoundry -npm install -cp .env.example .env -cp apps/web/.env.example apps/web/.env.local +./deploy.sh ``` -Configure any OpenAI-compatible model in the root `.env`, plus auth for formal test or real production (sample defaults lean formal test): +If a complete `.env` already exists, interactive deploy skips the configuration questions. To change ports or the public URL (secrets are preserved; `.env` is backed up first): ```bash -LLM_PROVIDER=openai-compatible -LLM_MODEL=qwen-plus # or deepseek-chat, gpt-4o, ... -LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 -LLM_API_KEY=replace-with-your-key - -DATAFOUNDRY_AUTH_MODE=password -AUTH_SESSION_SECRET=replace-with-at-least-32-random-characters -AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000 # real production: https://your.domain -AUTH_EMAIL_DELIVERY=test # real production: smtp + AUTH_SMTP_* +./deploy.sh deploy --reconfigure ``` -`apps/web/.env.local`: +Common management commands: ```bash -NEXT_PUBLIC_DATAFOUNDRY_AUTH_MODE=password -NEXT_PUBLIC_AGENT_RUNTIME_URL= -NEXT_PUBLIC_CONFIG_API_URL= -API_PROXY_TARGET=http://127.0.0.1:8787 +./deploy.sh status +./deploy.sh start +./deploy.sh stop +./deploy.sh logs +./deploy.sh doctor +./deploy.sh tui # optional: foreground TUI client (API must already be up) ``` -Build and start: +Open `http://127.0.0.1:3000/login`, register or sign in, create an OpenAI-compatible model profile, then go to `/data-tasks`. For remote hosts set `AUTH_PUBLIC_BASE_URL`. Re-running `./deploy.sh deploy` uses a maintenance window (stop the managed process group before install/build). + +### Windows / macOS / other: manual npm + +On native Windows, macOS, or other non-Ubuntu/Debian hosts, use manual npm. Install and run in the same environment; on Windows, do not share `node_modules` between Windows and WSL. ```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +npm install +cp .env.example .env +cp apps/web/.env.example apps/web/.env.local +# Edit .env / apps/web/.env.local (auth + optional model; see Quick Start) npm run build npm run build:web -npm run start:api # :8787 — /healthz liveness, /ready readiness -npm run start:web # :3000 — password mode via same-origin BFF -``` - -Open `http://127.0.0.1:3000/login`, register or sign in, go to `/data-tasks`, and ask: - -```text -Show me the tables in this datasource and explain the main fields of each. +npm run start # Web :3000 + API :8787 ``` -You will see the full chain: schema inspection → read-only SQL → SQL audit → table output → replayable run history. - -For real production, also configure SMTP and a reverse proxy: [`deploy/nginx.datafoundry.conf.example`](deploy/nginx.datafoundry.conf.example) (gzip/brotli static assets; keep `/api/copilotkit` uncompressed and unbuffered for SSE). - -> Full steps and the two-environment matrix: [Quick Start](docs/en/quick-start.md). Contributor hot-reload (`npm run dev`) is appendix-only. Connect your own PostgreSQL / MySQL / CSV and more: [Data Sources guide](docs/en/guides/data-sources.md). +> Full steps, the two-environment matrix,: [Quick Start](docs/en/quick-start.md). Connect your own PostgreSQL / MySQL / CSV and more: [Data Sources guide](docs/en/guides/data-sources.md). ## 🆚 How It Differs From Coding Agents And SQL Chatbots diff --git a/README_zh.md b/README_zh.md index d56d7e96..c80202d8 100644 --- a/README_zh.md +++ b/README_zh.md @@ -71,66 +71,54 @@ DataFoundry 0.2 在首个可用版本上，进一步补齐了有状态、可追 ## 🚀 正式态跑通 -默认按**正式态**部署：`password` 认证 + `build` / `start`（不要跑 `npm run dev`）。不需要准备业务数据库，包含 DTC 增长经营复盘在内的内置 demo 数据源开箱即用。 +正式态有两条路径（都不要跑 `npm run dev`）。本版本不提供 Docker / Compose。 -正式态分两种环境，**启动命令相同**： +### 推荐：Ubuntu / Debian 一键部署 -| 环境 | 用途 | 邮箱 | 公网地址 | -| --- | --- | --- | --- | -| **正式测试** | 本机 / 内网验收 | `AUTH_EMAIL_DELIVERY=test`（链接打控制台） | `http://127.0.0.1:3000` | -| **真实生产** | 对外服务 | `AUTH_EMAIL_DELIVERY=smtp` | 公网 HTTPS + 反代 | +`./deploy.sh` 自动生成配置、安装依赖、构建（含 Web、API 与 TUI），并以 detached 后台进程启动 Web + API——关闭终端一般不会停止服务。TUI 会在部署时构建就绪，但它是前台交互客户端：请另开终端执行 `./deploy.sh tui`（或 `npm run start:tui`），**不会**随 stack 后台常驻。部署时不要求填写模型 Key——登录后在 Web 中创建并启用模型即可。**不支持**原生 Windows / macOS。 ```bash git clone https://github.com/datagallery-lab/datafoundry.git cd datafoundry -npm install -cp .env.example .env -cp apps/web/.env.example apps/web/.env.local +./deploy.sh ``` -在根目录 `.env` 配置模型，并按正式测试或真实生产填好认证（样例默认偏正式测试）： +若已有完整 `.env`，交互部署会跳过配置问答。要改端口或公开访问地址时（会保留密钥并先备份 `.env`）： ```bash -LLM_PROVIDER=openai-compatible -LLM_MODEL=qwen-plus # 或 deepseek-chat、gpt-4o…… -LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 -LLM_API_KEY=replace-with-your-key - -DATAFOUNDRY_AUTH_MODE=password -AUTH_SESSION_SECRET=replace-with-at-least-32-random-characters -AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000 # 真实生产改为 https://你的域名 -AUTH_EMAIL_DELIVERY=test # 真实生产改为 smtp，并填 AUTH_SMTP_* +./deploy.sh deploy --reconfigure ``` -`apps/web/.env.local`： +常用管理命令： ```bash -NEXT_PUBLIC_DATAFOUNDRY_AUTH_MODE=password -NEXT_PUBLIC_AGENT_RUNTIME_URL= -NEXT_PUBLIC_CONFIG_API_URL= -API_PROXY_TARGET=http://127.0.0.1:8787 +./deploy.sh status +./deploy.sh start +./deploy.sh stop +./deploy.sh logs +./deploy.sh doctor +./deploy.sh tui # 可选：前台启动 TUI（需 API 已在运行） ``` -构建并启动： +打开 `http://127.0.0.1:3000/login` 注册登录，在模型配置中创建 OpenAI-compatible Profile，然后进入 `/data-tasks`。远程部署请设置 `AUTH_PUBLIC_BASE_URL`；重复执行 `./deploy.sh deploy` 会进入维护窗口（先停止旧进程再安装/构建）。 + +### Windows / macOS / 其他：手动 npm + +原生 Windows、macOS 或其他非 Ubuntu/Debian 环境请用手动 npm。请在同一环境内安装和运行；Windows 用户不要在 Windows 和 WSL 之间共用 `node_modules`。 ```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +npm install +cp .env.example .env +cp apps/web/.env.example apps/web/.env.local +# 编辑 .env / apps/web/.env.local（认证与可选模型，见快速开始） npm run build npm run build:web -npm run start:api # :8787 — /healthz 存活，/ready 就绪 -npm run start:web # :3000 — password 模式走同源 BFF -``` - -打开 `http://127.0.0.1:3000/login` 注册登录后进入 `/data-tasks`，提问： - -```text -帮我查看数据源里有哪些表，并说明每张表的主要字段。 +npm run start # Web :3000 + API :8787 ``` -你会看到完整的一条链路：schema 检查 → 只读 SQL → SQL 审计 → 表格产出 → 可回放的 run history。 - -真实生产请再配 SMTP 与反代：[`deploy/nginx.datafoundry.conf.example`](deploy/nginx.datafoundry.conf.example)（静态 gzip/brotli；`/api/copilotkit` 关压缩与缓冲以保护 SSE）。 - -> 完整步骤与两种环境对照见 [快速开始](docs/zh/quick-start.md)；贡献者热更新（`npm run dev`）仅见该文档附录。接入自己的 PostgreSQL / MySQL / CSV 等见 [数据源指南](docs/zh/guides/data-sources.md)。 +> 完整步骤与两种环境对照见 [快速开始](docs/zh/quick-start.md)。接入自己的 PostgreSQL / MySQL / CSV 等见 [数据源指南](docs/zh/guides/data-sources.md)。 ## 🆚 和 Coding Agent、SQL Chatbot 有什么不同 diff --git a/apps/api/src/auth/config.ts b/apps/api/src/auth/config.ts index 00259a8d..ecdccf5e 100644 --- a/apps/api/src/auth/config.ts +++ b/apps/api/src/auth/config.ts @@ -1,8 +1,12 @@ export type AuthMode = "dev" | "password"; +export type RegistrationMode = "open" | "closed"; export type PasswordAuthConfig = { mode: AuthMode; publicBaseUrl: string; + registrationMode: RegistrationMode; + cookiePath: string; + cookieSecure: boolean; sessionSecret: string; emailDelivery: "smtp" | "test"; smtp?: { @@ -15,13 +19,65 @@ export type PasswordAuthConfig = { }; }; +export function validateAuthPublicUrl(raw: string): { + publicBaseUrl: string; + loopback: boolean; + cookiePath: string; + cookieSecure: boolean; +} { + let url: URL; + try { + url = new URL(raw); + } catch { + throw new Error("AUTH_CONFIG_INVALID:AUTH_PUBLIC_BASE_URL must be a valid absolute URL."); + } + + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("AUTH_CONFIG_INVALID:AUTH_PUBLIC_BASE_URL must use http or https."); + } + if (url.username || url.password) { + throw new Error("AUTH_CONFIG_INVALID:AUTH_PUBLIC_BASE_URL must not include credentials."); + } + if (url.hash) { + throw new Error("AUTH_CONFIG_INVALID:AUTH_PUBLIC_BASE_URL must not include a fragment."); + } + + const host = url.hostname.toLowerCase().replace(/^\[(.*)\]$/, "$1"); + const loopback = host === "localhost" || host === "127.0.0.1" || host === "::1"; + if (url.protocol === "http:" && !loopback) { + throw new Error( + "AUTH_CONFIG_INVALID:AUTH_PUBLIC_BASE_URL HTTP is only allowed for loopback hosts (localhost, 127.0.0.1, ::1); use HTTPS for other hosts." + ); + } + + const normalized = new URL(url.href); + normalized.hash = ""; + normalized.search = ""; + // Keep pathname (including deployment prefix); strip only a trailing slash on root-equivalent leaves. + if (normalized.pathname.length > 1) { + normalized.pathname = normalized.pathname.replace(/\/+$/, ""); + } + + const cookiePath = normalized.pathname === "/" ? "/" : normalized.pathname; + + return { + publicBaseUrl: normalized.origin + (normalized.pathname === "/" ? "" : normalized.pathname), + loopback, + cookiePath, + cookieSecure: url.protocol === "https:" + }; +} + export function loadPasswordAuthConfig(env: Record): PasswordAuthConfig { const mode = parseAuthMode(env.DATAFOUNDRY_AUTH_MODE, env.NODE_ENV); const config: PasswordAuthConfig = { mode, publicBaseUrl: env.AUTH_PUBLIC_BASE_URL ?? "", + registrationMode: parseRegistrationMode(env.AUTH_REGISTRATION_MODE, mode === "password"), + cookiePath: "/", + cookieSecure: false, sessionSecret: env.AUTH_SESSION_SECRET ?? "", - emailDelivery: env.AUTH_EMAIL_DELIVERY === "test" ? "test" : "smtp" + emailDelivery: parseEmailDelivery(env.AUTH_EMAIL_DELIVERY) }; if (env.SMTP_HOST || env.AUTH_SMTP_HOST) { config.smtp = { @@ -37,6 +93,16 @@ export function loadPasswordAuthConfig(env: Record): } if (mode === "password") { validatePasswordAuthConfig(config); + } else if (config.publicBaseUrl) { + // Dev mode may still set a public URL; validate lightly when present. + try { + const validated = validateAuthPublicUrl(config.publicBaseUrl); + config.publicBaseUrl = validated.publicBaseUrl; + config.cookiePath = validated.cookiePath; + config.cookieSecure = validated.cookieSecure; + } catch { + // Keep legacy dev startups tolerant when AUTH_PUBLIC_BASE_URL is unused. + } } return config; } @@ -48,6 +114,29 @@ function parseAuthMode(value: string | undefined, nodeEnv: string | undefined): return nodeEnv === "production" ? "password" : "dev"; } +function parseRegistrationMode(value: string | undefined, required: boolean): RegistrationMode { + if (value === undefined || value.trim() === "") { + if (required) { + throw new Error("AUTH_CONFIG_MISSING:AUTH_REGISTRATION_MODE is required in password mode."); + } + return "open"; + } + if (value === "open" || value === "closed") { + return value; + } + throw new Error("AUTH_CONFIG_INVALID:AUTH_REGISTRATION_MODE must be open or closed."); +} + +function parseEmailDelivery(value: string | undefined): "smtp" | "test" { + if (value === undefined || value.trim() === "") { + return "smtp"; + } + if (value === "smtp" || value === "test") { + return value; + } + throw new Error("AUTH_CONFIG_INVALID:AUTH_EMAIL_DELIVERY must be smtp or test."); +} + function validatePasswordAuthConfig(config: PasswordAuthConfig): void { if (config.sessionSecret.length < 32) { throw new Error("AUTH_CONFIG_MISSING:AUTH_SESSION_SECRET must be at least 32 characters."); @@ -55,6 +144,16 @@ function validatePasswordAuthConfig(config: PasswordAuthConfig): void { if (!config.publicBaseUrl) { throw new Error("AUTH_CONFIG_MISSING:AUTH_PUBLIC_BASE_URL is required."); } + const validated = validateAuthPublicUrl(config.publicBaseUrl); + config.publicBaseUrl = validated.publicBaseUrl; + config.cookiePath = validated.cookiePath; + config.cookieSecure = validated.cookieSecure; + + if (config.emailDelivery === "test" && !validated.loopback) { + throw new Error( + "AUTH_CONFIG_INVALID:AUTH_EMAIL_DELIVERY=test is only allowed with a loopback AUTH_PUBLIC_BASE_URL." + ); + } if (config.emailDelivery === "smtp") { if (!config.smtp?.host || !config.smtp.from) { throw new Error("AUTH_CONFIG_MISSING:SMTP host and from address are required."); diff --git a/apps/api/src/auth/cookies.ts b/apps/api/src/auth/cookies.ts index 81623500..3c2b38e1 100644 --- a/apps/api/src/auth/cookies.ts +++ b/apps/api/src/auth/cookies.ts @@ -3,6 +3,11 @@ import type { IncomingMessage, ServerResponse } from "node:http"; export const SESSION_COOKIE = "df_session"; export const CSRF_COOKIE = "df_csrf"; +export type CookieSecurityOptions = { + path: string; + secure: boolean; +}; + export function parseCookies(request: IncomingMessage): Record { const header = request.headers.cookie; if (!header) { @@ -19,21 +24,53 @@ export function parseCookies(request: IncomingMessage): Record { export function appendAuthCookies(response: ServerResponse, input: { csrfToken: string; maxAgeSeconds: number; + path: string; sessionToken: string; + secure: boolean; }): void { appendSetCookie(response, serializeCookie(SESSION_COOKIE, input.sessionToken, { httpOnly: true, - maxAgeSeconds: input.maxAgeSeconds + maxAgeSeconds: input.maxAgeSeconds, + path: input.path, + secure: input.secure })); appendSetCookie(response, serializeCookie(CSRF_COOKIE, input.csrfToken, { httpOnly: false, - maxAgeSeconds: input.maxAgeSeconds + maxAgeSeconds: input.maxAgeSeconds, + path: input.path, + secure: input.secure + })); +} + +export function appendCsrfCookie( + response: ServerResponse, + value: string, + input: { path: string; secure: boolean; maxAgeSeconds: number } +): void { + appendSetCookie(response, serializeCookie(CSRF_COOKIE, value, { + httpOnly: false, + maxAgeSeconds: input.maxAgeSeconds, + path: input.path, + secure: input.secure })); } -export function appendClearAuthCookies(response: ServerResponse): void { - appendSetCookie(response, serializeCookie(SESSION_COOKIE, "", { httpOnly: true, maxAgeSeconds: 0 })); - appendSetCookie(response, serializeCookie(CSRF_COOKIE, "", { httpOnly: false, maxAgeSeconds: 0 })); +export function appendClearAuthCookies( + response: ServerResponse, + options: CookieSecurityOptions +): void { + appendSetCookie(response, serializeCookie(SESSION_COOKIE, "", { + httpOnly: true, + maxAgeSeconds: 0, + path: options.path, + secure: options.secure + })); + appendSetCookie(response, serializeCookie(CSRF_COOKIE, "", { + httpOnly: false, + maxAgeSeconds: 0, + path: options.path, + secure: options.secure + })); } function appendSetCookie(response: ServerResponse, cookie: string): void { @@ -46,14 +83,26 @@ function appendSetCookie(response: ServerResponse, cookie: string): void { response.setHeader("Set-Cookie", cookies); } -function serializeCookie(name: string, value: string, input: { httpOnly: boolean; maxAgeSeconds: number }): string { - const secure = process.env.NODE_ENV === "production"; +function serializeCookie( + name: string, + value: string, + input: { httpOnly: boolean; maxAgeSeconds: number; path: string; secure: boolean } +): string { + const path = normalizeCookiePath(input.path); return [ `${name}=${encodeURIComponent(value)}`, - "Path=/", + `Path=${path}`, `Max-Age=${input.maxAgeSeconds}`, "SameSite=Lax", - ...(secure ? ["Secure"] : []), + ...(input.secure ? ["Secure"] : []), ...(input.httpOnly ? ["HttpOnly"] : []) ].join("; "); } + +function normalizeCookiePath(path: string): string { + if (!path || path === "/") { + return "/"; + } + const trimmed = path.replace(/\/+$/u, ""); + return trimmed.startsWith("/") ? trimmed : `/${trimmed}`; +} diff --git a/apps/api/src/auth/routes.ts b/apps/api/src/auth/routes.ts index f4bb3ac5..dded1ac9 100644 --- a/apps/api/src/auth/routes.ts +++ b/apps/api/src/auth/routes.ts @@ -2,10 +2,19 @@ import { createErrorResult, createSuccessResult, type AppErrorCode } from "@data import type { IncomingMessage, ServerResponse } from "node:http"; import type { AuthIdentity, AuthService } from "./service.js"; import { AuthError, userDto, workspaceDto } from "./service.js"; -import { appendAuthCookies, appendClearAuthCookies, CSRF_COOKIE, parseCookies, SESSION_COOKIE } from "./cookies.js"; +import { + appendAuthCookies, + appendClearAuthCookies, + appendCsrfCookie, + CSRF_COOKIE, + parseCookies, + SESSION_COOKIE +} from "./cookies.js"; export type AuthRouteContext = { authService: AuthService; + cookiePath: string; + cookieSecure: boolean; identity?: AuthIdentity; }; @@ -25,6 +34,10 @@ export async function handleAuthApiRequest( ? {} : await readJsonBody(request); + if (root === "status" && request.method === "GET") { + sendJson(response, 200, createSuccessResult(context.authService.getPublicStatus())); + return true; + } if (root === "register" && request.method === "POST") { const result = await context.authService.register({ email: requiredString(body.email, "email"), @@ -39,16 +52,22 @@ export async function handleAuthApiRequest( const result = await context.authService.login({ email: requiredString(body.email, "email"), password: requiredString(body.password, "password"), + client: optionalClient(body.client), ...requestMeta(request) }); appendAuthCookies(response, { csrfToken: result.csrfToken, maxAgeSeconds: result.maxAgeSeconds, - sessionToken: result.sessionToken + path: context.cookiePath, + sessionToken: result.sessionToken, + secure: context.cookieSecure }); sendJson(response, 200, createSuccessResult({ user: result.user, - workspace: result.workspace + workspace: result.workspace, + session: { + expiresAt: result.expiresAt + } })); return true; } @@ -76,6 +95,18 @@ export async function handleAuthApiRequest( } const identity = requireIdentity(context); + if (root === "csrf" && segments[1] === "refresh" && request.method === "POST") { + const rotated = context.authService.rotateCsrf(identity); + appendCsrfCookie(response, rotated.csrfToken, { + path: context.cookiePath, + secure: context.cookieSecure, + maxAgeSeconds: rotated.maxAgeSeconds + }); + sendJson(response, 200, createSuccessResult({ csrfToken: rotated.csrfToken }), { + "Cache-Control": "no-store" + }); + return true; + } if (isUnsafeMethod(request.method)) { context.authService.validateCsrf(identity, headerString(request.headers["x-csrf-token"])); } @@ -86,13 +117,19 @@ export async function handleAuthApiRequest( } if (root === "logout" && request.method === "POST") { const result = context.authService.logout(identity); - appendClearAuthCookies(response); + appendClearAuthCookies(response, { + path: context.cookiePath, + secure: context.cookieSecure + }); sendJson(response, 200, createSuccessResult(result)); return true; } if (root === "logout-all" && request.method === "POST") { const result = context.authService.logoutAll(identity); - appendClearAuthCookies(response); + appendClearAuthCookies(response, { + path: context.cookiePath, + secure: context.cookieSecure + }); sendJson(response, 200, createSuccessResult(result)); return true; } @@ -178,6 +215,16 @@ function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function optionalClient(value: unknown): "web" | "tui" | undefined { + if (value === undefined || value === null || value === "") { + return undefined; + } + if (value === "web" || value === "tui") { + return value; + } + throw new AuthError(400, "BAD_REQUEST", "client must be web or tui."); +} + function requestMeta(request: IncomingMessage): { ipAddress?: string | undefined; userAgent?: string | undefined } { return { ...(headerString(request.headers["x-forwarded-for"]) ?? request.socket.remoteAddress @@ -191,11 +238,21 @@ function headerString(value: string | string[] | undefined): string | undefined return Array.isArray(value) ? value[0] : value; } -function sendJson(response: ServerResponse, statusCode: number, body: unknown): void { +function sendJson( + response: ServerResponse, + statusCode: number, + body: unknown, + extraHeaders: Record = {} +): void { + const existingCookies = response.getHeader("Set-Cookie"); response.writeHead(statusCode, { "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Credentials": "true", - "Content-Type": "application/json; charset=utf-8" + "Content-Type": "application/json; charset=utf-8", + ...extraHeaders, + ...(existingCookies + ? { "Set-Cookie": existingCookies as string | string[] } + : {}) }); response.end(JSON.stringify(body)); } diff --git a/apps/api/src/auth/service.ts b/apps/api/src/auth/service.ts index 8115769e..082eceaf 100644 --- a/apps/api/src/auth/service.ts +++ b/apps/api/src/auth/service.ts @@ -10,10 +10,13 @@ import type { PasswordAuthConfig } from "./config.js"; import { AuthMailer } from "./mailer.js"; import { createSecretToken, hashPassword, hashToken, verifyPassword } from "./crypto.js"; -const SESSION_TTL_SECONDS = 60 * 60 * 24 * 30; +const WEB_SESSION_TTL_SECONDS = 30 * 24 * 60 * 60; +const TUI_SESSION_TTL_SECONDS = 7 * 24 * 60 * 60; const EMAIL_VERIFICATION_TTL_MS = 1000 * 60 * 60 * 24; const PASSWORD_RESET_TTL_MS = 1000 * 60 * 30; +export type AuthClientKind = "web" | "tui"; + export type AuthUserDto = { id: string; email?: string; @@ -40,6 +43,7 @@ export class AuthError extends Error { export class AuthService { private readonly mailer: AuthMailer; + private readonly dummyPasswordHash = hashPassword(createSecretToken()).then((result) => result.hash); constructor( private readonly metadataStore: MetadataStore, @@ -48,6 +52,13 @@ export class AuthService { this.mailer = new AuthMailer(config); } + getPublicStatus(): { publicBaseUrl: string; registrationEnabled: boolean } { + return { + publicBaseUrl: this.config.publicBaseUrl, + registrationEnabled: this.config.registrationMode === "open" + }; + } + async register(input: { displayName?: string | undefined; email: string; @@ -55,6 +66,9 @@ export class AuthService { password: string; userAgent?: string | undefined; }): Promise<{ user: AuthUserDto; workspace: AuthWorkspaceDto; verificationToken?: string }> { + if (this.config.registrationMode !== "open") { + throw new AuthError(403, "REGISTRATION_CLOSED", "Registration is closed for this deployment."); + } const email = normalizeEmail(input.email); assertPassword(input.password); this.checkRateLimit(`register:ip:${input.ipAddress ?? "unknown"}`, 5, 60 * 60); @@ -109,12 +123,14 @@ export class AuthService { } async login(input: { + client?: AuthClientKind | undefined; email: string; ipAddress?: string | undefined; password: string; userAgent?: string | undefined; }): Promise<{ csrfToken: string; + expiresAt: string; maxAgeSeconds: number; sessionToken: string; user: AuthUserDto; @@ -124,29 +140,34 @@ export class AuthService { this.checkRateLimit(`login:email:${email}`, 5, 60); this.checkRateLimit(`login:ip:${input.ipAddress ?? "unknown"}`, 20, 60); const user = this.metadataStore.users.findByEmail({ email }); - if (!user || user.disabled_at) { + const credential = user + ? this.metadataStore.userPasswordCredentials.find({ user_id: user.id }) + : undefined; + if (!user || user.disabled_at || !credential) { + await verifyPassword(await this.dummyPasswordHash, input.password); this.audit("auth.login_failed", { email, ipAddress: input.ipAddress, userAgent: input.userAgent }); throw new AuthError(401, "UNAUTHORIZED", "Invalid email or password."); } - if (!user.email_verified_at) { - this.audit("auth.login_unverified", { + if (!(await verifyPassword(credential.password_hash, input.password))) { + this.audit("auth.login_failed", { email, ipAddress: input.ipAddress, userAgent: input.userAgent, userId: user.id }); - throw new AuthError(403, "EMAIL_NOT_VERIFIED", "Email verification is required before login."); + throw new AuthError(401, "UNAUTHORIZED", "Invalid email or password."); } - const credential = this.metadataStore.userPasswordCredentials.find({ user_id: user.id }); - if (!credential || !(await verifyPassword(credential.password_hash, input.password))) { - this.audit("auth.login_failed", { + if (!user.email_verified_at) { + this.audit("auth.login_unverified", { email, ipAddress: input.ipAddress, userAgent: input.userAgent, userId: user.id }); - throw new AuthError(401, "UNAUTHORIZED", "Invalid email or password."); + throw new AuthError(403, "EMAIL_NOT_VERIFIED", "Email verification is required before login."); } + const maxAgeSeconds = input.client === "tui" ? TUI_SESSION_TTL_SECONDS : WEB_SESSION_TTL_SECONDS; + const expiresAt = new Date(Date.now() + maxAgeSeconds * 1000).toISOString(); const sessionToken = createSecretToken(); const csrfToken = createSecretToken(); const session = this.metadataStore.authSessions.create({ @@ -154,7 +175,7 @@ export class AuthService { user_id: user.id, token_hash: hashToken(sessionToken, this.config.sessionSecret), csrf_token_hash: hashToken(csrfToken, this.config.sessionSecret), - expires_at: new Date(Date.now() + SESSION_TTL_SECONDS * 1000).toISOString(), + expires_at: expiresAt, ...(input.ipAddress ? { ip_address: input.ipAddress } : {}), ...(input.userAgent ? { user_agent: input.userAgent } : {}) }); @@ -162,13 +183,14 @@ export class AuthService { this.audit("auth.login_succeeded", { email, ipAddress: input.ipAddress, - metadata: { sessionId: session.id }, + metadata: { sessionId: session.id, client: input.client ?? "web" }, userAgent: input.userAgent, userId: user.id }); return { csrfToken, - maxAgeSeconds: SESSION_TTL_SECONDS, + expiresAt: session.expires_at, + maxAgeSeconds, sessionToken, user: userDto(user), workspace: workspaceDto(workspace) @@ -291,12 +313,33 @@ export class AuthService { validateCsrf(identity: AuthIdentity, csrfToken: string | undefined): void { if (!identity.session || !csrfToken) { - throw new AuthError(403, "FORBIDDEN", "CSRF token is required."); + throw new AuthError(403, "CSRF_INVALID", "CSRF token is required."); } const tokenHash = hashToken(csrfToken, this.config.sessionSecret); if (tokenHash !== identity.session.csrf_token_hash) { - throw new AuthError(403, "FORBIDDEN", "CSRF token is invalid."); + throw new AuthError(403, "CSRF_INVALID", "CSRF token is invalid."); + } + } + + rotateCsrf(identity: AuthIdentity): { csrfToken: string; maxAgeSeconds: number } { + if (!identity.session) { + throw new AuthError(401, "UNAUTHORIZED", "Authentication required."); + } + const csrfToken = createSecretToken(); + try { + this.metadataStore.authSessions.rotateCsrf({ + id: identity.session.id, + csrf_token_hash: hashToken(csrfToken, this.config.sessionSecret) + }); + } catch (error) { + if (error instanceof Error && error.message.startsWith("AUTH_SESSION_CSRF_ROTATE_FAILED:")) { + throw new AuthError(401, "UNAUTHORIZED", "Authentication required."); + } + throw error; } + const remainingMs = Date.parse(identity.session.expires_at) - Date.now(); + const maxAgeSeconds = Math.max(0, Math.floor(remainingMs / 1000)); + return { csrfToken, maxAgeSeconds }; } logout(identity: AuthIdentity): { ok: boolean } { diff --git a/apps/api/src/interaction-runtime-adapter.test.ts b/apps/api/src/interaction-runtime-adapter.test.ts new file mode 100644 index 00000000..e372aa63 --- /dev/null +++ b/apps/api/src/interaction-runtime-adapter.test.ts @@ -0,0 +1,110 @@ +import { EventType, type BaseEvent } from "@ag-ui/client"; +import { describe, expect, it } from "vitest"; + +import { + buildHitlSuspendBridgeEvents, + type HitlToolCallBoundaryState, + type InteractionInterrupt +} from "./interaction-runtime-adapter.js"; + +const interrupt: InteractionInterrupt = { + args: { question: "继续吗？" }, + resumeSchema: { type: "string" }, + runId: "run-1", + suspendPayload: { question: "继续吗？" }, + toolCallId: "call_hitl_1", + toolName: "ask_user" +}; + +const interactionEvent = { + type: EventType.CUSTOM, + name: "interaction.requested", + value: { tool_call_id: interrupt.toolCallId }, + timestamp: 1 +} as BaseEvent; + +const onInterruptEvent = { + type: EventType.CUSTOM, + name: "on_interrupt", + value: JSON.stringify(interrupt), + timestamp: 2 +} as BaseEvent; + +function emptyState(): HitlToolCallBoundaryState { + return { + startedToolCallIds: new Set(), + endedToolCallIds: new Set() + }; +} + +describe("buildHitlSuspendBridgeEvents", () => { + it("synthesizes START then END when upstream never started the tool call", () => { + const state = emptyState(); + const events = buildHitlSuspendBridgeEvents({ + interrupt, + interactionEvent, + passthroughInterruptEvent: onInterruptEvent, + state + }); + + expect(events.map((event) => event.type)).toEqual([ + EventType.TOOL_CALL_START, + EventType.CUSTOM, + EventType.CUSTOM, + EventType.TOOL_CALL_END + ]); + expect(events[0]).toMatchObject({ + type: EventType.TOOL_CALL_START, + toolCallId: interrupt.toolCallId, + toolCallName: "ask_user" + }); + expect(events[1]).toBe(interactionEvent); + expect(events[2]).toBe(onInterruptEvent); + expect(events[3]).toMatchObject({ + type: EventType.TOOL_CALL_END, + toolCallId: interrupt.toolCallId + }); + expect(state.startedToolCallIds.has(interrupt.toolCallId)).toBe(true); + expect(state.endedToolCallIds.has(interrupt.toolCallId)).toBe(true); + }); + + it("still emits END before transport RUN_FINISHED when START already arrived", () => { + const state = emptyState(); + state.startedToolCallIds.add(interrupt.toolCallId); + + const events = buildHitlSuspendBridgeEvents({ + interrupt, + interactionEvent, + passthroughInterruptEvent: onInterruptEvent, + state + }); + + expect(events.map((event) => event.type)).toEqual([ + EventType.CUSTOM, + EventType.CUSTOM, + EventType.TOOL_CALL_END + ]); + expect(events.at(-1)).toMatchObject({ + type: EventType.TOOL_CALL_END, + toolCallId: interrupt.toolCallId + }); + // Caller appends transport-only RUN_FINISHED after this bridge sequence. + expect(events.some((event) => event.type === EventType.RUN_FINISHED)).toBe(false); + expect(state.endedToolCallIds.has(interrupt.toolCallId)).toBe(true); + }); + + it("does not duplicate END when the tool call already ended", () => { + const state = emptyState(); + state.startedToolCallIds.add(interrupt.toolCallId); + state.endedToolCallIds.add(interrupt.toolCallId); + + const events = buildHitlSuspendBridgeEvents({ + interrupt, + interactionEvent, + state + }); + + expect(events.map((event) => event.type)).toEqual([EventType.CUSTOM]); + expect(events[0]).toBe(interactionEvent); + }); +}); diff --git a/apps/api/src/interaction-runtime-adapter.ts b/apps/api/src/interaction-runtime-adapter.ts index cca5f335..d0fa759c 100644 --- a/apps/api/src/interaction-runtime-adapter.ts +++ b/apps/api/src/interaction-runtime-adapter.ts @@ -121,6 +121,57 @@ export const buildHitlToolCallStartEvent = (interrupt: InteractionInterrupt): Ba timestamp: Date.now() }) as BaseEvent; +/** + * Pair with {@link buildHitlToolCallStartEvent} before the transport-only RUN_FINISHED. + * Mastra's on_interrupt path never emits TOOL_CALL_END; without it, AbstractAgent + * verifyEvents rejects RUN_FINISHED while the tool call is still active. + */ +export const buildHitlToolCallEndEvent = (interrupt: InteractionInterrupt): BaseEvent => + ({ + type: EventType.TOOL_CALL_END, + toolCallId: interrupt.toolCallId, + toolCallName: interrupt.toolName, + timestamp: Date.now() + }) as BaseEvent; + +export type HitlToolCallBoundaryState = { + endedToolCallIds: Set; + startedToolCallIds: Set; +}; + +/** + * Server event-bridge for HITL suspend (persisted stream events only). + * Ensures START (if missing) and END (if missing) around the interaction / + * on_interrupt events. Caller must still send a transport-only RUN_FINISHED + * via the subscriber (not through the persist pipeline). + */ +export function buildHitlSuspendBridgeEvents(input: { + interrupt: InteractionInterrupt; + interactionEvent: BaseEvent; + passthroughInterruptEvent?: BaseEvent; + state: HitlToolCallBoundaryState; +}): BaseEvent[] { + const toolCallId = input.interrupt.toolCallId; + const events: BaseEvent[] = []; + + if (!input.state.startedToolCallIds.has(toolCallId)) { + events.push(buildHitlToolCallStartEvent(input.interrupt)); + input.state.startedToolCallIds.add(toolCallId); + } + + events.push(input.interactionEvent); + if (input.passthroughInterruptEvent) { + events.push(input.passthroughInterruptEvent); + } + + if (!input.state.endedToolCallIds.has(toolCallId)) { + events.push(buildHitlToolCallEndEvent(input.interrupt)); + input.state.endedToolCallIds.add(toolCallId); + } + + return events; +} + /** Extract a Mastra resume command from an AG-UI run request. */ export const extractInteractionResume = (input: RunAgentInput): InteractionResume | undefined => { if (!isRecord(input.forwardedProps) || !isRecord(input.forwardedProps.command)) { diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 395c74b1..6aa07187 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -65,7 +65,7 @@ import { resolveRunIdentity } from "./run-identity-orchestrator.js"; import { createRunMemoryAssembly } from "./run-memory-assembly.js"; import { extractLastUserText } from "./run-input.js"; import { - buildHitlToolCallStartEvent, + buildHitlSuspendBridgeEvents, extractInteractionResume, InteractionRuntimeAdapter } from "./interaction-runtime-adapter.js"; @@ -277,7 +277,12 @@ export const createServer = async (options: CreateServerOptions = {}): Promise(); + const endedToolCallIds = new Set(); const clearRunTimeout = (): void => { if (runTimeout) { clearTimeout(runTimeout); @@ -797,17 +803,23 @@ class DataFoundryAgUiAgent extends AbstractAgent { clearRunTimeout(); unregisterCancel(); suspended = true; - // R-018: persist TOOL_CALL_START with the interactions row when the stream - // never emitted one (common for Mastra on_interrupt before tool-start). - if (!startedToolCallIds.has(interactionRequested.interrupt.toolCallId)) { - emit(buildHitlToolCallStartEvent(interactionRequested.interrupt)); - startedToolCallIds.add(interactionRequested.interrupt.toolCallId); - } - emit(interactionRequested.event); - finalizer.suspend(); - // CopilotKit useInterrupt listens for the native Mastra interrupt event. - if (event.type === EventType.CUSTOM && event.name === "on_interrupt") { - emit(event); + // R-018 / AG-UI verifyEvents: close the tool-call span before transport RUN_FINISHED + // whether upstream already emitted START or Mastra skipped start/end entirely. + const bridgeEvents = buildHitlSuspendBridgeEvents({ + interrupt: interactionRequested.interrupt, + interactionEvent: interactionRequested.event, + ...(event.type === EventType.CUSTOM && event.name === "on_interrupt" + ? { passthroughInterruptEvent: event } + : {}), + state: { startedToolCallIds, endedToolCallIds } + }); + for (const bridgeEvent of bridgeEvents) { + if (bridgeEvent === interactionRequested.event) { + emit(bridgeEvent); + finalizer.suspend(); + continue; + } + emit(bridgeEvent); } // Stream must finalize so CopilotKit can surface the interrupt UI via onRunFinalized. // This synthetic terminal event is transport-only; suspended runs must not replay as finished. @@ -865,6 +877,13 @@ class DataFoundryAgUiAgent extends AbstractAgent { ) { startedToolCallIds.add(event.toolCallId); } + if ( + event.type === EventType.TOOL_CALL_END + && typeof event.toolCallId === "string" + && event.toolCallId.length > 0 + ) { + endedToolCallIds.add(event.toolCallId); + } emit(event); if (event.type === EventType.RUN_STARTED) { diff --git a/apps/web/.env.example b/apps/web/.env.example index 4299170a..14958d99 100755 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -1,11 +1,17 @@ # DataFoundry Web — formal deploy defaults (password + same-origin BFF). # +# Native one-click deploy (`./deploy.sh`) generates `apps/web/.env.local` for you. +# It uses same-origin BFF variables (empty NEXT_PUBLIC_* URLs) and sets +# API_PROXY_TARGET from the selected API host/port. +# # Keep in sync with root `.env` → DATAFOUNDRY_AUTH_MODE. # NEXT_PUBLIC_* values are baked in at `npm run build:web`. # # --- Formal test / real production (default) --- -# Same frontend settings for both; differ in root AUTH_EMAIL_DELIVERY and -# AUTH_PUBLIC_BASE_URL (see root .env.example). +# Same frontend settings for both; differ in root AUTH_EMAIL_DELIVERY, +# AUTH_PUBLIC_BASE_URL, and AUTH_REGISTRATION_MODE (see root .env.example). +# Registration UI may still follow NEXT_PUBLIC_DATAFOUNDRY_AUTH_MODE until +# M0A.5c reads GET /api/v1/auth/status; API policy is authoritative. NEXT_PUBLIC_DATAFOUNDRY_AUTH_MODE=password # Leave empty so the browser uses `/api/v1/*` and `/api/copilotkit` on the # Next origin (required for session cookies + CSRF). diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 00000000..6be7eb35 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/deploy/bootstrap.sh +source "${ROOT_DIR}/scripts/deploy/bootstrap.sh" + +main() { + check_supported_system + ensure_node_22 "$@" + exec node "$ROOT_DIR/scripts/deploy/cli.mjs" "$@" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/docs/en/architecture/overview.md b/docs/en/architecture/overview.md index 07e16913..99118ff9 100644 --- a/docs/en/architecture/overview.md +++ b/docs/en/architecture/overview.md @@ -109,10 +109,11 @@ The default path is formal mode (`password` + `build` / `start`); do not run `np ```bash npm run build && npm run build:web -npm run start:api -npm run start:web +npm run start ``` +`npm run start` manages Web and API with the same local-process deployment used previously. Separate `start:*` commands remain available for external process supervisors. + | Environment | Email | Public URL | | --- | --- | --- | | Formal test | `AUTH_EMAIL_DELIVERY=test` | Local / private | diff --git a/docs/en/index.md b/docs/en/index.md index 58845863..c25a869e 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -30,7 +30,7 @@ Review a realistic growth spike with schema inspection, read-only SQL, and trace ## Recommended path 1. Read [Product overview](overview.md) to confirm the problem space and capability boundaries. -2. Follow [Quick start](quick-start.md) to configure a model API key and run the built-in DTC Growth Review. +2. Follow [Quick start](quick-start.md): on Ubuntu / Debian prefer `./deploy.sh`; on Windows / macOS and other hosts use manual npm (`npm install` → configure → `build` / `start`). Then create a model profile in the Web UI and run the built-in DTC Growth Review. 3. Read [Capabilities](capabilities.md) to see coverage across Web, TUI, and backend API. 4. Choose [Web workbench guide](guides/web-workbench.md) or [TUI guide](guides/tui.md) based on your entry point. 5. When you need your own data, read [Data sources guide](guides/data-sources.md). diff --git a/docs/en/quick-start.md b/docs/en/quick-start.md index f838f361..100560e0 100644 --- a/docs/en/quick-start.md +++ b/docs/en/quick-start.md @@ -1,28 +1,81 @@ # Quick start -This guide is for first-time DataFoundry deployers. After reading it, you can start the Web workbench in the **formal** stack (`build` + `start`, `password` auth), configure a model service, and run a data analysis task against the built-in DTC Growth Review data source. +This guide is for first-time DataFoundry deployers. Formal mode has two paths; **both** start the Web workbench with `password` auth (do **not** run `npm run dev`): -Formal mode has two environments. **Startup commands are the same**; the main differences are email delivery and the public base URL: +| Path | Hosts | Entry | +| --- | --- | --- | +| **Recommended: one-click** | Ubuntu / Debian | `./deploy.sh` (config, dependencies, build including TUI, detached Web/API start, and health checks in one flow) | +| **Manual npm** | Windows, macOS, other Linux, or hand-edited env files | `npm install` → configure `.env` → `npm run build` / `build:web` → `npm run start` | -| Environment | Use for | `AUTH_EMAIL_DELIVERY` | `AUTH_PUBLIC_BASE_URL` | -| --- | --- | --- | --- | -| **Formal test** | Local / private acceptance | `test` (verification links go to the API console) | e.g. `http://127.0.0.1:3000` | -| **Real production** | Public service | `smtp` (real email) | Public HTTPS origin | +After deploy, configure a model in the Web UI and run an analysis against the built-in DTC Growth Review data source. Docker / Compose is **not** shipped in this release. -Do **not** run `npm run dev` / `dev:api` / `dev:web` in either formal environment. Contributor hot-reload is in the appendix. +## Requirements -You do not need a business database for the first run. You only need Node.js, npm, and a model API key compatible with the OpenAI `/chat/completions` interface. +- **One-click deploy**: Ubuntu or Debian (x86_64 / aarch64); Node.js 22 (the script can help install it after consent) +- **Manual npm**: Linux, macOS, or Windows; Node.js >= 22 and npm -## Requirements +Install and run the project in the same environment. On Windows, do not share `node_modules` between Windows and WSL. + +## Recommended: Ubuntu / Debian one-click deploy + +`./deploy.sh` does **not** support native Windows / macOS (use manual npm below). + +```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +./deploy.sh +``` + +On success Web + API keep running in the background (detached process group). Closing the terminal or pressing `Ctrl+C` in `./deploy.sh logs` does **not** stop DataFoundry — use `./deploy.sh stop`. One-click deploy also builds the TUI, but the TUI does **not** stay running with the stack; start it in another terminal when needed (see “Start the TUI” below). + +Open `http://127.0.0.1:3000/login` (or the Web URL printed by the script if the port differs), register and sign in, create/test/enable an OpenAI-compatible model profile, then go to `/data-tasks`. + +### Configuration rules + +- First run: the script generates `.env` and `apps/web/.env.local`, confirms ports / public URL. No model key is required at deploy time. +- Later interactive `./deploy.sh` / `./deploy.sh deploy`: if a complete `.env` already exists, configuration questions are skipped. +- To change ports or the public URL again (existing secrets are kept; `.env` is backed up first): + +```bash +./deploy.sh deploy --reconfigure +``` + +- Unattended / CI defaults (no prompts; fails immediately on port conflicts or install that needs a sudo password): + +```bash +./deploy.sh deploy --non-interactive +``` -- Node.js >= 22 -- npm -- Linux, macOS, or Windows -- A model API key—for example Qwen, DeepSeek, or another OpenAI-compatible service +`--reconfigure` and `--non-interactive` are mutually exclusive and only valid with `deploy`. -On Windows, install and run the project in the same environment. Do not share `node_modules` between Windows and WSL. +### Lifecycle commands -## 1. Install dependencies +```bash +./deploy.sh status # process + API / Web health +./deploy.sh start # start an existing build (no install/build) +./deploy.sh stop # stop only the managed process group +./deploy.sh restart # stop then start (no install/build) +./deploy.sh logs # follow runtime logs; Ctrl+C does not stop the stack +./deploy.sh doctor # read-only dependency / config / port / disk / health checks +./deploy.sh tui # optional: foreground TUI client (API must be healthy; not a managed service) +./deploy.sh help +``` + +`LLM_*` is not required during deploy. Set `AUTH_PUBLIC_BASE_URL` for remote hosts. Re-running deploy uses a maintenance window: stop the managed process group before `npm ci` and builds. + + +## Windows / macOS / other: manual npm deploy + +`./deploy.sh` targets **Ubuntu / Debian only** and does not support native Windows / macOS. On Windows, macOS, or other distros, install, configure, and start with npm as below. Use the same path for hand-edited env files or split processes. Do **not** run `npm run dev` in formal environments. Contributor hot-reload is in the appendix. + +Formal environments: + +| Environment | Use for | `AUTH_EMAIL_DELIVERY` | `AUTH_PUBLIC_BASE_URL` | +| --- | --- | --- | --- | +| **Formal test** | Local / private acceptance | `test` (verification links go to the API console) | e.g. `http://127.0.0.1:3000` | +| **Real production** | Public service | `smtp` (real email) | Public HTTPS origin | + +### 1. Install dependencies From the repository root: @@ -33,18 +86,18 @@ npm install `node -v` must report 22 or higher. The first install generates the local DTC Growth Review SQLite fixture and compiles workspace dependencies; time depends on your machine and network. -## 2. Configure environment variables +Install and run the project in the same environment. On Windows, do not share `node_modules` between Windows and WSL. -Copy the environment templates: +### 2. Configure environment variables ```bash cp .env.example .env cp apps/web/.env.example apps/web/.env.local ``` -### 2.1 Model (required for both formal environments) +#### 2.1 Model (optional server defaults) -Edit the root `.env` and set model configuration: +Edit the root `.env` for optional server-default models (you can also configure models only in the Web UI): ```bash LLM_PROVIDER=openai-compatible @@ -62,7 +115,7 @@ LLM_BASE_URL=https://api.deepseek.com LLM_API_KEY=your-api-key ``` -### 2.2 Formal test (recommended for first acceptance) +#### 2.2 Formal test (recommended for first acceptance) Root `.env`: @@ -87,7 +140,7 @@ API_PROXY_TARGET=http://127.0.0.1:8787 On register / password reset, copy the verification link from the **API process console**. -### 2.3 Real production +##### 2.3 Real production Start from the formal-test settings, then change to: @@ -106,7 +159,7 @@ AUTH_SMTP_PASSWORD= Keep the frontend on `password`, empty public API URLs, and `API_PROXY_TARGET`. Put a reverse proxy in front; see [`deploy/nginx.datafoundry.conf.example`](https://github.com/datagallery-lab/datafoundry/blob/main/deploy/nginx.datafoundry.conf.example) — compress static assets; keep `/api/copilotkit` uncompressed and unbuffered for SSE. -## 3. Build and start (same for formal test and real production) +### 3. Build and start (same for formal test and real production) ```bash npm run build @@ -126,7 +179,7 @@ Open [http://127.0.0.1:3000/login](http://127.0.0.1:3000/login) (or your public After changing any `NEXT_PUBLIC_*` value in `apps/web/.env.local`, run `npm run build:web` again. -## 4. Run your first question +## Run your first question On `/data-tasks`: @@ -149,12 +202,19 @@ Compare GMV, gross margin, ad spend, and refunds by channel. Explain which chann When you see schema inspection, SQL execution, and result output, the path is working. -## 5. Start the TUI +## Start the TUI + +One-click deploy builds the TUI during the build stage, but does **not** auto-start it and does **not** treat it as a managed background service. With the API running, start the foreground client in another terminal: + +```bash +./deploy.sh tui +# or: npm run start:tui +``` -With the backend running: +Optionally point at the deployed API URL (defaults to `API_PORT` from `.env`): ```bash -npm run start:tui +./deploy.sh tui --runtime-url http://127.0.0.1:8787/api/copilotkit ``` Demo mode without a backend: @@ -171,7 +231,17 @@ npm run start:tui -- --resume More commands: [TUI guide](guides/tui.md). -## 6. Troubleshooting +## Troubleshooting + +For one-click deploy, start with: + +```bash +./deploy.sh status +./deploy.sh doctor +./deploy.sh logs +``` + +On the manual npm path, confirm `npm run start` is still running and check that terminal's output. ### Wrong Node version @@ -181,9 +251,11 @@ Fix: ```bash node -v +# one-click: +./deploy.sh doctor ``` -Upgrade to Node.js 22 or higher, then run `npm install` again. +Upgrade to Node.js 22 or higher, then retry. One-click deploy can also install Node after consent; on the manual path, re-run `npm install`. ### Page does not load @@ -191,9 +263,9 @@ Symptom: Browser cannot open the workbench URL. Fix: -- Confirm `npm run start:web` is still running (do not use `dev` in formal mode). -- Check whether port 3000 is in use. -- If 3000 is taken, use the frontend port shown in terminal output. +- Run `./deploy.sh status` (or confirm `npm run start` is still running on the manual path). Do not use `dev` in formal mode. +- Check whether port 3000 is in use; if the deploy script chose another port, use the URL it printed. +- If the process is stopped: `./deploy.sh start`. ### Backend not running @@ -202,6 +274,7 @@ Symptom: Page loads but questions get no response, or the resource panel fails t Fix: ```bash +./deploy.sh status curl http://127.0.0.1:8787/healthz curl http://127.0.0.1:8787/ready ``` @@ -209,7 +282,8 @@ curl http://127.0.0.1:8787/ready If the health check fails: ```bash -npm run start:api +./deploy.sh start +# manual path: npm run start ``` ### No verification email diff --git a/docs/zh/architecture/overview.md b/docs/zh/architecture/overview.md index 4c633a70..6a933279 100644 --- a/docs/zh/architecture/overview.md +++ b/docs/zh/architecture/overview.md @@ -113,10 +113,11 @@ password 模式在 `/api/v1/auth/*` 下提供基于 Cookie 的会话、非安全 ```bash npm run build && npm run build:web -npm run start:api -npm run start:web +npm run start ``` +`npm run start` 仍以原有本地进程方式管理 Web 与 API。已有进程守护系统仍可使用拆分的 `start:*` 命令。 + | 环境 | 邮箱 | 公网地址 | | --- | --- | --- | | 正式测试 | `AUTH_EMAIL_DELIVERY=test` | 本机 / 内网 | diff --git a/docs/zh/index.md b/docs/zh/index.md index c39ce769..d539ec05 100644 --- a/docs/zh/index.md +++ b/docs/zh/index.md @@ -30,7 +30,7 @@ DataFoundry 是面向数据分析场景的 AI 工作台，把自然语言提问 ## 推荐体验路径 1. 阅读 [产品概览](overview.md)，确认 DataFoundry 解决的问题和能力边界。 -2. 按 [快速开始](quick-start.md) 配置模型 API Key，并使用内置 DTC Growth Review 跑通第一个问题。 +2. 按 [快速开始](quick-start.md)：Ubuntu / Debian 推荐 `./deploy.sh`；Windows / macOS 及其他环境用手动 npm（`npm install` → 配置 → `build` / `start`）。登录 Web 后创建模型 Profile，再用内置 DTC Growth Review 跑通第一个问题。 3. 查看 [能力全览](capabilities.md)，了解 Web 工作台、TUI 和后端 API 的能力覆盖。 4. 根据使用入口选择 [Web 工作台指南](guides/web-workbench.md) 或 [TUI 指南](guides/tui.md)。 5. 需要接入自有数据时，再阅读 [数据源指南](guides/data-sources.md)。 diff --git a/docs/zh/quick-start.md b/docs/zh/quick-start.md index 13d6f893..4cff8796 100644 --- a/docs/zh/quick-start.md +++ b/docs/zh/quick-start.md @@ -1,28 +1,81 @@ # 快速开始 -这篇文档面向第一次部署 DataFoundry 的用户。读完后，你可以按**正式态**启动 Web 工作台（`build` + `start`，`password` 认证），配置模型服务，用内置 DTC Growth Review 数据源跑通一次数据分析任务。 +这篇文档面向第一次部署 DataFoundry 的用户。正式态有两条路径，**启动后都是** `password` 认证的 Web 工作台（不要跑 `npm run dev`）： -正式态分两种环境，**启动命令相同**，差别主要在邮箱与公网地址： +| 路径 | 适用环境 | 入口 | +| --- | --- | --- | +| **推荐：一键部署** | Ubuntu / Debian | `./deploy.sh`（配置、依赖、构建含 TUI、detached 后台启动 Web/API 与健康检查一次完成） | +| **手动 npm** | Windows、macOS、其他 Linux，或需要手改环境变量时 | `npm install` → 配置 `.env` → `npm run build` / `build:web` → `npm run start` | -| 环境 | 用途 | `AUTH_EMAIL_DELIVERY` | `AUTH_PUBLIC_BASE_URL` | -| --- | --- | --- | --- | -| **正式测试** | 本机或内网验收、联调 | `test`（验证/重置链接打到控制台） | 如 `http://127.0.0.1:3000` | -| **真实生产** | 对外服务 | `smtp`（真实发信） | 公网 HTTPS 域名 | +部署完成后在 Web 中配置模型，再用内置 DTC Growth Review 数据源跑通一次分析。本版本**不提供** Docker / Compose。 -两种正式态都**不要跑** `npm run dev` / `dev:api` / `dev:web`。贡献者本地热更新见文末附录。 +## 环境要求 -首次体验不需要准备业务数据库。你只需要 Node.js、npm 和一个兼容 OpenAI `/chat/completions` 接口的模型 API Key。 +- **一键部署**：Ubuntu 或 Debian（x86_64 / aarch64）；Node.js 22（缺失时脚本可在确认后协助安装） +- **手动 npm**：Linux、macOS 或 Windows；Node.js >= 22 与 npm -## 环境要求 +请在同一环境内安装和运行项目。Windows 用户不要在 Windows 和 WSL 之间共用 `node_modules`。 + +## 推荐：Ubuntu / Debian 一键部署 + +`./deploy.sh` **不支持**原生 Windows / macOS（请改用下文手动 npm）。 + +```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +./deploy.sh +``` + +部署成功后 Web + API 在后台常驻（独立进程组）。关闭终端，或在 `./deploy.sh logs` 中按 `Ctrl+C`，**都不会**停止 DataFoundry——停止请用 `./deploy.sh stop`。一键部署会一并构建 TUI，但 TUI **不会**随 stack 后台常驻；需要时另开终端按成功提示启动（见下文「启动 TUI」）。 + +打开 `http://127.0.0.1:3000/login`（若端口被改过，以脚本打印的 Web 地址为准），注册并登录，在模型配置中创建、测试并启用 OpenAI-compatible Profile，然后进入 `/data-tasks`。 + +### 交互与配置规则 + +- 首次部署：脚本生成 `.env` 与 `apps/web/.env.local`，确认端口 / 公开访问地址。部署阶段不要求填写模型 Key。 +- 之后再执行交互式 `./deploy.sh` / `./deploy.sh deploy`：若已有完整 `.env`，会跳过配置问答。 +- 需要重新配置端口、DataLink 或公开访问地址时（保留现有密钥，并先备份 `.env`）： + +```bash +./deploy.sh deploy --reconfigure +``` + +- 无人值守 / CI 默认（不提问；端口冲突或需要 sudo 密码的安装会立即失败）： + +```bash +./deploy.sh deploy --non-interactive +``` -- Node.js >= 22 -- npm -- Linux、macOS 或 Windows -- 一个模型 API Key，例如通义千问、DeepSeek 或其他 OpenAI-compatible 服务 +`--reconfigure` 与 `--non-interactive` 互斥，且仅对 `deploy` 有效。 -Windows 用户请在同一个系统内安装和运行项目。不要在 Windows 和 WSL 之间共用 `node_modules`。 +### 生命周期命令 -## 1. 安装依赖 +```bash +./deploy.sh status # 进程与 API / Web 健康状态 +./deploy.sh start # 用已有构建启动（不安装、不构建） +./deploy.sh stop # 只停止受管进程组 +./deploy.sh restart # 停止后启动（不安装、不构建） +./deploy.sh logs # 跟随运行日志；Ctrl+C 不停止服务 +./deploy.sh doctor # 只读检查依赖 / 配置 / 端口 / 磁盘 / 健康 +./deploy.sh tui # 可选：前台启动 TUI（需 API 已健康；不是受管后台服务） +./deploy.sh help +``` + +部署时不要求填写 `LLM_*`。远程主机请设置 `AUTH_PUBLIC_BASE_URL`。重复部署会进入维护窗口：先停止受管进程组，再执行 `npm ci` 与构建。 + + +## Windows / macOS / 其他：手动 npm 部署 + +`./deploy.sh` **仅面向 Ubuntu / Debian**，不支持原生 Windows / macOS。在 Windows、macOS 或其他发行版上，请按下列步骤用 npm 安装、配置并启动。需要手改环境变量或拆分进程时，也可走这条路径。两种正式态都**不要跑** `npm run dev`。贡献者热更新见文末附录。 + +正式态对照： + +| 环境 | 用途 | `AUTH_EMAIL_DELIVERY` | `AUTH_PUBLIC_BASE_URL` | +| --- | --- | --- | --- | +| **正式测试** | 本机或内网验收、联调 | `test`（验证/重置链接打到控制台） | 如 `http://127.0.0.1:3000` | +| **真实生产** | 对外服务 | `smtp`（真实发信） | 公网 HTTPS 域名 | + +### 1. 安装依赖 在仓库根目录执行： @@ -33,18 +86,18 @@ npm install `node -v` 输出需要不低于 22。首次安装会生成本地 DTC Growth Review SQLite fixture，并编译工作区依赖；耗时取决于机器和网络。 -## 2. 配置环境变量 +请在同一环境内安装和运行项目。Windows 用户不要在 Windows 和 WSL 之间共用 `node_modules`。 -复制环境变量模板： +### 2. 配置环境变量 ```bash cp .env.example .env cp apps/web/.env.example apps/web/.env.local ``` -### 2.1 模型（两种正式态都要） +#### 2.1 模型（可选 server-default） -打开根目录 `.env`，填写模型配置： +打开根目录 `.env`，可填写可选的服务端默认模型（也可仅在 Web 中配置）： ```bash LLM_PROVIDER=openai-compatible @@ -62,7 +115,7 @@ LLM_BASE_URL=https://api.deepseek.com LLM_API_KEY=你的_API_Key ``` -### 2.2 正式测试（推荐首次验收） +#### 2.2 正式测试（推荐首次验收） 根目录 `.env`： @@ -87,7 +140,7 @@ API_PROXY_TARGET=http://127.0.0.1:8787 注册/重置密码时，验证链接会打印在 **API 进程控制台**，复制到浏览器即可。 -### 2.3 真实生产 +#### 2.3 真实生产 在正式测试配置基础上改为： @@ -106,7 +159,7 @@ AUTH_SMTP_PASSWORD= 前端同样保持 `password` + 空公开 API URL + `API_PROXY_TARGET`。对外入口请用反代，样例见 [`deploy/nginx.datafoundry.conf.example`](https://github.com/datagallery-lab/datafoundry/blob/main/deploy/nginx.datafoundry.conf.example)：静态资源压缩，SSE 路径 `/api/copilotkit` 关闭 gzip 与 `proxy_buffering`。 -## 3. 构建并启动（正式测试 / 真实生产相同） +### 3. 构建并启动（正式测试 / 真实生产相同） ```bash npm run build @@ -126,7 +179,7 @@ curl http://127.0.0.1:8787/ready # Mastra / builtin 就绪（含 startup_ms 改过 `apps/web/.env.local` 中的 `NEXT_PUBLIC_*` 后，需要重新执行 `npm run build:web`。 -## 4. 跑通第一个问题 +## 跑通第一个问题 打开 `/data-tasks` 后： @@ -149,12 +202,19 @@ curl http://127.0.0.1:8787/ready # Mastra / builtin 就绪（含 startup_ms 你看到 schema 检查、SQL 执行和结果产出后，说明链路已经跑通。 -## 5. 启动 TUI +## 启动 TUI + +一键部署会在构建阶段准备好 TUI，但**不会**自动启动，也不会把它当作后台受管进程。后端（API）运行后，另开终端启动前台客户端： + +```bash +./deploy.sh tui +# 或：npm run start:tui +``` -后端运行后，可以启动终端界面： +指定当前部署的 API 地址（可选；默认使用 `.env` 中的 `API_PORT`）： ```bash -npm run start:tui +./deploy.sh tui --runtime-url http://127.0.0.1:8787/api/copilotkit ``` 演示模式不需要后端： @@ -171,7 +231,17 @@ npm run start:tui -- --resume 更多命令见 [TUI 指南](guides/tui.md)。 -## 6. 排查 +## 排查 + +一键部署路径请先看： + +```bash +./deploy.sh status +./deploy.sh doctor +./deploy.sh logs +``` + +手动 npm 路径则确认 `npm run start` 仍在运行，并检查对应终端输出。 ### Node 版本不对 @@ -181,9 +251,11 @@ npm run start:tui -- --resume ```bash node -v +# 一键部署： +./deploy.sh doctor ``` -升级到 Node.js 22 或更高版本后，重新执行 `npm install`。 +升级到 Node.js 22 或更高版本后重试。一键部署也可让 `./deploy.sh` 在确认后协助安装；手动路径则重新执行 `npm install`。 ### 页面打不开 @@ -191,9 +263,9 @@ node -v 处理： -- 确认 `npm run start:web` 还在运行（正式态不要开 `dev`）。 -- 检查 3000 端口是否被占用。 -- 如果 3000 被占用，查看终端输出中的实际前端端口。 +- 先执行 `./deploy.sh status`（手动路径则确认 `npm run start` 仍在运行）。正式态不要开 `dev`。 +- 检查 3000 端口是否被占用；若部署时改过端口，以脚本打印的 Web 地址为准。 +- 若进程已停止：`./deploy.sh start`。 ### 后端未启动 @@ -202,6 +274,7 @@ node -v 处理： ```bash +./deploy.sh status curl http://127.0.0.1:8787/healthz curl http://127.0.0.1:8787/ready ``` @@ -209,7 +282,8 @@ curl http://127.0.0.1:8787/ready 如果健康检查失败： ```bash -npm run start:api +./deploy.sh start +# 手动路径：npm run start ``` ### 注册收不到邮件 diff --git a/package.json b/package.json index feaad081..f9f25c40 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,10 @@ "dev:web": "npm --workspace @datafoundry/web run dev", "build:web": "npm --workspace @datafoundry/web run build", "start:web": "npm --prefix apps/web run start", + "start": "node scripts/start.mjs", + "test:datalink-stack": "node --test scripts/datalink-stack-config.test.mjs", + "test:deploy": "node --test scripts/deploy/*.test.mjs scripts/stack-runtime-config.test.mjs", + "smoke:native-deploy": "node scripts/deploy/smoke-native-deploy.mjs", "test:web": "npm --workspace @datafoundry/web run test", "dev:tui": "npm --workspace @datafoundry/tui run dev", "start:tui": "npm --workspace @datafoundry/tui run start --", @@ -80,6 +84,7 @@ "smoke:long-term-memory": "npm run build && node scripts/smoke-long-term-memory.mjs", "smoke:memory-recall-shadow": "npm run build && node scripts/smoke-memory-recall-shadow.mjs", "smoke:metadata": "npm run build && node scripts/smoke-metadata.mjs", + "test:auth-foundation": "npm run build && node --test scripts/lib/authenticated-test-client.test.mjs scripts/auth-foundation.test.mjs", "smoke:auth": "npm run build && node scripts/smoke-auth.mjs", "smoke:files": "npm run build && node scripts/smoke-files.mjs", "smoke:skills": "npm run build && node scripts/smoke-skills.mjs", diff --git a/packages/agent-runtime/src/tools/governed-tool-factory.test.ts b/packages/agent-runtime/src/tools/governed-tool-factory.test.ts index d724e625..72845e54 100644 --- a/packages/agent-runtime/src/tools/governed-tool-factory.test.ts +++ b/packages/agent-runtime/src/tools/governed-tool-factory.test.ts @@ -140,6 +140,65 @@ describe("GovernedToolFactory protocol boundary", () => { }); }); + it("bypasses ActionRouter for externally resolved HITL tools so suspend can propagate", async () => { + let actionRouterCalled = false; + let suspendedPayload: unknown; + const runScope = { modelName: "test", resourceId: "user-1", runId: "run-1", sessionId: "session-1" }; + const boundary = createToolObservationBoundary({ identity: runScope }); + const dispatcher = new ToolObservationDispatcher(boundary.packager, runScope); + const factory = new GovernedToolFactory(dispatcher, undefined, undefined, { + actionRouter: { + execute: async () => { + actionRouterCalled = true; + return { + rawResult: { shouldNotRun: true }, + observation: { shouldNotRun: true }, + contextPackageRef: { packageId: "context-1", revision: 1 }, + contextPackage: { + version: 2 as const, + packageId: "context-1", + revision: 1, + items: [], + groups: [], + sourceSnapshots: [], + artifactRefs: [], + auditRefs: [], + truncation: [] + } + }; + } + }, + externallyResolvedToolNames: new Set(["ask_user"]), + runId: "run-1", + segmentId: "segment-1" + }); + const tool = factory.governTool("ask_user", { + execute: async (_input: unknown, options?: unknown) => { + const agent = (options as { agent?: { + suspend?: (payload: unknown) => Promise; + } } | undefined)?.agent; + await agent?.suspend?.({ question: "Which datasource?" }); + return undefined; + } + }); + + const result = await tool.execute?.( + { question: "Which datasource?" }, + { + agent: { + toolCallId: "call-ask", + suspend: async (payload: unknown) => { + suspendedPayload = payload; + } + } + } + ); + + expect(actionRouterCalled).toBe(false); + expect(result).toBeUndefined(); + expect(suspendedPayload).toEqual({ question: "Which datasource?" }); + }); + it("warns the model not to repeat an external action whose state commit failed", async () => { const emitted: unknown[] = []; const runScope = { modelName: "test", resourceId: "user-1", runId: "run-1", sessionId: "session-1" }; diff --git a/packages/agent-runtime/src/tools/governed-tool-factory.ts b/packages/agent-runtime/src/tools/governed-tool-factory.ts index 7f5ae1fe..5486ca51 100644 --- a/packages/agent-runtime/src/tools/governed-tool-factory.ts +++ b/packages/agent-runtime/src/tools/governed-tool-factory.ts @@ -91,7 +91,12 @@ export class GovernedToolFactory { const toolCallId = toolCallIdFromOptions(options); const toolInput = args[0]; try { - if (this.actionRouter) { + // HITL tools (ask_user / submit_plan) must call Mastra suspend directly. + // Routing them through ActionRouter records a synthetic success for `undefined` + // and breaks the on_interrupt → interaction.requested contract. + const useActionRouter = Boolean(this.actionRouter) + && !this.externallyResolvedToolNames.has(toolName); + if (useActionRouter && this.actionRouter) { const segmentId = this.getSegmentId?.() ?? this.segmentId; if (!this.runId || !segmentId) { throw new Error("GOVERNED_ACTION_ROUTER_SCOPE_REQUIRED"); diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 0301c756..717b7fd4 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -255,12 +255,14 @@ export type ArtifactSummary = { export type AppErrorCode = | "BAD_REQUEST" | "CONFLICT" + | "CSRF_INVALID" | "DATASOURCE_TEST_FAILED" | "EMAIL_NOT_VERIFIED" | "FORBIDDEN" | "INTERNAL_ERROR" | "JOB_NOT_FOUND" | "PROVIDER_TEST_FAILED" + | "REGISTRATION_CLOSED" | "REVISION_CONFLICT" | "SECRET_MASTER_KEY_REQUIRED" | "UNAUTHORIZED" diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts index 66e8a027..397cd733 100644 --- a/packages/metadata/src/index.ts +++ b/packages/metadata/src/index.ts @@ -1014,6 +1014,18 @@ export class AuthSessionRepository { .run(input.last_seen_at ?? new Date().toISOString(), input.id); } + rotateCsrf(input: { id: string; csrf_token_hash: string }): AuthSessionRecord { + const result = this.db.prepare(` + UPDATE auth_sessions + SET csrf_token_hash = ?, last_seen_at = ? + WHERE id = ? AND revoked_at IS NULL AND expires_at > ? + `).run(input.csrf_token_hash, new Date().toISOString(), input.id, new Date().toISOString()); + if (result.changes !== 1) { + throw new Error(`AUTH_SESSION_CSRF_ROTATE_FAILED:${input.id}`); + } + return this.get({ id: input.id }); + } + revoke(input: { id: string; revoked_at?: string }): void { this.db.prepare("UPDATE auth_sessions SET revoked_at = COALESCE(revoked_at, ?) WHERE id = ?") .run(input.revoked_at ?? new Date().toISOString(), input.id); @@ -1324,6 +1336,12 @@ export class SessionRepository { try { const placeholders = sessionIds.map(() => "?").join(", "); const scope = [input.user_id, ...sessionIds]; + const runIds = this.db.prepare(` + SELECT id FROM runs WHERE user_id = ? AND session_id IN (${placeholders}) + `).all(...scope) + .map((row) => (isRecord(row) && typeof row.id === "string" ? row.id : null)) + .filter((id): id is string => Boolean(id)); + const runPlaceholders = runIds.map(() => "?").join(", "); this.db.prepare(` DELETE FROM session_branches @@ -1361,6 +1379,17 @@ export class SessionRepository { DELETE FROM trace_sections WHERE user_id = ? AND session_id IN (${placeholders}) `).run(...scope); + if (runIds.length > 0) { + this.db.prepare(` + DELETE FROM protocol_state_snapshots + WHERE user_id = ? AND run_id IN (${runPlaceholders}) + `).run(input.user_id, ...runIds); + this.db.prepare(` + DELETE FROM protocol_event_journal + WHERE user_id = ? AND run_id IN (${runPlaceholders}) + `).run(input.user_id, ...runIds); + } + this.db.prepare(` DELETE FROM context_package_snapshots WHERE user_id = ? AND session_id IN (${placeholders}) `).run(...scope); @@ -1389,14 +1418,7 @@ export class SessionRepository { DELETE FROM run_events WHERE user_id = ? AND session_id IN (${placeholders}) `).run(...scope); - const runIds = this.db.prepare(` - SELECT id FROM runs WHERE user_id = ? AND session_id IN (${placeholders}) - `).all(...scope) - .map((row) => (isRecord(row) && typeof row.id === "string" ? row.id : null)) - .filter((id): id is string => Boolean(id)); - if (runIds.length > 0) { - const runPlaceholders = runIds.map(() => "?").join(", "); this.db.prepare(` DELETE FROM sql_audit_logs WHERE user_id = ? AND run_id IN (${runPlaceholders}) diff --git a/packages/metadata/src/session-delete.test.ts b/packages/metadata/src/session-delete.test.ts new file mode 100644 index 00000000..8159ba9a --- /dev/null +++ b/packages/metadata/src/session-delete.test.ts @@ -0,0 +1,71 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { createMetadataStore } from "./index.js"; + +describe("SessionRepository.delete", () => { + it("deletes persisted protocol state and journal events with the session", () => { + const root = mkdtempSync(join(tmpdir(), "session-delete-protocol-")); + const metadata = createMetadataStore({ database_path: join(root, "metadata.sqlite") }); + try { + metadata.sessions.create({ user_id: "dev-user", id: "session-1", title: "Protocol" }); + metadata.runs.create({ + user_id: "dev-user", + id: "run-1", + session_id: "session-1", + user_input: "test", + status: "completed" + }); + metadata.contextPackageSnapshots.create({ + user_id: "dev-user", + session_id: "session-1", + run_id: "run-1", + package_id: "context-1", + revision: 0, + payload: {} + }); + metadata.protocolStates.compareAndSetWithEvents({ + user_id: "dev-user", + run_id: "run-1", + segment_id: "segment-1", + expected_revision: -1, + state: { + protocolId: "general-task", + protocolVersion: "1", + runId: "run-1", + segmentId: "segment-1", + revision: 0, + phase: "work", + status: "completed", + contextPackageRef: { packageId: "context-1", revision: 0 }, + actions: [], + completionRejections: 0, + domain: {} + } + }, [{ + eventId: "event-1", + type: "protocol.run.completed", + runId: "run-1", + segmentId: "segment-1", + revision: 0 + }]); + + expect(metadata.sessions.delete({ user_id: "dev-user", session_id: "session-1" })).toEqual({ + deleted: true, + deletedSessionIds: ["session-1"] + }); + expect(metadata.sessions.list({ user_id: "dev-user" })).toHaveLength(0); + expect(metadata.protocolStates.find({ + user_id: "dev-user", + run_id: "run-1", + segment_id: "segment-1" + })).toBeUndefined(); + expect(metadata.protocolStates.pendingEvents({ user_id: "dev-user", run_id: "run-1" })).toEqual([]); + } finally { + metadata.close(); + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/auth-foundation.test.mjs b/scripts/auth-foundation.test.mjs new file mode 100644 index 00000000..ac9e8ac8 --- /dev/null +++ b/scripts/auth-foundation.test.mjs @@ -0,0 +1,659 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import { randomUUID } from "node:crypto"; + +import { createServer as createApiServer } from "../apps/api/dist/server.js"; +import { createMetadataStore } from "../packages/metadata/dist/index.js"; +import { + loadPasswordAuthConfig, + validateAuthPublicUrl +} from "../apps/api/dist/auth/config.js"; +import { + appendAuthCookies, + appendClearAuthCookies +} from "../apps/api/dist/auth/cookies.js"; + +const SCRIPTS_ROOT = dirname(fileURLToPath(import.meta.url)); + +const FORMAL_HTTP_AUTH_TARGETS = [ + "run-dacomp6-complex-case.mjs", + "seed-dtc-growth-demo.mjs", + "seed-local-fixtures.mjs", + "smoke-agent-protocol-deepseek.mjs", + "smoke-ask-user-interrupt.mjs", + "smoke-auth.mjs", + "smoke-config-api.mjs", + "smoke-copilotkit-run.mjs", + "smoke-copilotkit.mjs", + "smoke-interaction-run-id.mjs", + "smoke-password-frontend-isolation.mjs", + "smoke-server-datasources-e2e.mjs", + "test-builtin-dtc-growth-datasource.mjs", + "verify-token-usage-display.mjs" +]; + +const PUBLIC_HTTP_TARGETS = [ + "deploy/health.mjs", + "deploy/smoke-native-deploy.mjs" +]; + +const DIRECT_METADATA_FIXTURE_TARGETS = []; + +const AUTH_FOUNDATION_HARNESS_TARGETS = [ + "auth-foundation.test.mjs", + "lib/authenticated-test-client.test.mjs" +]; + +const FORBIDDEN_DEV_AUTH_PATTERNS = [ + /X-Dev-Token/i, + /\bdev-token\b/, + /Authorization\s*:\s*["']?Bearer\s+dev\b/i +]; + +function listMjsFiles(dir) { + /** @type {string[]} */ + const out = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) { + out.push(...listMjsFiles(full)); + } else if (entry.endsWith(".mjs")) { + out.push(full); + } + } + return out; +} + +function isHttpScript(source) { + return /\bfetch\s*\(|\bhttps?\.request\s*\(/.test(source); +} + +const SECRET = "auth-foundation-session-secret-32b!"; + +function baseEnv(overrides = {}) { + return { + DATAFOUNDRY_AUTH_MODE: "password", + AUTH_SESSION_SECRET: SECRET, + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + AUTH_EMAIL_DELIVERY: "test", + AUTH_REGISTRATION_MODE: "open", + ...overrides + }; +} + +test("validateAuthPublicUrl allows loopback HTTP without secure cookies", () => { + for (const raw of [ + "http://127.0.0.1:3000", + "http://localhost:3000", + "http://[::1]:3000" + ]) { + const result = validateAuthPublicUrl(raw); + assert.equal(result.loopback, true); + assert.equal(result.cookieSecure, false); + assert.ok(result.publicBaseUrl.startsWith("http://")); + } +}); + +test("validateAuthPublicUrl allows HTTPS with secure cookies and path", () => { + const result = validateAuthPublicUrl("https://example.com/datafoundry"); + assert.equal(result.loopback, false); + assert.equal(result.cookieSecure, true); + assert.equal(result.publicBaseUrl, "https://example.com/datafoundry"); + assert.equal(result.cookiePath, "/datafoundry"); +}); + +test("validateAuthPublicUrl uses root cookie path for origin-only URLs", () => { + const result = validateAuthPublicUrl("https://example.com"); + assert.equal(result.cookiePath, "/"); + assert.equal(result.publicBaseUrl, "https://example.com"); +}); + +test("validateAuthPublicUrl rejects non-loopback HTTP", () => { + assert.throws( + () => validateAuthPublicUrl("http://192.168.1.10:3000"), + /AUTH_PUBLIC_BASE_URL|loopback|HTTPS/i + ); +}); + +test("validateAuthPublicUrl rejects illegal URL shapes", () => { + assert.throws(() => validateAuthPublicUrl("ftp://localhost:3000")); + assert.throws(() => validateAuthPublicUrl("http://user:pass@localhost:3000")); + assert.throws(() => validateAuthPublicUrl("http://localhost:3000#frag")); +}); + +test("password config matrix", () => { + const cases = [ + { + name: "loopback 127 with test mail", + env: baseEnv({ AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", AUTH_EMAIL_DELIVERY: "test" }), + ok: true, + cookieSecure: false + }, + { + name: "loopback localhost with test mail", + env: baseEnv({ AUTH_PUBLIC_BASE_URL: "http://localhost:3000", AUTH_EMAIL_DELIVERY: "test" }), + ok: true, + cookieSecure: false + }, + { + name: "loopback ipv6 with test mail", + env: baseEnv({ AUTH_PUBLIC_BASE_URL: "http://[::1]:3000", AUTH_EMAIL_DELIVERY: "test" }), + ok: true, + cookieSecure: false + }, + { + name: "lan http rejected", + env: baseEnv({ AUTH_PUBLIC_BASE_URL: "http://192.168.1.10:3000", AUTH_EMAIL_DELIVERY: "test" }), + ok: false + }, + { + name: "https smtp allowed", + env: baseEnv({ + AUTH_PUBLIC_BASE_URL: "https://example.com/datafoundry", + AUTH_EMAIL_DELIVERY: "smtp", + SMTP_HOST: "smtp.example.com", + SMTP_FROM: "noreply@example.com" + }), + ok: true, + cookieSecure: true, + cookiePath: "/datafoundry" + }, + { + name: "https test mail rejected", + env: baseEnv({ + AUTH_PUBLIC_BASE_URL: "https://example.com", + AUTH_EMAIL_DELIVERY: "test" + }), + ok: false + }, + { + name: "invalid registration mode rejected", + env: baseEnv({ AUTH_REGISTRATION_MODE: "maybe" }), + ok: false + }, + { + name: "missing registration mode rejected in password mode", + env: baseEnv({ AUTH_REGISTRATION_MODE: "" }), + ok: false + }, + { + name: "invalid email delivery rejected", + env: baseEnv({ AUTH_EMAIL_DELIVERY: "console" }), + ok: false + } + ]; + + for (const item of cases) { + if (item.ok) { + const config = loadPasswordAuthConfig(item.env); + assert.equal(config.cookieSecure, item.cookieSecure, item.name); + assert.equal(config.cookiePath, item.cookiePath ?? "/", item.name); + assert.ok(config.registrationMode === "open" || config.registrationMode === "closed", item.name); + } else { + assert.throws(() => loadPasswordAuthConfig(item.env), Error, item.name); + } + } +}); + +async function captureSetCookie(write) { + const server = createServer((req, res) => { + write(res); + res.end("ok"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + assert(address && typeof address === "object"); + try { + const response = await fetch(`http://127.0.0.1:${address.port}/`); + return response.headers.getSetCookie(); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } +} + +test("appendAuthCookies and clear use the same secure flag", async () => { + const secureCookies = await captureSetCookie((res) => { + appendAuthCookies(res, { + sessionToken: "session", + csrfToken: "csrf", + maxAgeSeconds: 60, + path: "/", + secure: true + }); + }); + assert.ok(secureCookies.every((cookie) => /;\s*Secure/i.test(cookie))); + + const clearSecure = await captureSetCookie((res) => { + appendClearAuthCookies(res, { path: "/", secure: true }); + }); + assert.ok(clearSecure.every((cookie) => /;\s*Secure/i.test(cookie))); + + const insecureCookies = await captureSetCookie((res) => { + appendAuthCookies(res, { + sessionToken: "session", + csrfToken: "csrf", + maxAgeSeconds: 60, + path: "/", + secure: false + }); + }); + assert.ok(insecureCookies.every((cookie) => !/;\s*Secure/i.test(cookie))); + + const clearInsecure = await captureSetCookie((res) => { + appendClearAuthCookies(res, { path: "/", secure: false }); + }); + assert.ok(clearInsecure.every((cookie) => !/;\s*Secure/i.test(cookie))); +}); + +test("cookie helpers derive Path from deployment prefix", async () => { + const cookies = await captureSetCookie((res) => { + appendAuthCookies(res, { + sessionToken: "session", + csrfToken: "csrf", + maxAgeSeconds: 60, + path: "/datafoundry", + secure: true + }); + }); + assert.ok(cookies.every((cookie) => /;\s*Path=\/datafoundry(?:;|$)/i.test(cookie))); + + const cleared = await captureSetCookie((res) => { + appendClearAuthCookies(res, { path: "/datafoundry", secure: true }); + }); + assert.ok(cleared.every((cookie) => /;\s*Path=\/datafoundry(?:;|$)/i.test(cookie))); +}); + +test("cookie helpers no longer read NODE_ENV for Secure", async () => { + const previous = process.env.NODE_ENV; + process.env.NODE_ENV = "production"; + try { + const cookies = await captureSetCookie((res) => { + appendAuthCookies(res, { + sessionToken: "session", + csrfToken: "csrf", + maxAgeSeconds: 60, + path: "/", + secure: false + }); + }); + assert.ok(cookies.every((cookie) => !/;\s*Secure/i.test(cookie))); + } finally { + if (previous === undefined) { + delete process.env.NODE_ENV; + } else { + process.env.NODE_ENV = previous; + } + } +}); + +async function withPasswordApi(envOverrides, run) { + const root = mkdtempSync(join(tmpdir(), "datafoundry-auth-foundation-")); + const previous = { ...process.env }; + Object.assign(process.env, { + DATAFOUNDRY_AUTH_MODE: "password", + AUTH_SESSION_SECRET: SECRET, + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + AUTH_EMAIL_DELIVERY: "test", + AUTH_REGISTRATION_MODE: "open", + EMBEDDING_API_KEY: "", + MASTRA_STORAGE_PATH: join(root, "mastra.sqlite"), + STORAGE_ROOT_DIR: root, + ...envOverrides + }); + + const metadataStore = createMetadataStore({ + database_path: join(root, "metadata.sqlite"), + secret_master_key: "auth-foundation-secret-master-key" + }); + const server = await createApiServer({ metadataStore }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + assert(address && typeof address === "object"); + const baseUrl = `http://127.0.0.1:${address.port}`; + + try { + await run({ baseUrl, metadataStore }); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + setImmediate(() => server.closeAllConnections?.()); + }); + for (const key of Object.keys(process.env)) { + if (!(key in previous)) { + delete process.env[key]; + } + } + Object.assign(process.env, previous); + } +} + +test("GET /api/v1/auth/status is public and returns safe fields only", async () => { + await withPasswordApi({ AUTH_REGISTRATION_MODE: "open" }, async ({ baseUrl }) => { + const response = await fetch(`${baseUrl}/api/v1/auth/status`); + assert.equal(response.status, 200); + const body = await response.json(); + assert.deepEqual(Object.keys(body.data).sort(), ["publicBaseUrl", "registrationEnabled"]); + assert.equal(body.data.publicBaseUrl, "http://127.0.0.1:3000"); + assert.equal(body.data.registrationEnabled, true); + const serialized = JSON.stringify(body); + assert.doesNotMatch(serialized, /AUTH_SESSION_SECRET|smtp|secret|8787|master/i); + }); +}); + +test("closed registration rejects register with REGISTRATION_CLOSED", async () => { + await withPasswordApi({ AUTH_REGISTRATION_MODE: "closed" }, async ({ baseUrl, metadataStore }) => { + const status = await fetch(`${baseUrl}/api/v1/auth/status`); + const statusBody = await status.json(); + assert.equal(statusBody.data.registrationEnabled, false); + + const email = `${randomUUID()}@example.test`; + const response = await fetch(`${baseUrl}/api/v1/auth/register`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + email, + password: "correct-horse", + displayName: "Closed User" + }) + }); + assert.equal(response.status, 403); + const body = await response.json(); + assert.equal(body.error.code, "REGISTRATION_CLOSED"); + assert.equal(metadataStore.users.findByEmail({ email }), undefined); + }); +}); + +test("open registration still allows register", async () => { + await withPasswordApi({ AUTH_REGISTRATION_MODE: "open" }, async ({ baseUrl }) => { + const email = `${randomUUID()}@example.test`; + const response = await fetch(`${baseUrl}/api/v1/auth/register`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + email, + password: "correct-horse", + displayName: "Open User" + }) + }); + assert.equal(response.status, 201, await response.clone().text()); + }); +}); + +async function registerVerify(baseUrl, { email, password, displayName }) { + const registered = await fetch(`${baseUrl}/api/v1/auth/register`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email, password, displayName }) + }); + assert.equal(registered.status, 201, await registered.clone().text()); + const body = await registered.json(); + const verified = await fetch(`${baseUrl}/api/v1/auth/verify-email`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ token: body.data.verificationToken }) + }); + assert.equal(verified.status, 200, await verified.clone().text()); + return body; +} + +test("CSRF_INVALID and recoverable csrf refresh", async () => { + await withPasswordApi({}, async ({ baseUrl }) => { + const email = `${randomUUID()}@example.test`; + const password = "correct-horse-battery"; + await registerVerify(baseUrl, { email, password, displayName: "Csrf User" }); + + const login = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email, password }) + }); + assert.equal(login.status, 200); + const setCookies = login.headers.getSetCookie(); + const cookieHeader = setCookies.map((item) => item.split(";", 1)[0]).join("; "); + const oldCsrf = setCookies + .find((item) => item.startsWith("df_csrf=")) + ?.split(";", 1)[0] + ?.slice("df_csrf=".length); + assert.ok(oldCsrf); + + const missing = await fetch(`${baseUrl}/api/v1/datasources`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Cookie: cookieHeader + }, + body: JSON.stringify({ + id: `ds-${randomUUID()}`, + name: "No CSRF", + type: "sqlite", + settings: { filePath: ":memory:" } + }) + }); + assert.equal(missing.status, 403); + assert.equal((await missing.json()).error.code, "CSRF_INVALID"); + + const anonymousRefresh = await fetch(`${baseUrl}/api/v1/auth/csrf/refresh`, { + method: "POST" + }); + assert.equal(anonymousRefresh.status, 401); + + const refresh = await fetch(`${baseUrl}/api/v1/auth/csrf/refresh`, { + method: "POST", + headers: { Cookie: cookieHeader } + }); + assert.equal(refresh.status, 200, await refresh.clone().text()); + assert.equal(refresh.headers.get("cache-control"), "no-store"); + const refreshBody = await refresh.json(); + const newCsrf = refreshBody.data.csrfToken; + assert.equal(typeof newCsrf, "string"); + assert.notEqual(newCsrf, decodeURIComponent(oldCsrf)); + const refreshedCookies = refresh.headers.getSetCookie(); + assert.ok(refreshedCookies.some((item) => item.startsWith("df_csrf="))); + const refreshedCookieHeader = [ + ...cookieHeader + .split("; ") + .filter((part) => !part.startsWith("df_csrf=")), + `df_csrf=${encodeURIComponent(newCsrf)}` + ].join("; "); + + const oldStillInvalid = await fetch(`${baseUrl}/api/v1/datasources`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Cookie: cookieHeader, + "X-CSRF-Token": decodeURIComponent(oldCsrf) + }, + body: JSON.stringify({ + id: `ds-${randomUUID()}`, + name: "Old CSRF", + type: "sqlite", + settings: { filePath: ":memory:" } + }) + }); + assert.equal(oldStillInvalid.status, 403); + assert.equal((await oldStillInvalid.json()).error.code, "CSRF_INVALID"); + + const withNew = await fetch(`${baseUrl}/api/v1/datasources`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Cookie: refreshedCookieHeader, + "X-CSRF-Token": newCsrf + }, + body: JSON.stringify({ + id: `ds-${randomUUID()}`, + name: "New CSRF", + type: "sqlite", + settings: { filePath: ":memory:" } + }) + }); + assert.equal(withNew.status, 201, await withNew.clone().text()); + }); +}); + +test("login anti-enumeration and client session lifetimes", async () => { + await withPasswordApi({}, async ({ baseUrl, metadataStore }) => { + const password = "correct-horse-battery"; + const verifiedEmail = `${randomUUID()}@example.test`; + const unverifiedEmail = `${randomUUID()}@example.test`; + + await registerVerify(baseUrl, { + email: verifiedEmail, + password, + displayName: "Verified" + }); + + const unverified = await fetch(`${baseUrl}/api/v1/auth/register`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + email: unverifiedEmail, + password, + displayName: "Unverified" + }) + }); + assert.equal(unverified.status, 201); + + const missing = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email: `${randomUUID()}@example.test`, password: "whatever-password" }) + }); + assert.equal(missing.status, 401); + const missingBody = await missing.json(); + assert.equal(missingBody.error.message, "Invalid email or password."); + + const wrongPassword = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email: verifiedEmail, password: "wrong-password" }) + }); + assert.equal(wrongPassword.status, 401); + const wrongBody = await wrongPassword.json(); + assert.equal(wrongBody.error.message, "Invalid email or password."); + + const unverifiedWrong = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email: unverifiedEmail, password: "wrong-password" }) + }); + assert.equal(unverifiedWrong.status, 401); + assert.equal((await unverifiedWrong.json()).error.message, "Invalid email or password."); + + const unverifiedCorrect = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email: unverifiedEmail, password }) + }); + assert.equal(unverifiedCorrect.status, 403); + assert.equal((await unverifiedCorrect.json()).error.code, "EMAIL_NOT_VERIFIED"); + + const illegalClient = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email: verifiedEmail, password, client: "mobile" }) + }); + assert.equal(illegalClient.status, 400); + + const webLogin = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email: verifiedEmail, password, client: "web" }) + }); + assert.equal(webLogin.status, 200, await webLogin.clone().text()); + const webBody = await webLogin.json(); + assert.equal(typeof webBody.data.session.expiresAt, "string"); + const webExpires = Date.parse(webBody.data.session.expiresAt); + const webDelta = webExpires - Date.now(); + assert.ok(Math.abs(webDelta - 30 * 24 * 60 * 60 * 1000) < 120_000); + + const webCookie = webLogin.headers.getSetCookie().find((item) => item.startsWith("df_session=")); + assert.match(webCookie ?? "", /Max-Age=2592000/i); + + const tuiLogin = await fetch(`${baseUrl}/api/v1/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email: verifiedEmail, password, client: "tui" }) + }); + assert.equal(tuiLogin.status, 200, await tuiLogin.clone().text()); + const tuiBody = await tuiLogin.json(); + const tuiExpires = Date.parse(tuiBody.data.session.expiresAt); + const tuiDelta = tuiExpires - Date.now(); + assert.ok(Math.abs(tuiDelta - 7 * 24 * 60 * 60 * 1000) < 120_000); + const tuiCookie = tuiLogin.headers.getSetCookie().find((item) => item.startsWith("df_session=")); + assert.match(tuiCookie ?? "", /Max-Age=604800/i); + + const sessions = metadataStore.authSessions.listByUser + ? metadataStore.authSessions.listByUser({ user_id: webBody.data.user.id }) + : undefined; + if (Array.isArray(sessions) && sessions.length > 0) { + const matching = sessions.find((session) => session.expires_at === tuiBody.data.session.expiresAt) + ?? sessions.find((session) => session.expires_at === webBody.data.session.expiresAt); + assert.ok(matching, "expected session expires_at to match login response"); + } + }); +}); + +test("scripts/**/*.mjs HTTP auth classification gate", () => { + const classified = new Set([ + ...FORMAL_HTTP_AUTH_TARGETS, + ...PUBLIC_HTTP_TARGETS, + ...DIRECT_METADATA_FIXTURE_TARGETS, + ...AUTH_FOUNDATION_HARNESS_TARGETS + ]); + const httpScripts = listMjsFiles(SCRIPTS_ROOT) + .map((fullPath) => ({ + fullPath, + relativePath: relative(SCRIPTS_ROOT, fullPath).replaceAll("\\", "/") + })) + .filter(({ fullPath }) => isHttpScript(readFileSync(fullPath, "utf8"))); + + const unclassified = httpScripts + .map((item) => item.relativePath) + .filter((path) => !classified.has(path)) + .sort(); + assert.deepEqual( + unclassified, + [], + `Unclassified HTTP scripts must be listed in FORMAL_HTTP_AUTH_TARGETS, PUBLIC_HTTP_TARGETS, DIRECT_METADATA_FIXTURE_TARGETS, or AUTH_FOUNDATION_HARNESS_TARGETS:\n${unclassified.join("\n")}` + ); + + const formalViolations = []; + for (const relativePath of FORMAL_HTTP_AUTH_TARGETS) { + const fullPath = join(SCRIPTS_ROOT, relativePath); + const source = readFileSync(fullPath, "utf8"); + for (const pattern of FORBIDDEN_DEV_AUTH_PATTERNS) { + if (pattern.test(source)) { + formalViolations.push(`${relativePath} matches ${pattern}`); + } + } + if (!source.includes("authenticated-test-client")) { + formalViolations.push( + `${relativePath} must import scripts/lib/authenticated-test-client (shared password-auth client)` + ); + } + const syntaxCheck = spawnSync(process.execPath, ["--check", fullPath], { + encoding: "utf8" + }); + if (syntaxCheck.status !== 0) { + const detail = [syntaxCheck.stderr, syntaxCheck.stdout].filter(Boolean).join("\n").trim(); + formalViolations.push( + `${relativePath} failed node --check${detail ? `:\n${detail}` : ""}` + ); + } + } + assert.deepEqual( + formalViolations, + [], + `FORMAL_HTTP_AUTH_TARGETS must not use development auth bypasses, must use the shared client, and must pass node --check:\n${formalViolations.join("\n")}` + ); +}); diff --git a/scripts/datalink-stack-config.mjs b/scripts/datalink-stack-config.mjs new file mode 100644 index 00000000..c3c12374 --- /dev/null +++ b/scripts/datalink-stack-config.mjs @@ -0,0 +1,28 @@ +import { resolve } from "node:path"; + +export function datalinkEnabled(env = process.env) { + return ["1", "true", "yes", "on"].includes((env.DATALINK_ENABLED ?? "").trim().toLowerCase()); +} + +export function resolveDatalinkEnv(root, env = process.env) { + return { + ...env, + DATALINK_CONFIG_PATH: + env.DATALINK_CONFIG_PATH?.trim() || resolve(root, "services/datalink/datalink_config.json"), + DATALINK_GRAPH_DB_PATH: + env.DATALINK_GRAPH_DB_PATH?.trim() || resolve(root, "storage/datalink/datalink.db"), + DATALINK_API_HOST: env.DATALINK_API_HOST?.trim() || "127.0.0.1", + DATALINK_API_PORT: String(readPort(env.DATALINK_API_PORT, 8081, "DATALINK_API_PORT")), + DATALINK_MCP_HOST: env.DATALINK_MCP_HOST?.trim() || "127.0.0.1", + DATALINK_MCP_PORT: String(readPort(env.DATALINK_MCP_PORT, 8080, "DATALINK_MCP_PORT")), + }; +} + +function readPort(value, fallback, name) { + if (!value?.trim()) return fallback; + const port = Number(value); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error(`${name} must be an integer between 1 and 65535.`); + } + return port; +} diff --git a/scripts/datalink-stack-config.test.mjs b/scripts/datalink-stack-config.test.mjs new file mode 100644 index 00000000..bae5026e --- /dev/null +++ b/scripts/datalink-stack-config.test.mjs @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { datalinkEnabled, resolveDatalinkEnv } from "./datalink-stack-config.mjs"; + +test("DataLink is opt-in", () => { + assert.equal(datalinkEnabled({}), false); + assert.equal(datalinkEnabled({ DATALINK_ENABLED: "true" }), true); + assert.equal(datalinkEnabled({ DATALINK_ENABLED: "1" }), true); + assert.equal(datalinkEnabled({ DATALINK_ENABLED: "false" }), false); +}); + +test("DataLink paths and ports have repository-owned defaults", () => { + const env = resolveDatalinkEnv("/repo", {}); + assert.equal(env.DATALINK_CONFIG_PATH, "/repo/services/datalink/datalink_config.json"); + assert.equal(env.DATALINK_GRAPH_DB_PATH, "/repo/storage/datalink/datalink.db"); + assert.equal(env.DATALINK_API_PORT, "8081"); + assert.equal(env.DATALINK_MCP_PORT, "8080"); +}); + +test("invalid DataLink ports fail before processes start", () => { + assert.throws( + () => resolveDatalinkEnv("/repo", { DATALINK_API_PORT: "70000" }), + /DATALINK_API_PORT/, + ); +}); diff --git a/scripts/deploy/args.mjs b/scripts/deploy/args.mjs new file mode 100644 index 00000000..93ebf48e --- /dev/null +++ b/scripts/deploy/args.mjs @@ -0,0 +1,87 @@ +const COMMANDS = new Set([ + "deploy", + "start", + "stop", + "restart", + "status", + "logs", + "doctor", + "tui", + "help" +]); + +export function parseDeployArgs(argv = []) { + const tokens = [...argv]; + let command = "deploy"; + let reconfigure = false; + let nonInteractive = false; + let runtimeUrl = null; + let sawCommand = false; + + for (let i = 0; i < tokens.length; i += 1) { + const token = tokens[i]; + if (token === "--reconfigure") { + reconfigure = true; + continue; + } + if (token === "--non-interactive") { + nonInteractive = true; + continue; + } + if (token === "--runtime-url") { + const value = tokens[i + 1]; + if (!value || value.startsWith("-")) { + throw new Error("--runtime-url requires a URL value"); + } + runtimeUrl = value; + i += 1; + continue; + } + if (token.startsWith("-")) { + throw new Error(`Unknown flag: ${token}`); + } + if (!COMMANDS.has(token)) { + throw new Error(`Unknown command: ${token}`); + } + if (sawCommand) { + throw new Error(`Unexpected extra command: ${token}`); + } + command = token; + sawCommand = true; + } + + if (reconfigure && nonInteractive) { + throw new Error("--reconfigure and --non-interactive are mutually exclusive"); + } + if (reconfigure && command !== "deploy") { + throw new Error("--reconfigure is only valid with deploy"); + } + if (nonInteractive && command !== "deploy") { + throw new Error("--non-interactive is only valid with deploy"); + } + if (runtimeUrl != null && command !== "tui") { + throw new Error("--runtime-url is only valid with tui"); + } + + return { command, reconfigure, nonInteractive, runtimeUrl }; +} + +export function deploymentHelp() { + return `./deploy.sh [deploy] [--reconfigure | --non-interactive] +./deploy.sh start|stop|restart|status|logs|doctor|tui|help +./deploy.sh tui [--runtime-url ] + +deploy Configure, install, build (including TUI), start, and verify. Existing deployments use a maintenance window. +start Start an existing build. Does not install or build. +stop Stop only the managed DataFoundry process group. Keeps configuration and data. +restart Stop and start an existing build. Does not install or build. +status Read process state and probe actual service health. +logs Show recent runtime logs and continue following them. Ctrl+C does not stop DataFoundry. +doctor Run read-only dependency, configuration, port, permission, disk, and health checks. +tui Optional: start the TUI client in the foreground after deploy (not a managed background service). +help Show this help. + +One-click deploy builds the TUI so it is ready. Start it in another terminal with ./deploy.sh tui +(or npm run start:tui). It does not stay running with the Web/API stack. +`; +} diff --git a/scripts/deploy/args.test.mjs b/scripts/deploy/args.test.mjs new file mode 100644 index 00000000..bce3c4e7 --- /dev/null +++ b/scripts/deploy/args.test.mjs @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { deploymentHelp, parseDeployArgs } from "./args.mjs"; + +test("defaults to deploy", () => { + assert.deepEqual(parseDeployArgs([]), { + command: "deploy", + reconfigure: false, + nonInteractive: false, + runtimeUrl: null + }); +}); + +test("accepts flags before or after deploy", () => { + assert.deepEqual(parseDeployArgs(["--non-interactive"]), { + command: "deploy", + reconfigure: false, + nonInteractive: true, + runtimeUrl: null + }); + assert.deepEqual(parseDeployArgs(["deploy", "--reconfigure"]), { + command: "deploy", + reconfigure: true, + nonInteractive: false, + runtimeUrl: null + }); + assert.deepEqual(parseDeployArgs(["--reconfigure", "deploy"]), { + command: "deploy", + reconfigure: true, + nonInteractive: false, + runtimeUrl: null + }); +}); + +test("supports every lifecycle command", () => { + for (const command of ["start", "stop", "restart", "status", "logs", "doctor", "tui", "help"]) { + assert.equal(parseDeployArgs([command]).command, command); + } +}); + +test("tui accepts optional --runtime-url", () => { + assert.deepEqual(parseDeployArgs(["tui"]), { + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: null + }); + assert.deepEqual(parseDeployArgs(["tui", "--runtime-url", "http://127.0.0.1:9000/api/copilotkit"]), { + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: "http://127.0.0.1:9000/api/copilotkit" + }); + assert.deepEqual(parseDeployArgs(["--runtime-url", "http://example/api/copilotkit", "tui"]), { + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: "http://example/api/copilotkit" + }); +}); + +test("rejects unknown commands and invalid flag combinations", () => { + assert.throws(() => parseDeployArgs(["start", "--reconfigure"]), /--reconfigure is only valid with deploy/); + assert.throws(() => parseDeployArgs(["status", "--non-interactive"]), /--non-interactive is only valid with deploy/); + assert.throws( + () => parseDeployArgs(["deploy", "--reconfigure", "--non-interactive"]), + /mutually exclusive/ + ); + assert.throws(() => parseDeployArgs(["deploy", "--runtime-url", "http://x"]), /--runtime-url is only valid with tui/); + assert.throws(() => parseDeployArgs(["tui", "--runtime-url"]), /--runtime-url requires a URL value/); + assert.throws(() => parseDeployArgs(["wat"]), /Unknown command/); +}); + +test("help lists lifecycle commands including tui and build readiness note", () => { + const help = deploymentHelp(); + for (const command of ["deploy", "start", "stop", "restart", "status", "logs", "doctor", "tui", "help"]) { + assert.match(help, new RegExp(`^${command}\\b`, "m")); + } + assert.match(help, /builds the TUI|build \(including TUI\)/i); + assert.match(help, /\.\/deploy\.sh tui/); + assert.match(help, /does not stay running/i); +}); diff --git a/scripts/deploy/bootstrap.sh b/scripts/deploy/bootstrap.sh new file mode 100755 index 00000000..d277cb99 --- /dev/null +++ b/scripts/deploy/bootstrap.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Shared bootstrap helpers for native deploy (sourced by ./deploy.sh). + +check_supported_system() { + local os_release="${DATAFOUNDRY_OS_RELEASE_FILE:-/etc/os-release}" + if [[ ! -r "${os_release}" ]]; then + echo "Unsupported operating system: ${os_release} is missing. DataFoundry native deploy supports Ubuntu/Debian only." >&2 + exit 1 + fi + # shellcheck disable=SC1090,SC1091 + source "${os_release}" + case "${ID:-}" in + ubuntu|debian) ;; + *) + echo "Unsupported operating system: ${ID:-unknown}. DataFoundry native deploy supports Ubuntu/Debian only." >&2 + exit 1 + ;; + esac + + local arch + arch="${DATAFOUNDRY_UNAME_M:-$(uname -m)}" + case "${arch}" in + x86_64|amd64|aarch64|arm64) ;; + *) + echo "Unsupported architecture: ${arch}. DataFoundry native deploy supports x86_64/amd64 and aarch64/arm64 only." >&2 + exit 1 + ;; + esac +} + +command_is_readonly() { + local token + for token in "$@"; do + case "${token}" in + status|logs|stop|doctor|help) return 0 ;; + esac + done + return 1 +} + +has_non_interactive_flag() { + local token + for token in "$@"; do + [[ "${token}" == "--non-interactive" ]] && return 0 + done + return 1 +} + +node_major_version() { + local version + version="$(node --version 2>/dev/null || true)" + if [[ "${version}" =~ ^v([0-9]+) ]]; then + echo "${BASH_REMATCH[1]}" + return 0 + fi + return 1 +} + +can_install_noninteractive() { + if [[ "$(id -u)" -eq 0 ]]; then + return 0 + fi + if sudo -n true >/dev/null 2>&1; then + return 0 + fi + return 1 +} + +install_node_22() { + local setup + local nodesource_url="https://deb.nodesource.com/setup_22.x" + echo "Node.js 22 is required." + echo "Installer source: ${nodesource_url}" + echo "Commands:" + echo " curl -fsSL ${nodesource_url} -o /tmp/nodesource_setup.sh" + echo " bash /tmp/nodesource_setup.sh" + echo " apt-get install -y nodejs" + + if has_non_interactive_flag "$@"; then + if ! can_install_noninteractive; then + echo "Non-interactive Node.js installation requires root or passwordless sudo." >&2 + exit 1 + fi + else + local answer + read -r -p "Install Node.js 22 from NodeSource now? [y/N]: " answer + case "${answer}" in + y|Y|yes|YES) ;; + *) + echo "Node.js 22 is required. Install it and re-run ./deploy.sh." >&2 + exit 1 + ;; + esac + fi + + setup="$(mktemp)" + trap 'rm -f "${setup}"' RETURN + curl -fsSL "${nodesource_url}" -o "${setup}" + if [[ "$(id -u)" -eq 0 ]]; then + bash "${setup}" + apt-get install -y nodejs + elif has_non_interactive_flag "$@"; then + sudo -n bash "${setup}" + sudo -n apt-get install -y nodejs + else + sudo bash "${setup}" + sudo apt-get install -y nodejs + fi + rm -f "${setup}" + trap - RETURN + + if ! node --version >/dev/null 2>&1 || ! npm --version >/dev/null 2>&1; then + echo "Node.js 22 installation did not produce working node/npm commands." >&2 + exit 1 + fi + local major + major="$(node_major_version || true)" + if [[ -z "${major}" || "${major}" -lt 22 ]]; then + echo "Node.js 22+ is required after installation; found $(node --version 2>/dev/null || echo none)." >&2 + exit 1 + fi +} + +ensure_node_22() { + local major="" + if command -v node >/dev/null 2>&1; then + major="$(node_major_version || true)" + fi + + if [[ -n "${major}" && "${major}" -ge 22 ]]; then + return 0 + fi + + if command_is_readonly "$@"; then + echo "Node.js 22+ is required for this command. Install Node.js 22 and re-run ./deploy.sh $*." >&2 + exit 1 + fi + + if [[ -n "${major}" && "${major}" -lt 22 ]]; then + echo "Unsupported Node.js version: $(node --version). DataFoundry requires Node.js 22.x." >&2 + fi + + install_node_22 "$@" +} diff --git a/scripts/deploy/bootstrap.test.mjs b/scripts/deploy/bootstrap.test.mjs new file mode 100644 index 00000000..21ede8e7 --- /dev/null +++ b/scripts/deploy/bootstrap.test.mjs @@ -0,0 +1,163 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmod, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const DEPLOY_SH = path.join(ROOT, "deploy.sh"); +const INSTALL_SH = path.join(ROOT, "scripts/deploy/install-dependency.sh"); + +function runBash(args, options = {}) { + return spawnSync("bash", args, { + cwd: ROOT, + encoding: "utf8", + env: { ...process.env, ...(options.env ?? {}) }, + input: options.input, + timeout: options.timeout ?? 10_000 + }); +} + +async function makeFakePath(binaries) { + const dir = await mkdtemp(path.join(os.tmpdir(), "datafoundry-path-")); + for (const [name, body] of Object.entries(binaries)) { + const filePath = path.join(dir, name); + await writeFile(filePath, body, { mode: 0o755 }); + await chmod(filePath, 0o755); + } + return dir; +} + +test("./deploy.sh help delegates without installing when Node 22+ exists", () => { + const result = runBash([DEPLOY_SH, "help"]); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /deploy\s+Configure/); + assert.match(result.stdout, /\bstart\b/); + assert.match(result.stdout, /\bstop\b/); + assert.match(result.stdout, /\brestart\b/); + assert.match(result.stdout, /\bstatus\b/); + assert.match(result.stdout, /\blogs\b/); + assert.match(result.stdout, /\bdoctor\b/); + assert.match(result.stdout, /\bhelp\b/); + assert.doesNotMatch(result.stdout + result.stderr, /Install Node\.js|nodesource|apt-get install/i); +}); + +test("unsupported OS exits 1 with a precise message", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "datafoundry-os-")); + const osRelease = path.join(dir, "os-release"); + await writeFile(osRelease, 'ID=fedora\nVERSION_ID="40"\n'); + const result = runBash([DEPLOY_SH, "help"], { + env: { DATAFOUNDRY_OS_RELEASE_FILE: osRelease } + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Unsupported operating system: fedora/); +}); + +test("unsupported architecture exits 1 with a precise message", async () => { + const result = runBash([DEPLOY_SH, "help"], { + env: { DATAFOUNDRY_UNAME_M: "ppc64le" } + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Unsupported architecture: ppc64le/); +}); + +test("Node 20 is rejected before CLI starts", async () => { + const fakePath = await makeFakePath({ + node: "#!/bin/bash\necho v20.11.0\n", + npm: "#!/bin/bash\necho 10.0.0\n", + curl: "#!/bin/bash\necho curl-should-not-run >&2\nexit 99\n", + sudo: "#!/bin/bash\nexit 1\n", + id: "#!/bin/bash\necho 1000\n" + }); + const script = ` +set -euo pipefail +export PATH="${fakePath}:/bin" +hash -r +command -v node +node --version +source "${DEPLOY_SH}" +ensure_node_22 deploy --non-interactive +echo SHOULD_NOT_REACH +`; + const result = runBash(["-c", script], { + env: { + PATH: `${fakePath}:/bin`, + HOME: os.tmpdir() + }, + input: "" + }); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + assert.notEqual(result.status, 0, output); + assert.match(output, /Unsupported Node\.js version|Node\.js 22|passwordless sudo|requires root/i); + assert.doesNotMatch(output, /SHOULD_NOT_REACH/); + assert.doesNotMatch(output, /curl-should-not-run/); +}); + +test("interactive Node installation prints repository and command, then asks once", async () => { + const fakePath = await makeFakePath({ + curl: "#!/bin/bash\nexit 99\n", + sudo: "#!/bin/bash\nexit 1\n", + id: "#!/bin/bash\necho 1000\n" + }); + const result = runBash( + ["-c", `source "${DEPLOY_SH}"; install_node_22 deploy`], + { + env: { + PATH: `${fakePath}:/bin:/usr/bin`, + HOME: os.tmpdir() + }, + input: "n\n" + } + ); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + assert.notEqual(result.status, 0, output); + assert.match(output, /deb\.nodesource\.com\/setup_22\.x/); + assert.match(output, /Node\.js 22 is required/); + assert.match(output, /apt-get install -y nodejs/); + assert.match(output, /re-run \.\/deploy\.sh/); +}); + +test("--non-interactive never reads stdin and fails without root/passwordless sudo", async () => { + const fakePath = await makeFakePath({ + id: "#!/usr/bin/env bash\necho 1000\n", + sudo: "#!/usr/bin/env bash\nexit 1\n", + curl: "#!/usr/bin/env bash\necho unexpectedly-called >&2; exit 99\n" + }); + const result = runBash( + ["-c", `source "${DEPLOY_SH}"; install_node_22 deploy --non-interactive; echo SHOULD_NOT_REACH`], + { + env: { + PATH: `${fakePath}:/bin:/usr/bin`, + HOME: os.tmpdir() + }, + input: "y\ny\ny\n" + } + ); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + assert.equal(result.status, 1, output); + assert.match(output, /root or passwordless sudo/i); + assert.doesNotMatch(output, /unexpectedly-called|SHOULD_NOT_REACH/); +}); + +test("installer accepts only node, python, or uv", () => { + const bad = runBash([INSTALL_SH, "ruby"]); + assert.equal(bad.status, 2); + assert.match(bad.stderr, /Usage: install-dependency\.sh /); +}); + +test("no code path executes curl | bash", async () => { + const bootstrap = path.join(ROOT, "scripts/deploy/bootstrap.sh"); + const sources = [ + await readFile(DEPLOY_SH, "utf8"), + await readFile(bootstrap, "utf8"), + await readFile(INSTALL_SH, "utf8") + ]; + for (const source of sources) { + assert.doesNotMatch(source, /curl[^\n]*\|\s*(bash|sh)/); + } + // Node bootstrap and dependency installer download to a temp file first. + assert.match(await readFile(bootstrap, "utf8"), /mktemp/); + assert.match(await readFile(INSTALL_SH, "utf8"), /mktemp/); +}); diff --git a/scripts/deploy/cli.mjs b/scripts/deploy/cli.mjs new file mode 100644 index 00000000..d60a2098 --- /dev/null +++ b/scripts/deploy/cli.mjs @@ -0,0 +1,803 @@ +import { spawn } from "node:child_process"; +import { constants, existsSync } from "node:fs"; +import { access, appendFile, copyFile, mkdir, readFile, statfs, writeFile } from "node:fs/promises"; +import { createInterface } from "node:readline"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { deploymentHelp, parseDeployArgs } from "./args.mjs"; +import { + ensureDeploymentEnvironment, + isCompleteDeploymentConfig, + parseDeploymentEnvironment, + redactSensitiveText, + renderWebEnvironment, + updateDeploymentEnvironment, + writeDeploymentConfiguration +} from "./config.mjs"; +import { createDeploymentController } from "./controller.mjs"; +import { ensureDependencies, inspectDependencies } from "./dependencies.mjs"; +import { collectDeploymentHealth, probeHttp, waitForDeployment } from "./health.mjs"; +import { probePort, selectDeploymentPort, verifySelectedPorts } from "./ports.mjs"; +import { + deploymentPaths, + isProcessAlive, + readDeploymentState, + startManagedStack, + stopManagedStack, + verifyManagedProcessForStop, + writeDeploymentState +} from "./process-state.mjs"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const TUI_ENTRY = "apps/tui/dist/index.js"; + +export function resolveTuiRuntimeUrl(env = {}, runtimeUrl = null) { + const explicit = String(runtimeUrl ?? "").trim(); + if (explicit) return explicit; + const port = Number(env.API_PORT || 8787); + return `http://127.0.0.1:${port}/api/copilotkit`; +} + +const DATALINK_INTRO = `可选能力：DataLink 语义服务 + +DataLink 会根据表结构和数据画像建立语义关系图，帮助 Agent： +- 理解表和字段的业务含义 +- 发现可信的表关联与 JOIN 路径 +- 减少选错表、猜错字段和盲目 JOIN +- 在 Web 中查看图谱、探索关系并管理已接入的表 + +1. 不启用（默认） +2. 启用 DataLink`; + +export async function inspectWritablePath(targetPath) { + let candidate = path.resolve(targetPath); + while (true) { + try { + await access(candidate, constants.W_OK); + return { writable: true, checkedPath: candidate, exists: existsSync(targetPath) }; + } catch (error) { + const parent = path.dirname(candidate); + if (parent === candidate) { + return { + writable: false, + checkedPath: candidate, + exists: existsSync(targetPath), + error: error?.message ?? String(error) + }; + } + candidate = parent; + } + } +} + +function overlayProcessEnv(text, processEnv = {}) { + const updates = {}; + for (const key of [ + "WEB_HOST", + "WEB_PORT", + "API_HOST", + "API_PORT", + "DATALINK_ENABLED", + "DATALINK_MCP_PORT", + "DATALINK_API_PORT", + "AUTH_PUBLIC_BASE_URL", + "AUTH_SESSION_SECRET", + "SECRET_MASTER_KEY" + ]) { + if (processEnv[key] != null && String(processEnv[key]).trim() !== "") { + updates[key] = String(processEnv[key]); + } + } + return Object.keys(updates).length > 0 ? updateDeploymentEnvironment(text, updates) : text; +} + +export function collectManagedPorts(env = {}, state = null) { + const ports = new Set(); + for (const key of ["WEB_PORT", "API_PORT", "DATALINK_MCP_PORT", "DATALINK_API_PORT"]) { + const port = Number(env[key]); + if (Number.isInteger(port) && port > 0) ports.add(port); + } + if (state?.ports && typeof state.ports === "object") { + for (const value of Object.values(state.ports)) { + const port = Number(value); + if (Number.isInteger(port) && port > 0) ports.add(port); + } + } + return ports; +} + +export function resolveAuthPublicBaseUrl(existing, oldWebPort, newWebPort) { + const url = String(existing ?? "").trim(); + if (!url) return `http://127.0.0.1:${newWebPort}`; + + try { + const parsed = new URL(url); + const isLoopback = /^(127\.0\.0\.1|localhost)$/i.test(parsed.hostname); + const explicitPort = parsed.port; + + if (isLoopback && url.includes(`:${oldWebPort}`)) { + return url.replace(`:${oldWebPort}`, `:${newWebPort}`); + } + if (!explicitPort && !isLoopback) { + return url; + } + if (isLoopback) { + return `http://127.0.0.1:${newWebPort}`; + } + return url; + } catch { + return `http://127.0.0.1:${newWebPort}`; + } +} + +function formatDeployLogTimestamp(date = new Date()) { + return date.toISOString().replace(/[-:TZ.]/g, "").slice(0, 14); +} + +export async function createDeployLogWriter(root, options = {}) { + const logsDir = path.join(root, "storage/logs"); + await mkdir(logsDir, { recursive: true }); + const stamp = options.timestamp ?? formatDeployLogTimestamp(); + const logFile = path.join(logsDir, `deploy-${stamp}.log`); + const latestFile = path.join(logsDir, "deploy-latest.log"); + const relativeLogPath = path.join("storage/logs", `deploy-${stamp}.log`); + + await writeFile(logFile, `[${new Date().toISOString()}] deploy started\n`, { + encoding: "utf8", + mode: 0o600 + }); + + let pendingLine = ""; + + async function flushBufferedLines(text, { finalize = false } = {}) { + pendingLine += String(text ?? ""); + let ready = ""; + + while (true) { + const newlineIndex = pendingLine.indexOf("\n"); + if (newlineIndex === -1) break; + ready += pendingLine.slice(0, newlineIndex + 1); + pendingLine = pendingLine.slice(newlineIndex + 1); + } + + if (finalize && pendingLine.length > 0) { + ready += pendingLine; + pendingLine = ""; + } + + if (ready.length > 0) { + await appendFile(logFile, redactSensitiveText(ready), { encoding: "utf8", mode: 0o600 }); + } + } + + return { + logPath: relativeLogPath, + async append(text) { + await flushBufferedLines(text); + }, + async finalize() { + await flushBufferedLines("", { finalize: true }); + await copyFile(logFile, latestFile); + } + }; +} + +const MAX_PORT_MENU_INVALID_CHOICES = 5; + +async function selectPortWithHint(options) { + return selectDeploymentPort({ + ...options, + ask: async (prompt) => { + if (!prompt.includes("请选择")) { + return options.ask(prompt); + } + + for (let attempt = 0; attempt < MAX_PORT_MENU_INVALID_CHOICES; attempt += 1) { + const answer = await options.ask(prompt); + if (!answer || ["1", "2", ""].includes(String(answer).trim())) { + return answer; + } + options.print("请输入 1 或 2"); + } + + throw new Error("port selection cancelled: too many invalid choices"); + } + }); +} + +export async function configureDeploymentInteractively(options) { + const { + root, + sourceText, + reconfigure, + nonInteractive, + ask, + print, + probe, + processEnv = {}, + write = false, + timestamp + } = options; + + let text = overlayProcessEnv(sourceText ?? "", processEnv); + const originalEnv = parseDeploymentEnvironment(text); + const completeBeforeFill = isCompleteDeploymentConfig(originalEnv); + let ensured = ensureDeploymentEnvironment(text); + text = ensured.text; + let env = ensured.env; + const oldWebPort = env.WEB_PORT; + + if (!reconfigure && !nonInteractive && completeBeforeFill && sourceText?.trim()) { + return { env, envText: text, webText: renderWebEnvironment(env), wrote: false }; + } + + const deploymentState = root ? await readDeploymentState(root).catch(() => null) : null; + const managedPorts = collectManagedPorts(env, deploymentState); + + if (!nonInteractive) { + print(DATALINK_INTRO); + const choice = String((await ask("请选择 [1]：")) ?? "").trim() || "1"; + env.DATALINK_ENABLED = choice === "2" ? "true" : "false"; + text = updateDeploymentEnvironment(text, { DATALINK_ENABLED: env.DATALINK_ENABLED }); + } + + const reserved = new Set(); + const webPort = await selectPortWithHint({ + label: "Web", + defaultPort: Number(env.WEB_PORT || 3000), + reserved, + managedPorts, + nonInteractive, + ask, + print, + probe + }); + reserved.add(webPort); + const apiPort = await selectPortWithHint({ + label: "API", + defaultPort: Number(env.API_PORT || 8787), + reserved, + managedPorts, + nonInteractive, + ask, + print, + probe + }); + reserved.add(apiPort); + + const updates = { + WEB_PORT: String(webPort), + API_PORT: String(apiPort), + AUTH_PUBLIC_BASE_URL: resolveAuthPublicBaseUrl(env.AUTH_PUBLIC_BASE_URL, oldWebPort, webPort) + }; + + if (env.DATALINK_ENABLED === "true") { + const mcpPort = await selectPortWithHint({ + label: "DataLink MCP", + defaultPort: Number(env.DATALINK_MCP_PORT || 8080), + reserved, + managedPorts, + nonInteractive, + ask, + print, + probe + }); + reserved.add(mcpPort); + const restPort = await selectPortWithHint({ + label: "DataLink REST", + defaultPort: Number(env.DATALINK_API_PORT || 8081), + reserved, + managedPorts, + nonInteractive, + ask, + print, + probe + }); + updates.DATALINK_MCP_PORT = String(mcpPort); + updates.DATALINK_API_PORT = String(restPort); + } + + if (!nonInteractive) { + const defaultUrl = updates.AUTH_PUBLIC_BASE_URL; + const answer = String((await ask(`浏览器公开访问地址 [${defaultUrl}]：`)) ?? "").trim(); + if (answer) { + let url; + try { + url = new URL(answer); + } catch { + throw new Error("AUTH_PUBLIC_BASE_URL must be a valid URL"); + } + const urlPort = url.port || (url.protocol === "https:" ? "443" : "80"); + if (String(urlPort) !== String(webPort)) { + const confirm = String( + (await ask(`公开地址端口 ${urlPort} 与 Web 端口 ${webPort} 不一致。确认为反向代理？[y/N]：`)) ?? "" + ) + .trim() + .toLowerCase(); + if (confirm !== "y" && confirm !== "yes") { + throw new Error("Public URL port must match the selected Web port"); + } + } + updates.AUTH_PUBLIC_BASE_URL = answer; + } + } + + text = updateDeploymentEnvironment(text, updates); + ensured = ensureDeploymentEnvironment(text); + text = ensured.text; + env = ensured.env; + + if ( + (env.WEB_HOST === "0.0.0.0" || env.WEB_HOST === "::") && + /^https?:\/\/(127\.0\.0\.1|localhost)(:|\/|$)/i.test(env.AUTH_PUBLIC_BASE_URL) + ) { + print("警告：WEB_HOST 对外绑定，但 AUTH_PUBLIC_BASE_URL 仅适合本机访问。远程访问请设置正确的公开地址。"); + } + + const webText = renderWebEnvironment(env); + let wrote = false; + if (write) { + await writeDeploymentConfiguration(root, text, webText, { + backup: Boolean(reconfigure), + timestamp + }); + wrote = true; + } + + return { env, envText: text, webText, wrote }; +} + +function createReadlineAsk() { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + const ask = (prompt) => + new Promise((resolve) => { + rl.question(prompt, (answer) => resolve(answer)); + }); + ask.close = () => rl.close(); + return ask; +} + +async function loadRootEnvText(root) { + try { + return await readFile(path.join(root, ".env"), "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return ""; + throw error; + } +} + +function runCommand(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd ?? ROOT, + env: options.env ?? process.env, + stdio: options.log ? ["ignore", "pipe", "pipe"] : (options.stdio ?? "inherit") + }); + + if (options.log) { + const tee = (stream, target) => { + stream.on("data", (chunk) => { + target.write(chunk); + options.log.append(chunk).catch(() => {}); + }); + }; + tee(child.stdout, process.stdout); + tee(child.stderr, process.stderr); + } + + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve(); + else { + const error = new Error(`${command} ${args.join(" ")} failed with code ${code}`); + if (options.logPath) error.logPath = options.logPath; + reject(error); + } + }); + }); +} + +function createRealDeps(context) { + const { root, parsed, ask, print } = context; + let config = null; + let deployLog = null; + + const commandOptions = () => ({ + cwd: root, + log: deployLog ?? undefined, + logPath: deployLog?.logPath + }); + + return { + async beginDeployLog() { + deployLog = await createDeployLogWriter(root); + return deployLog.logPath; + }, + async finalizeDeployLog() { + if (deployLog) await deployLog.finalize(); + }, + async loadConfiguration() { + const sourceText = await loadRootEnvText(root); + const overlaid = overlayProcessEnv(sourceText, process.env); + const generateSecrets = parsed.command === "deploy"; + const ensured = ensureDeploymentEnvironment(overlaid, { generateSecrets }); + config = { env: ensured.env, envText: ensured.text, sourceText, generatedKeys: ensured.generatedKeys }; + return config; + }, + async preflight(current, options) { + if (options?.command === "start") { + const apiDist = path.join(root, "apps/api/dist/index.js"); + const webBuild = path.join(root, "apps/web/.next/BUILD_ID"); + const webEnvLocal = path.join(root, "apps/web/.env.local"); + if (!existsSync(apiDist) || !existsSync(webBuild) || !existsSync(webEnvLocal)) { + throw new Error("请先运行 ./deploy.sh deploy"); + } + const diskEnv = parseDeploymentEnvironment(current.sourceText ?? ""); + if (!isCompleteDeploymentConfig(diskEnv)) { + throw new Error("缺少合法完整的 .env 配置，请先运行 ./deploy.sh deploy"); + } + } + }, + async configure(current) { + const configured = await configureDeploymentInteractively({ + root, + sourceText: current.sourceText ?? "", + reconfigure: parsed.reconfigure, + nonInteractive: parsed.nonInteractive, + ask, + print, + processEnv: process.env, + write: false + }); + config = { ...current, ...configured }; + return config; + }, + async checkDependencies(current) { + await ensureDependencies({ + datalinkEnabled: ["1", "true", "yes", "on"].includes( + String(current.env.DATALINK_ENABLED ?? "").toLowerCase() + ), + nonInteractive: parsed.nonInteractive, + ask, + print + }); + }, + async selectPorts(current) { + return current; + }, + async writeConfiguration(current) { + await writeDeploymentConfiguration(root, current.envText, current.webText, { + backup: parsed.reconfigure + }); + }, + async isRunning() { + const state = await readDeploymentState(root); + return Boolean(state?.pid && isProcessAlive(state.pid)); + }, + async stop() { + await stopManagedStack(root); + }, + async installProject() { + await runCommand("npm", ["ci"], commandOptions()); + }, + async buildTypeScript() { + await runCommand("npm", ["run", "build"], commandOptions()); + }, + async buildWeb() { + await runCommand("npm", ["run", "build:web"], commandOptions()); + }, + async buildTui() { + await runCommand("npm", ["run", "build:tui"], commandOptions()); + const entry = path.join(root, TUI_ENTRY); + if (!existsSync(entry)) { + throw new Error(`TUI build did not produce ${TUI_ENTRY}`); + } + }, + async installDataLink() { + await runCommand("uv", ["sync", "--project", "services/datalink", "--locked"], commandOptions()); + }, + async ensureTuiReady() { + const entry = path.join(root, TUI_ENTRY); + if (!existsSync(entry)) { + throw new Error(`TUI is not built (${TUI_ENTRY}). Run ./deploy.sh deploy first.`); + } + }, + async checkApiForTui(current) { + const apiBaseUrl = `http://127.0.0.1:${current.env.API_PORT}`; + const health = await probeHttp(`${apiBaseUrl}/healthz`); + if (!(health.ok && health.status === 200)) { + throw new Error( + `API is not healthy at ${apiBaseUrl}/healthz. Start the stack with ./deploy.sh start (or deploy), then retry ./deploy.sh tui.` + ); + } + }, + async startTui(current, parsed) { + const runtimeUrl = resolveTuiRuntimeUrl(current.env, parsed?.runtimeUrl); + print(`Starting TUI against ${runtimeUrl} (foreground client; Ctrl+C exits TUI only).`); + await runCommand("npm", ["run", "start:tui", "--", "--runtime-url", runtimeUrl], { + cwd: root, + stdio: "inherit" + }); + }, + + async verifyPorts(current) { + const services = [ + { label: "Web", port: current.env.WEB_PORT }, + { label: "API", port: current.env.API_PORT } + ]; + if (["1", "true", "yes", "on"].includes(String(current.env.DATALINK_ENABLED).toLowerCase())) { + services.push( + { label: "DataLink MCP", port: current.env.DATALINK_MCP_PORT }, + { label: "DataLink REST", port: current.env.DATALINK_API_PORT } + ); + } + // After stop-old, managed ports may still be briefly held by our previous process. + // Treat them as owned so verify-ports-again does not false-fail during redeploy. + const state = await readDeploymentState(root).catch(() => null); + const managedPorts = collectManagedPorts(current.env, state); + await verifySelectedPorts(services, { managedPorts }); + }, + async start(current) { + await startManagedStack(root, { + env: { ...process.env, ...current.env }, + ports: { + web: Number(current.env.WEB_PORT), + api: Number(current.env.API_PORT) + }, + datalinkEnabled: ["1", "true", "yes", "on"].includes( + String(current.env.DATALINK_ENABLED).toLowerCase() + ) + }); + }, + async waitForHealth(current) { + try { + return await waitForDeployment({ + checkProcessAlive: async () => { + const state = await readDeploymentState(root); + return Boolean(state?.pid && isProcessAlive(state.pid)); + }, + apiBaseUrl: `http://127.0.0.1:${current.env.API_PORT}`, + webUrl: `http://127.0.0.1:${current.env.WEB_PORT}`, + datalinkEnabled: ["1", "true", "yes", "on"].includes( + String(current.env.DATALINK_ENABLED).toLowerCase() + ), + datalinkRestUrl: `http://127.0.0.1:${current.env.DATALINK_API_PORT}`, + datalinkMcpHost: current.env.DATALINK_MCP_HOST || "127.0.0.1", + datalinkMcpPort: Number(current.env.DATALINK_MCP_PORT || 8080) + }); + } catch (error) { + const latest = await readDeploymentState(root); + if (latest?.pid) { + await writeDeploymentState(root, { ...latest, status: "unhealthy" }); + } + throw error; + } + }, + async markHealthy(current) { + const state = await readDeploymentState(root); + if (!state?.pid || !isProcessAlive(state.pid)) { + throw new Error("Managed process is not alive; refusing to mark deployment healthy"); + } + if (process.platform === "linux") { + const verification = await verifyManagedProcessForStop(state.pid, state.launchId); + if (!verification.allowed) { + throw new Error( + `Refusing to mark healthy: launch marker does not identify DataFoundry (${verification.reason})` + ); + } + } + await writeDeploymentState(root, { ...state, status: "healthy" }); + if (deployLog) await deployLog.finalize(); + print(`DataFoundry is healthy at ${current.env.AUTH_PUBLIC_BASE_URL}`); + print("Next: open Web, register/login, then create and enable a model profile."); + print("TUI is built and ready. Start it in another terminal (foreground client, not a background service):"); + print(" ./deploy.sh tui"); + print(" # or: npm run start:tui"); + }, + + async status() { + const state = await readDeploymentState(root); + const env = + config?.env ?? + ensureDeploymentEnvironment(await loadRootEnvText(root), { generateSecrets: false }).env; + const summary = await collectDeploymentHealth({ + checkProcessAlive: async () => Boolean(state?.pid && isProcessAlive(state.pid)), + apiBaseUrl: `http://127.0.0.1:${env.API_PORT}`, + webUrl: `http://127.0.0.1:${env.WEB_PORT}`, + datalinkEnabled: ["1", "true", "yes", "on"].includes(String(env.DATALINK_ENABLED).toLowerCase()), + datalinkRestUrl: `http://127.0.0.1:${env.DATALINK_API_PORT}`, + datalinkMcpHost: env.DATALINK_MCP_HOST || "127.0.0.1", + datalinkMcpPort: Number(env.DATALINK_MCP_PORT || 8080) + }); + print(`进程 ${summary.process}`); + print(`API ${summary.apiHealth}/${summary.apiReady}`); + print(`Web ${summary.web}`); + print(`DataLink ${summary.datalinkRest}`); + return summary; + }, + async logs() { + const logPath = deploymentPaths(root).runtimeLog; + await mkdir(path.dirname(logPath), { recursive: true }); + if (!existsSync(logPath)) await runCommand("touch", [logPath]); + await runCommand("tail", ["-n", "200", "-F", logPath]); + }, + async doctor() { + await runDeploymentDoctor(root, { print }); + }, + async printHelp(text) { + print(text); + }, + async reportFailure(failure) { + if (deployLog) await deployLog.finalize().catch(() => {}); + for (const line of formatDeploymentFailure(failure).split("\n")) { + print(line); + } + } + }; +} + +export async function runDeploymentDoctor(root, options = {}) { + const print = options.print ?? (() => {}); + const lines = []; + const record = (line) => { + lines.push(line); + print(line); + }; + + record(`os: ${process.platform} ${os.release()}`); + record(`arch: ${os.arch()}`); + + const envText = await loadRootEnvText(root).catch(() => ""); + const ensured = ensureDeploymentEnvironment(envText, { generateSecrets: false }); + const env = ensured.env; + const datalinkOn = ["1", "true", "yes", "on"].includes(String(env.DATALINK_ENABLED).toLowerCase()); + + const deps = await inspectDependencies({ datalinkEnabled: datalinkOn, run: options.run }); + for (const entry of deps) { + record( + `dependency ${entry.name}: ${entry.status}${entry.foundVersion ? ` (${entry.foundVersion})` : ""}` + ); + } + + const configIssues = []; + if (!env.WEB_PORT) configIssues.push("WEB_PORT missing"); + if (!env.API_PORT) configIssues.push("API_PORT missing"); + if (!env.AUTH_PUBLIC_BASE_URL) configIssues.push("AUTH_PUBLIC_BASE_URL missing"); + record(`config: ${configIssues.length === 0 ? "ok" : configIssues.join(", ")}`); + record(`config web=${env.WEB_PORT} api=${env.API_PORT} public=${env.AUTH_PUBLIC_BASE_URL}`); + + const state = await readDeploymentState(root); + const managedPorts = collectManagedPorts(env, state); + const portServices = [ + { label: "web", port: env.WEB_PORT }, + { label: "api", port: env.API_PORT } + ]; + if (datalinkOn) { + portServices.push( + { label: "datalink-mcp", port: env.DATALINK_MCP_PORT }, + { label: "datalink-rest", port: env.DATALINK_API_PORT } + ); + } + + const probe = options.probe ?? ((host, port) => probePort(host, port)); + for (const service of portServices) { + const port = Number(service.port); + if (!Number.isInteger(port)) { + record(`port ${service.label}: invalid (${service.port})`); + continue; + } + const result = await probe("0.0.0.0", port); + if (result.available) { + record(`port ${service.label} ${port}: available`); + } else if (managedPorts.has(port)) { + record(`port ${service.label} ${port}: in-use (managed)`); + } else { + record(`port ${service.label} ${port}: in-use (${result.owner ?? "unknown"})`); + } + } + + const storageRoot = path.join(root, env.STORAGE_ROOT_DIR || "storage"); + const envPath = path.join(root, ".env"); + const storageWritability = await inspectWritablePath(storageRoot); + if (storageWritability.writable) { + record( + `permissions storage: writable (${storageWritability.exists ? storageRoot : storageWritability.checkedPath})` + ); + } else { + record( + `permissions storage: not writable (${storageWritability.error ?? storageWritability.checkedPath})` + ); + } + record(`permissions .env: ${existsSync(envPath) ? "present" : "missing"}`); + + try { + const diskPath = existsSync(storageRoot) ? storageRoot : root; + const stats = await statfs(diskPath); + const freeGiB = ((stats.bfree * stats.bsize) / (1024 ** 3)).toFixed(1); + record(`disk free: ${freeGiB} GiB (${diskPath})`); + } catch (error) { + record(`disk: unavailable (${error.message})`); + } + + if (state?.pid) { + const alive = isProcessAlive(state.pid); + record(`pid: ${state.pid} status=${state.status ?? "unknown"} alive=${alive}`); + } else { + record("pid: none"); + } + + if (state?.pid && isProcessAlive(state.pid)) { + const summary = await collectDeploymentHealth( + { + processAlive: true, + apiBaseUrl: `http://127.0.0.1:${env.API_PORT}`, + webUrl: `http://127.0.0.1:${env.WEB_PORT}`, + datalinkEnabled: datalinkOn, + datalinkRestUrl: `http://127.0.0.1:${env.DATALINK_API_PORT}`, + datalinkMcpHost: env.DATALINK_MCP_HOST || "127.0.0.1", + datalinkMcpPort: Number(env.DATALINK_MCP_PORT || 8080) + }, + options.healthOptions + ); + record( + `health process=${summary.process} api=${summary.apiHealth}/${summary.apiReady} web=${summary.web} datalink=${summary.datalinkRest}` + ); + } else { + record("health: skipped (process not running)"); + } + + return { lines, env, state }; +} + +export function formatDeploymentFailure(failure) { + const summary = failure.summary ?? failure.error?.message ?? failure.stage; + const lines = [`✗ ${summary}`]; + if (failure.maintenanceWindow) { + lines.push("更新处于维护窗口，服务当前已停止；现有数据未被修改。"); + } else if (failure.configurationChanged) { + lines.push("配置可能已更新；现有数据未被修改。"); + } else { + lines.push("现有数据未被修改。"); + } + lines.push(`完整日志：${failure.logPath}`); + lines.push(`修复后重试：${failure.retryCommand}`); + lines.push(`诊断命令：${failure.doctorCommand}`); + if (failure.error?.message && failure.error.message !== summary) { + lines.push(redactSensitiveText(failure.error.message)); + } + return lines.join("\n"); +} + +export async function main(argv = process.argv.slice(2), options = {}) { + const root = options.root ?? ROOT; + let parsed; + try { + parsed = parseDeployArgs(argv); + } catch (error) { + process.stderr.write(`${error.message}\n`); + return 2; + } + + const print = options.print ?? ((message) => process.stdout.write(`${message}\n`)); + const ask = options.ask ?? createReadlineAsk(); + try { + if (parsed.command === "help") { + print(deploymentHelp()); + return 0; + } + const controller = createDeploymentController(createRealDeps({ root, parsed, ask, print })); + await controller.run(parsed); + return 0; + } catch (error) { + if (!error?.failure) print(redactSensitiveText(error.message ?? String(error))); + return 1; + } finally { + ask.close?.(); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exitCode = await main(process.argv.slice(2)); +} diff --git a/scripts/deploy/cli.test.mjs b/scripts/deploy/cli.test.mjs new file mode 100644 index 00000000..f3c81895 --- /dev/null +++ b/scripts/deploy/cli.test.mjs @@ -0,0 +1,400 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { mkdtemp, readdir, readFile, writeFile, mkdir } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + configureDeploymentInteractively, + collectManagedPorts, + createDeployLogWriter, + inspectWritablePath, + resolveAuthPublicBaseUrl, + resolveTuiRuntimeUrl, + runDeploymentDoctor +} from "./cli.mjs"; + +function createAsk(answers) { + const queue = [...answers]; + return async () => { + if (queue.length === 0) throw new Error("unexpected prompt"); + return queue.shift(); + }; +} + +test("first run explains DataLink and defaults to option 1", async () => { + const lines = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const result = await configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: false, + ask: createAsk(["1", "1", "1", ""]), + print: (line) => lines.push(String(line)), + probe: async () => ({ available: true, owner: null }) + }); + assert.match(lines.join("\n"), /DataLink 语义服务/); + assert.equal(result.env.DATALINK_ENABLED, "false"); + assert.equal(result.env.WEB_PORT, "3000"); +}); + +test("option 2 enables DataLink and selects MCP/REST ports", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const result = await configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: false, + ask: createAsk(["2", "1", "1", "1", "1", ""]), + print: () => {}, + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(result.env.DATALINK_ENABLED, "true"); + assert.equal(result.env.DATALINK_MCP_PORT, "8080"); + assert.equal(result.env.DATALINK_API_PORT, "8081"); +}); + +test("existing complete config skips prompts unless reconfigure", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const source = [ + "WEB_PORT=3310", + "API_PORT=8877", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "DATALINK_ENABLED=false", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3310", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + let asked = 0; + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: false, + nonInteractive: false, + ask: async () => { + asked += 1; + throw new Error("must not prompt"); + }, + print: () => {}, + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(asked, 0); + assert.equal(result.env.WEB_PORT, "3310"); +}); + +test("partial .env is not treated as complete before fill and still prompts", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-partial-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const lines = []; + let asked = 0; + const result = await configureDeploymentInteractively({ + root, + sourceText: "FOO=bar\nCUSTOM=keep-me\n", + reconfigure: false, + nonInteractive: false, + ask: createAsk(["1", "1", "1", ""]), + print: (line) => { + lines.push(String(line)); + asked += 1; + }, + probe: async () => ({ available: true, owner: null }) + }); + assert.match(lines.join("\n"), /DataLink 语义服务/); + assert.equal(result.env.DATALINK_ENABLED, "false"); + assert.match(result.envText, /CUSTOM=keep-me/); + assert.ok(asked > 0); +}); + +test("non-interactive never calls ask and warns on loopback public URL with remote bind", async () => { + const lines = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const result = await configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: true, + processEnv: { WEB_HOST: "0.0.0.0" }, + ask: async () => assert.fail("must not prompt"), + print: (line) => lines.push(String(line)), + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(result.env.DATALINK_ENABLED, "false"); + assert.match(lines.join("\n"), /本机访问|local-machine|仅适合本机/i); +}); + +test("port menu rejects n with explicit hint", async () => { + const lines = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const ask = createAsk(["1", "n", "1", "1", ""]); + await configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: false, + ask, + print: (line) => lines.push(String(line)), + probe: async () => ({ available: true, owner: null }) + }); + assert.match(lines.join("\n"), /请输入 1 或 2/); +}); + +test("port menu aborts after too many invalid choices", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + await assert.rejects( + () => + configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: false, + ask: createAsk(["1", "n", "n", "n", "n", "n"]), + print: () => {}, + probe: async () => ({ available: true, owner: null }) + }), + /too many invalid choices/ + ); +}); + +test("reconfigure creates backup and keeps secrets", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const source = [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "DATALINK_ENABLED=false", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + await writeFile(path.join(root, ".env"), `${source}\n`); + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: true, + nonInteractive: false, + ask: createAsk(["1", "1", "1", ""]), + print: () => {}, + probe: async () => ({ available: true, owner: null }), + write: true, + timestamp: "20260722-150000" + }); + assert.match(result.envText, /AUTH_SESSION_SECRET=existing-session-secret-value/); + assert.equal( + await readFile(path.join(root, ".env.backup-20260722-150000"), "utf8"), + `${source}\n` + ); +}); + +test("collectManagedPorts gathers env and deployment state ports", () => { + const ports = collectManagedPorts( + { WEB_PORT: "3310", API_PORT: "8877", DATALINK_MCP_PORT: "8080" }, + { ports: { web: 3310, api: 8877 } } + ); + assert.deepEqual([...ports].sort((a, b) => a - b), [3310, 8080, 8877]); +}); + +test("reconfigure reuses managed listening ports during update preflight", async () => { + const probes = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + await mkdir(path.join(root, "storage/run"), { recursive: true }); + const source = [ + "WEB_PORT=3310", + "API_PORT=8877", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "DATALINK_ENABLED=false", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3310", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + await writeFile( + path.join(root, "storage/run/deployment.json"), + `${JSON.stringify({ + pid: 99999, + launchId: "launch-1", + status: "healthy", + startedAt: "2026-07-22T00:00:00.000Z", + ports: { web: 3310, api: 8877 }, + datalinkEnabled: false + })}\n` + ); + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: true, + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + print: () => {}, + probe: async (port) => { + probes.push(port); + return { available: false, owner: "datafoundry pid=99999" }; + } + }); + assert.equal(result.env.WEB_PORT, "3310"); + assert.equal(result.env.API_PORT, "8877"); + assert.ok(probes.includes(3310)); + assert.ok(probes.includes(8877)); +}); + +test("resolveAuthPublicBaseUrl preserves HTTPS proxy URLs without explicit port", () => { + assert.equal( + resolveAuthPublicBaseUrl("https://prod.example.com", "3000", "3001"), + "https://prod.example.com" + ); + assert.equal( + resolveAuthPublicBaseUrl("http://127.0.0.1:3000", "3000", "3001"), + "http://127.0.0.1:3001" + ); + assert.equal( + resolveAuthPublicBaseUrl("https://prod.example.com:8443", "8443", "3001"), + "https://prod.example.com:8443" + ); + assert.equal( + resolveAuthPublicBaseUrl("https://prod.example.com:3000", "3000", "3001"), + "https://prod.example.com:3000" + ); + assert.equal( + resolveAuthPublicBaseUrl("http://example.com:3000", "3000", "3310"), + "http://example.com:3000" + ); +}); + +test("reconfigure keeps HTTPS public URL when ports are unchanged", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const source = [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "DATALINK_ENABLED=false", + "AUTH_PUBLIC_BASE_URL=https://prod.example.com", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: true, + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + print: () => {}, + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(result.env.AUTH_PUBLIC_BASE_URL, "https://prod.example.com"); +}); + +test("createDeployLogWriter redacts secrets and updates deploy-latest.log", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-log-")); + const writer = await createDeployLogWriter(root, { timestamp: "20260722143000" }); + await writer.append("AUTH_SESSION_SECRET=fixture-deploy-secret-at-least-32-chars\nnpm ci output\n"); + await writer.finalize(); + const logText = await readFile(path.join(root, "storage/logs/deploy-20260722143000.log"), "utf8"); + const latestText = await readFile(path.join(root, "storage/logs/deploy-latest.log"), "utf8"); + assert.doesNotMatch(logText, /fixture-deploy-secret-at-least-32-chars/); + assert.match(logText, /npm ci output/); + assert.equal(latestText, logText); + assert.equal(writer.logPath, "storage/logs/deploy-20260722143000.log"); +}); + +test("createDeployLogWriter redacts secrets split across append chunks", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-log-chunk-")); + const writer = await createDeployLogWriter(root, { timestamp: "20260722143001" }); + await writer.append("AUTH_SESSION_SECRET=fixture-deploy-secret-"); + await writer.append("at-least-32-chars\nnpm ci output\n"); + await writer.finalize(); + const logText = await readFile(path.join(root, "storage/logs/deploy-20260722143001.log"), "utf8"); + assert.doesNotMatch(logText, /fixture-deploy-secret-at-least-32-chars/); + assert.match(logText, /AUTH_SESSION_SECRET=\*{4,8}/); + assert.match(logText, /npm ci output/); +}); + +test("runDeploymentDoctor reports os, deps, config, ports, permissions, disk, pid, and health", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-doctor-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + await writeFile( + path.join(root, ".env"), + [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "DATALINK_ENABLED=false", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000" + ].join("\n") + ); + const beforeEntries = await readdir(path.join(root, "storage")); + const lines = []; + const result = await runDeploymentDoctor(root, { + print: (line) => lines.push(line), + run: async (command, args = []) => { + const key = [command, ...args].join(" "); + if (key === "node --version") return { stdout: "v22.14.0\n" }; + if (key === "npm --version") return { stdout: "10.9.0\n" }; + throw new Error(`missing mock for ${key}`); + }, + probe: async () => ({ available: true, owner: null }) + }); + const joined = lines.join("\n"); + assert.match(joined, /^os: /m); + assert.match(joined, /^arch: /m); + assert.match(joined, /dependency node: ok/); + assert.match(joined, /config: ok/); + assert.match(joined, /port web 3000: available/); + assert.match(joined, /permissions storage: writable/); + assert.match(joined, /disk free:/); + assert.match(joined, /pid: none/); + assert.match(joined, /health: skipped/); + assert.ok(result.lines.length >= 8); + const afterEntries = await readdir(path.join(root, "storage")); + assert.deepEqual(afterEntries, beforeEntries); + assert.ok(afterEntries.every((entry) => !entry.startsWith(".doctor-write-"))); +}); + +test("inspectWritablePath checks access without creating files", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-writable-")); + const missingChild = path.join(root, "storage", "nested"); + const result = await inspectWritablePath(missingChild); + assert.equal(result.writable, true); + assert.equal(result.exists, false); + assert.equal(existsSync(missingChild), false); + assert.equal(existsSync(path.join(root, "storage")), false); +}); + +test("resolveTuiRuntimeUrl defaults to deployed API port", () => { + assert.equal( + resolveTuiRuntimeUrl({ API_PORT: "8877" }), + "http://127.0.0.1:8877/api/copilotkit" + ); + assert.equal( + resolveTuiRuntimeUrl({ API_PORT: "8877" }, "http://example/api/copilotkit"), + "http://example/api/copilotkit" + ); +}); diff --git a/scripts/deploy/config.mjs b/scripts/deploy/config.mjs new file mode 100644 index 00000000..ec970e10 --- /dev/null +++ b/scripts/deploy/config.mjs @@ -0,0 +1,225 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +/** Minimal dotenv parser — avoids requiring a root `dotenv` dependency for deploy scripts. */ +export function parseDeploymentEnvironment(sourceText = "") { + const env = {}; + for (const rawLine of String(sourceText).split(/\r?\n/u)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const matched = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/u.exec(line); + if (!matched) continue; + let value = matched[2] ?? ""; + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + env[matched[1]] = value; + } + return env; +} + +const DEFAULTS = { + WEB_HOST: "0.0.0.0", + WEB_PORT: "3000", + API_HOST: "127.0.0.1", + API_PORT: "8787", + DATAFOUNDRY_AUTH_MODE: "password", + AUTH_EMAIL_DELIVERY: "test", + AUTH_REGISTRATION_MODE: "open", + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + DATALINK_ENABLED: "false", + DATALINK_MCP_HOST: "127.0.0.1", + DATALINK_MCP_PORT: "8080", + DATALINK_API_HOST: "127.0.0.1", + DATALINK_API_PORT: "8081", + STORAGE_ROOT_DIR: "storage", + METADATA_DB_PATH: "storage/metadata/workbench.sqlite" +}; + +const SECRET_KEYS = ["AUTH_SESSION_SECRET", "SECRET_MASTER_KEY"]; +const PLACEHOLDER_SECRETS = new Set(["", "change-me", "replace-me"]); +const SENSITIVE_KEY_PATTERN = /KEY|SECRET|TOKEN|PASSWORD|COOKIE|AUTHORIZATION/i; +const SENSITIVE_JSON_KEY_PATTERN = /^(?:api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|client[_-]?secret|secret|token|password|authorization|auth)$/i; + +export function isPlaceholderSecret(value) { + return value == null || PLACEHOLDER_SECRETS.has(String(value).trim()); +} + +export function isCompleteDeploymentConfig(env = {}) { + return Boolean( + String(env.WEB_PORT ?? "").trim() && + String(env.API_PORT ?? "").trim() && + String(env.AUTH_PUBLIC_BASE_URL ?? "").trim() && + String(env.DATALINK_ENABLED ?? "").trim() && + !isPlaceholderSecret(env.AUTH_SESSION_SECRET) && + !isPlaceholderSecret(env.SECRET_MASTER_KEY) + ); +} + +function maskSecret(value = "") { + return "*".repeat(Math.min(8, Math.max(4, String(value).length || 4))); +} + +function assertNoNewlines(value, key) { + if (String(value).includes("\n") || String(value).includes("\r")) { + throw new Error(`${key} must not contain newline characters`); + } +} + +function upsertEnvText(sourceText, updates) { + const lines = sourceText.length > 0 ? sourceText.replace(/\r\n/g, "\n").split("\n") : []; + if (lines.length > 0 && lines.at(-1) === "") lines.pop(); + + const seen = new Set(); + const next = lines.map((line) => { + const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line); + if (!match) return line; + const key = match[1]; + if (!(key in updates)) return line; + seen.add(key); + assertNoNewlines(updates[key], key); + return `${key}=${updates[key]}`; + }); + + for (const [key, value] of Object.entries(updates)) { + if (seen.has(key)) continue; + assertNoNewlines(value, key); + next.push(`${key}=${value}`); + } + + return `${next.join("\n")}\n`; +} + +export function updateDeploymentEnvironment(sourceText, updates) { + return upsertEnvText(sourceText ?? "", updates); +} + +export function ensureDeploymentEnvironment(sourceText, options = {}) { + const randomSecret = options.randomSecret ?? (() => randomBytes(32).toString("base64url")); + const generateSecrets = options.generateSecrets !== false; + const parsed = parseDeploymentEnvironment(sourceText ?? ""); + const updates = {}; + const generatedKeys = []; + + for (const [key, value] of Object.entries(DEFAULTS)) { + if (parsed[key] == null || String(parsed[key]).trim() === "") { + updates[key] = value; + generatedKeys.push(key); + } + } + + for (const key of SECRET_KEYS) { + if (isPlaceholderSecret(parsed[key])) { + if (!generateSecrets) continue; + updates[key] = randomSecret(); + generatedKeys.push(key); + } + } + + if ( + (parsed.AUTH_PUBLIC_BASE_URL == null || String(parsed.AUTH_PUBLIC_BASE_URL).trim() === "") && + updates.AUTH_PUBLIC_BASE_URL == null + ) { + const webPort = updates.WEB_PORT ?? parsed.WEB_PORT ?? DEFAULTS.WEB_PORT; + updates.AUTH_PUBLIC_BASE_URL = `http://127.0.0.1:${webPort}`; + generatedKeys.push("AUTH_PUBLIC_BASE_URL"); + } + + const text = Object.keys(updates).length > 0 + ? updateDeploymentEnvironment(sourceText ?? "", updates) + : sourceText?.endsWith("\n") || sourceText === "" + ? sourceText ?? "" + : `${sourceText}\n`; + + const env = { ...parseDeploymentEnvironment(text) }; + return { text, env, generatedKeys }; +} + +export function renderWebEnvironment(env) { + const authMode = env.DATAFOUNDRY_AUTH_MODE?.trim() || "password"; + const apiHost = env.API_HOST?.trim() || "127.0.0.1"; + const apiPort = env.API_PORT?.trim() || "8787"; + return [ + `NEXT_PUBLIC_DATAFOUNDRY_AUTH_MODE=${authMode}`, + "NEXT_PUBLIC_AGENT_RUNTIME_URL=", + "NEXT_PUBLIC_CONFIG_API_URL=", + `API_PROXY_TARGET=http://${apiHost}:${apiPort}`, + "" + ].join("\n"); +} + +async function writeAtomic(filePath, content, mode = 0o600) { + await mkdir(path.dirname(filePath), { recursive: true }); + const tempPath = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${Date.now()}.tmp` + ); + try { + await writeFile(tempPath, content, { encoding: "utf8", mode }); + await rename(tempPath, filePath); + } catch (error) { + await rm(tempPath, { force: true }).catch(() => {}); + throw error; + } +} + +export async function writeDeploymentConfiguration(root, rootText, webText, options = {}) { + const envPath = path.join(root, ".env"); + const webPath = path.join(root, "apps/web/.env.local"); + let backupPath; + + if (options.backup) { + const stamp = options.timestamp ?? new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 14); + backupPath = path.join(root, `.env.backup-${stamp}`); + const { readFile } = await import("node:fs/promises"); + let existing = ""; + try { + existing = await readFile(envPath, "utf8"); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + await writeAtomic(backupPath, existing, 0o600); + } + + await writeAtomic(envPath, rootText, 0o600); + await writeAtomic(webPath, webText, 0o600); + return { envPath, webPath, backupPath }; +} + +export function redactSensitiveText(text) { + let result = String(text ?? ""); + + result = result.replace(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/gm, (full, key, value) => { + if (!SENSITIVE_KEY_PATTERN.test(key)) return full; + return `${key}=${maskSecret(value)}`; + }); + + result = result.replace( + /("([A-Za-z_][A-Za-z0-9_]*)"\s*:\s*")([^"]*)(")/g, + (full, prefix, key, value, suffix) => { + if (!SENSITIVE_KEY_PATTERN.test(key) && !SENSITIVE_JSON_KEY_PATTERN.test(key)) return full; + return `${prefix}${maskSecret(value)}${suffix}`; + } + ); + + result = result.replace( + /(Authorization:\s*Bearer\s+)(\S+)/gi, + (_, prefix) => `${prefix}${maskSecret("bearer-token")}` + ); + + result = result.replace( + /(https?:\/\/)([^/\s:@]+):([^/\s@]+)@/gi, + (_, protocol) => `${protocol}****:****@` + ); + + result = result.replace( + /\b((?:sk|rk|pk|tok)-[A-Za-z0-9_-]{8,}|(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{16,}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g, + (value) => maskSecret(value) + ); + + return result; +} diff --git a/scripts/deploy/config.test.mjs b/scripts/deploy/config.test.mjs new file mode 100644 index 00000000..7fcbabae --- /dev/null +++ b/scripts/deploy/config.test.mjs @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + ensureDeploymentEnvironment, + isCompleteDeploymentConfig, + parseDeploymentEnvironment, + redactSensitiveText, + renderWebEnvironment, + updateDeploymentEnvironment, + writeDeploymentConfiguration +} from "./config.mjs"; + +test("creates safe defaults without model settings", () => { + const result = ensureDeploymentEnvironment("", { randomSecret: () => "generated-secret-value" }); + assert.equal(result.env.WEB_PORT, "3000"); + assert.equal(result.env.API_PORT, "8787"); + assert.equal(result.env.DATALINK_ENABLED, "false"); + assert.equal(result.env.AUTH_SESSION_SECRET, "generated-secret-value"); + assert.equal(result.env.SECRET_MASTER_KEY, "generated-secret-value"); + assert.equal(result.env.LLM_API_KEY, undefined); +}); + +test("preserves existing secrets and unrelated values", () => { + const source = "AUTH_SESSION_SECRET=existing-session\nSECRET_MASTER_KEY=existing-master\nCUSTOM_VALUE=keep-me\n"; + const result = ensureDeploymentEnvironment(source, { randomSecret: () => "replacement" }); + assert.match(result.text, /AUTH_SESSION_SECRET=existing-session/); + assert.match(result.text, /SECRET_MASTER_KEY=existing-master/); + assert.match(result.text, /CUSTOM_VALUE=keep-me/); +}); + +test("renders same-origin Web BFF configuration", () => { + const text = renderWebEnvironment({ + DATAFOUNDRY_AUTH_MODE: "password", + API_HOST: "127.0.0.1", + API_PORT: "8877" + }); + assert.match(text, /NEXT_PUBLIC_AGENT_RUNTIME_URL=$/m); + assert.match(text, /NEXT_PUBLIC_CONFIG_API_URL=$/m); + assert.match(text, /API_PROXY_TARGET=http:\/\/127\.0\.0\.1:8877/); +}); + +test("reconfigure creates a backup and atomically writes both files", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "datafoundry-config-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + await writeFile(path.join(root, ".env"), "AUTH_SESSION_SECRET=old\nSECRET_MASTER_KEY=old-master\n"); + const result = ensureDeploymentEnvironment(await readFile(path.join(root, ".env"), "utf8")); + const written = await writeDeploymentConfiguration(root, result.text, renderWebEnvironment(result.env), { + backup: true, + timestamp: "20260722-120000" + }); + assert.equal(await readFile(written.backupPath, "utf8"), "AUTH_SESSION_SECRET=old\nSECRET_MASTER_KEY=old-master\n"); + assert.match(await readFile(path.join(root, "apps/web/.env.local"), "utf8"), /API_PROXY_TARGET=/); + + if (process.platform !== "win32") { + for (const filePath of [ + path.join(root, ".env"), + path.join(root, "apps/web/.env.local"), + written.backupPath + ]) { + assert.equal((await stat(filePath)).mode & 0o777, 0o600, filePath); + } + } +}); + +test("updateDeploymentEnvironment upserts values while preserving comments", () => { + const source = "# comment\nWEB_PORT=3000\nCUSTOM=keep\n"; + const text = updateDeploymentEnvironment(source, { WEB_PORT: "3310", API_PORT: "8877" }); + assert.match(text, /# comment/); + assert.match(text, /WEB_PORT=3310/); + assert.match(text, /API_PORT=8877/); + assert.match(text, /CUSTOM=keep/); +}); + +test("redactSensitiveText masks secret-like keys", () => { + const redacted = redactSensitiveText("LLM_API_KEY=super-secret\nAUTH_SESSION_SECRET=abc\nWEB_PORT=3000\n"); + assert.match(redacted, /LLM_API_KEY=\*+/); + assert.match(redacted, /AUTH_SESSION_SECRET=\*+/); + assert.match(redacted, /WEB_PORT=3000/); + assert.doesNotMatch(redacted, /super-secret/); +}); + +test("redactSensitiveText masks JSON keys, bearer tokens, URL userinfo, and token prefixes", () => { + const redacted = redactSensitiveText( + [ + '{"apiKey":"json-secret-value","api_key":"also-secret"}', + "Authorization: Bearer fixture-deploy-secret-at-least-32-chars", + "https://user:fixture-deploy-secret-at-least-32-chars@example.com/path", + "token sk-abcdefghijklmnop", + "WEB_PORT=3000" + ].join("\n") + ); + assert.doesNotMatch(redacted, /json-secret-value|also-secret|fixture-deploy-secret-at-least-32-chars|sk-abcdefghijklmnop|user:fixture/); + assert.match(redacted, /Authorization: Bearer \*+/i); + assert.match(redacted, /https:\/\/\*\*\*\*:\*\*\*\*@example\.com\/path/); + assert.match(redacted, /WEB_PORT=3000/); +}); + +test("ensureDeploymentEnvironment can skip secret generation", () => { + const result = ensureDeploymentEnvironment("WEB_PORT=3000\n", { generateSecrets: false }); + assert.equal(result.env.AUTH_SESSION_SECRET, undefined); + assert.equal(result.env.SECRET_MASTER_KEY, undefined); + assert.ok(result.generatedKeys.includes("WEB_HOST")); + assert.ok(!result.generatedKeys.includes("AUTH_SESSION_SECRET")); +}); + +test("isCompleteDeploymentConfig rejects placeholders and partial env", () => { + assert.equal(isCompleteDeploymentConfig(parseDeploymentEnvironment("FOO=bar\n")), false); + assert.equal( + isCompleteDeploymentConfig({ + WEB_PORT: "3000", + API_PORT: "8787", + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + DATALINK_ENABLED: "false", + AUTH_SESSION_SECRET: "change-me", + SECRET_MASTER_KEY: "replace-me" + }), + false + ); + assert.equal( + isCompleteDeploymentConfig({ + WEB_PORT: "3000", + API_PORT: "8787", + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + DATALINK_ENABLED: "false", + AUTH_SESSION_SECRET: "existing-session-secret-value", + SECRET_MASTER_KEY: "existing-master-secret-value" + }), + true + ); +}); diff --git a/scripts/deploy/controller.mjs b/scripts/deploy/controller.mjs new file mode 100644 index 00000000..a9504fc3 --- /dev/null +++ b/scripts/deploy/controller.mjs @@ -0,0 +1,167 @@ +import { deploymentHelp } from "./args.mjs"; + +function datalinkEnabled(env = {}) { + return ["1", "true", "yes", "on"].includes(String(env.DATALINK_ENABLED ?? "").trim().toLowerCase()); +} + +export function createDeploymentController(deps) { + async function runDeploy(parsed) { + const calls = []; + let configurationChanged = false; + let serviceStopped = false; + let stage = "load-config"; + let config; + let logPath = "storage/logs/deploy-latest.log"; + + try { + if (deps.beginDeployLog) { + logPath = await deps.beginDeployLog(); + } + + stage = "load-config"; + config = await deps.loadConfiguration({ reconfigure: parsed.reconfigure }); + calls.push("load-config"); + + stage = "preflight"; + await deps.preflight(config, parsed); + calls.push("preflight"); + + stage = "configure"; + config = await deps.configure(config, parsed); + calls.push("configure"); + + stage = "check-dependencies"; + await deps.checkDependencies(config, parsed); + calls.push("check-dependencies"); + + stage = "select-ports"; + config = await deps.selectPorts(config, parsed); + calls.push("select-ports"); + + stage = "write-config"; + await deps.writeConfiguration(config, parsed); + configurationChanged = true; + calls.push("write-config"); + + const running = await deps.isRunning(); + if (running) { + stage = "stop-old"; + await deps.stop(); + serviceStopped = true; + calls.push("stop-old"); + } + + stage = "install"; + await deps.installProject(config, parsed); + calls.push("install"); + + stage = "build-typescript"; + await deps.buildTypeScript(config, parsed); + calls.push("build-typescript"); + + stage = "build-web"; + await deps.buildWeb(config, parsed); + calls.push("build-web"); + + stage = "build-tui"; + await deps.buildTui(config, parsed); + calls.push("build-tui"); + + if (datalinkEnabled(config.env)) { + stage = "install-datalink"; + await deps.installDataLink(config, parsed); + calls.push("install-datalink"); + } + + stage = "verify-ports-again"; + await deps.verifyPorts(config, parsed); + calls.push("verify-ports-again"); + + stage = "start"; + await deps.start(config, parsed); + calls.push("start"); + + stage = "wait-for-health"; + await deps.waitForHealth(config, parsed); + calls.push("wait-for-health"); + + stage = "mark-healthy"; + await deps.markHealthy(config, parsed); + calls.push("mark-healthy"); + + return { ok: true, calls, configurationChanged, serviceStopped }; + } catch (error) { + const failure = { + stage, + summary: error?.message ?? String(error), + error, + configurationChanged, + serviceStopped, + maintenanceWindow: serviceStopped, + oldServiceRunning: false, + logPath: error?.logPath ?? logPath, + retryCommand: "./deploy.sh deploy", + doctorCommand: "./deploy.sh doctor", + calls + }; + await deps.reportFailure(failure); + const wrapped = new Error(error?.message ?? String(error)); + wrapped.failure = failure; + throw wrapped; + } + } + + async function runStart() { + const config = await deps.loadConfiguration({ reconfigure: false }); + await deps.preflight(config, { command: "start" }); + await deps.start(config, { command: "start" }); + await deps.waitForHealth(config, { command: "start" }); + await deps.markHealthy(config, { command: "start" }); + return { ok: true }; + } + + async function runStop() { + await deps.stop(); + return { ok: true }; + } + + async function runRestart() { + await deps.stop(); + return runStart(); + } + + async function runTui(parsed) { + const config = await deps.loadConfiguration({ reconfigure: false }); + await deps.ensureTuiReady(config, parsed); + await deps.checkApiForTui(config, parsed); + await deps.startTui(config, parsed); + return { ok: true }; + } + + return { + async run(parsed) { + switch (parsed.command) { + case "deploy": + return runDeploy(parsed); + case "start": + return runStart(); + case "stop": + return runStop(); + case "restart": + return runRestart(); + case "status": + return deps.status(); + case "logs": + return deps.logs(); + case "doctor": + return deps.doctor(); + case "tui": + return runTui(parsed); + case "help": + return deps.printHelp(deploymentHelp()); + default: + throw new Error(`Unknown command: ${parsed.command}`); + } + } + }; +} diff --git a/scripts/deploy/controller.test.mjs b/scripts/deploy/controller.test.mjs new file mode 100644 index 00000000..c459ce47 --- /dev/null +++ b/scripts/deploy/controller.test.mjs @@ -0,0 +1,172 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createDeploymentController } from "./controller.mjs"; + +test("deploy update path uses exact maintenance-window ordering", async () => { + const controller = createDeploymentController({ + async loadConfiguration() { + return { env: { DATALINK_ENABLED: "false" } }; + }, + async preflight() {}, + async configure(config) { + return config; + }, + async checkDependencies() {}, + async selectPorts(config) { + return config; + }, + async writeConfiguration() {}, + async isRunning() { + return true; + }, + async stop() {}, + async installProject() {}, + async buildTypeScript() {}, + async buildWeb() {}, + async buildTui() {}, + async installDataLink() {}, + async verifyPorts() {}, + async start() {}, + async waitForHealth() {}, + async markHealthy() {}, + async reportFailure() {} + }); + + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + assert.deepEqual(result.calls, [ + "load-config", + "preflight", + "configure", + "check-dependencies", + "select-ports", + "write-config", + "stop-old", + "install", + "build-typescript", + "build-web", + "build-tui", + "verify-ports-again", + "start", + "wait-for-health", + "mark-healthy" + ]); + assert.ok(!result.calls.includes("start-tui")); +}); + +test("build failure after stop never starts and reports maintenance window", async () => { + let failure; + const controller = createDeploymentController({ + async loadConfiguration() { + return { env: { DATALINK_ENABLED: "false" } }; + }, + async preflight() {}, + async configure(config) { + return config; + }, + async checkDependencies() {}, + async selectPorts(config) { + return config; + }, + async writeConfiguration() {}, + async isRunning() { + return true; + }, + async stop() {}, + async installProject() {}, + async buildTypeScript() { + const error = new Error("build failed"); + error.logPath = "storage/logs/deploy-20260722.log"; + throw error; + }, + async buildWeb() {}, + async buildTui() {}, + async installDataLink() {}, + async verifyPorts() {}, + async start() {}, + async waitForHealth() {}, + async markHealthy() {}, + async reportFailure(details) { + failure = details; + } + }); + + const storage = { sentinel: Buffer.from("keep-me") }; + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /build failed/ + ); + assert.ok(!failure.calls.includes("start")); + assert.ok(failure.calls.includes("stop-old")); + assert.equal(failure.maintenanceWindow, true); + assert.equal(failure.serviceStopped, true); + assert.equal(failure.stage, "build-typescript"); + assert.equal(storage.sentinel.toString(), "keep-me"); +}); + +test("stop and help dispatch", async () => { + const calls = []; + const controller = createDeploymentController({ + async stop() { + calls.push("stop"); + }, + async printHelp() { + calls.push("help"); + } + }); + await controller.run({ command: "stop" }); + await controller.run({ command: "help" }); + assert.deepEqual(calls, ["stop", "help"]); +}); + +test("optional tui command checks readiness and API then starts foreground client", async () => { + const calls = []; + const controller = createDeploymentController({ + async loadConfiguration() { + calls.push("load-config"); + return { env: { API_PORT: "8787" } }; + }, + async ensureTuiReady() { + calls.push("ensure-tui-ready"); + }, + async checkApiForTui() { + calls.push("check-api-for-tui"); + }, + async startTui(_config, parsed) { + calls.push(`start-tui:${parsed.runtimeUrl ?? "default"}`); + } + }); + await controller.run({ + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: "http://127.0.0.1:9000/api/copilotkit" + }); + assert.deepEqual(calls, [ + "load-config", + "ensure-tui-ready", + "check-api-for-tui", + "start-tui:http://127.0.0.1:9000/api/copilotkit" + ]); +}); + +test("tui aborts before start when API check fails", async () => { + const calls = []; + const controller = createDeploymentController({ + async loadConfiguration() { + return { env: { API_PORT: "8787" } }; + }, + async ensureTuiReady() { + calls.push("ensure-tui-ready"); + }, + async checkApiForTui() { + calls.push("check-api-for-tui"); + throw new Error("API is not healthy"); + }, + async startTui() { + calls.push("start-tui"); + } + }); + await assert.rejects(() => controller.run({ command: "tui", runtimeUrl: null }), /API is not healthy/); + assert.deepEqual(calls, ["ensure-tui-ready", "check-api-for-tui"]); + assert.ok(!calls.includes("start-tui")); +}); diff --git a/scripts/deploy/dependencies.mjs b/scripts/deploy/dependencies.mjs new file mode 100644 index 00000000..a2fd52a4 --- /dev/null +++ b/scripts/deploy/dependencies.mjs @@ -0,0 +1,165 @@ +import { spawn } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +async function defaultRun(command, args = [], options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd ?? ROOT, + env: options.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve({ stdout, stderr, code }); + else { + const error = new Error(stderr.trim() || `${command} exited ${code}`); + error.code = code; + error.stdout = stdout; + error.stderr = stderr; + reject(error); + } + }); + }); +} + +function parseNodeVersion(text) { + const match = /v?(\d+)\.(\d+)\.(\d+)/.exec(text ?? ""); + if (!match) return null; + return { major: Number(match[1]), minor: Number(match[2]), patch: Number(match[3]), raw: match[0] }; +} + +function parsePythonVersion(text) { + const match = /Python\s+(\d+)\.(\d+)\.(\d+)/i.exec(text ?? ""); + if (!match) return null; + return { major: Number(match[1]), minor: Number(match[2]), patch: Number(match[3]), raw: match[0] }; +} + +async function safeRun(run, command, args = []) { + try { + return await run(command, args); + } catch (error) { + return { error, stdout: error?.stdout ?? "", stderr: error?.stderr ?? "" }; + } +} + +export async function inspectDependencies(options = {}) { + const run = options.run ?? defaultRun; + const datalinkEnabled = Boolean(options.datalinkEnabled); + const entries = []; + + const node = await safeRun(run, "node", ["--version"]); + const nodeVersion = parseNodeVersion(node.stdout || node.stderr); + entries.push({ + name: "node", + required: true, + foundVersion: nodeVersion?.raw ?? null, + minimumVersion: "22", + status: nodeVersion && nodeVersion.major >= 22 ? "ok" : "missing", + installAction: "node" + }); + + const npm = await safeRun(run, "npm", ["--version"]); + const npmVersion = (npm.stdout || "").trim() || null; + entries.push({ + name: "npm", + required: true, + foundVersion: npmVersion, + minimumVersion: "any", + status: npmVersion ? "ok" : "missing", + installAction: "node" + }); + + if (datalinkEnabled) { + const python = await safeRun(run, "python3", ["--version"]); + const pythonVersion = parsePythonVersion(python.stdout || python.stderr); + const pythonOk = pythonVersion && (pythonVersion.major > 3 || (pythonVersion.major === 3 && pythonVersion.minor >= 10)); + entries.push({ + name: "python", + required: true, + foundVersion: pythonVersion?.raw ?? null, + minimumVersion: "3.10", + status: pythonOk ? "ok" : "missing", + installAction: "python" + }); + + const uv = await safeRun(run, "uv", ["--version"]); + const uvVersion = (uv.stdout || "").trim() || null; + entries.push({ + name: "uv", + required: true, + foundVersion: uvVersion, + minimumVersion: "any", + status: uvVersion ? "ok" : "missing", + installAction: "uv" + }); + } + + return entries; +} + +async function canInstallNonInteractive(options, run) { + if ((options.uid ?? process.getuid?.() ?? 1000) === 0) return true; + const result = await safeRun(run, "sudo", ["-n", "true"]); + return !result.error; +} + +export async function ensureDependencies(options = {}) { + const run = options.run ?? defaultRun; + const ask = options.ask; + const install = options.install ?? (async (action, installOptions = {}) => { + const args = [path.join(ROOT, "scripts/deploy/install-dependency.sh"), action]; + if (installOptions.nonInteractive) { + args.push("--non-interactive"); + } + await run("bash", args); + }); + const print = options.print ?? ((message) => process.stdout.write(`${message}\n`)); + + let entries = await inspectDependencies(options); + const missing = entries.filter((entry) => entry.status !== "ok"); + if (missing.length === 0) return entries; + + for (const entry of missing) { + const action = entry.installAction; + const commandHint = `bash scripts/deploy/install-dependency.sh ${action}`; + if (options.nonInteractive) { + const allowed = await canInstallNonInteractive(options, run); + if (!allowed) { + throw new Error( + `Missing ${entry.name}. Non-interactive install requires root or passwordless sudo. Or run: ${commandHint}` + ); + } + print(`Installing ${entry.name} via ${commandHint}`); + await install(action, { nonInteractive: true }); + continue; + } + + print(`缺少依赖：${entry.name}（需要 ${entry.minimumVersion}，当前 ${entry.foundVersion ?? "未找到"}）`); + print(`将执行：${commandHint}`); + const answer = String(await ask("是否安装？[y/N]：" )).trim().toLowerCase(); + if (answer !== "y" && answer !== "yes") { + throw new Error(`Dependency ${entry.name} is required. Install with: ${commandHint}`); + } + await install(action, { nonInteractive: false }); + } + + entries = await inspectDependencies(options); + const stillMissing = entries.filter((entry) => entry.status !== "ok"); + if (stillMissing.length > 0) { + throw new Error( + `Dependencies still missing after install: ${stillMissing.map((e) => e.name).join(", ")}` + ); + } + return entries; +} diff --git a/scripts/deploy/dependencies.test.mjs b/scripts/deploy/dependencies.test.mjs new file mode 100644 index 00000000..51fdfd35 --- /dev/null +++ b/scripts/deploy/dependencies.test.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { ensureDependencies, inspectDependencies } from "./dependencies.mjs"; + +function runner(map) { + return async (command, args = []) => { + const key = [command, ...args].join(" "); + if (!(key in map) && !(command in map)) { + const error = new Error(`missing mock for ${key}`); + error.code = "ENOENT"; + throw error; + } + const value = map[key] ?? map[command]; + if (typeof value === "function") return value(args); + if (value instanceof Error) throw value; + return value; + }; +} + +test("requires Node 22 and npm; skips Python/uv when DataLink disabled", async () => { + const entries = await inspectDependencies({ + datalinkEnabled: false, + run: runner({ + "node --version": { stdout: "v22.14.0\n" }, + "npm --version": { stdout: "10.9.0\n" } + }) + }); + assert.equal(entries.find((e) => e.name === "node").status, "ok"); + assert.equal(entries.find((e) => e.name === "npm").status, "ok"); + assert.equal(entries.find((e) => e.name === "python"), undefined); + assert.equal(entries.find((e) => e.name === "uv"), undefined); +}); + +test("checks Python 3.10+ and uv when DataLink enabled", async () => { + const entries = await inspectDependencies({ + datalinkEnabled: true, + run: runner({ + "node --version": { stdout: "v22.14.0\n" }, + "npm --version": { stdout: "10.9.0\n" }, + "python3 --version": { stdout: "Python 3.9.0\n" }, + "uv --version": { stdout: "uv 0.6.0\n" } + }) + }); + assert.equal(entries.find((e) => e.name === "python").status, "missing"); + assert.equal(entries.find((e) => e.name === "uv").status, "ok"); +}); + +test("interactive refusal returns corrective install command", async () => { + await assert.rejects( + () => ensureDependencies({ + datalinkEnabled: false, + nonInteractive: false, + ask: async () => "n", + run: runner({ + "node --version": { stdout: "v20.0.0\n" }, + "npm --version": { stdout: "10.0.0\n" } + }), + install: async () => assert.fail("must not install") + }), + /install-dependency\.sh node|Node\.js 22/ + ); +}); + +test("non-interactive install requires root or passwordless sudo", async () => { + await assert.rejects( + () => ensureDependencies({ + datalinkEnabled: false, + nonInteractive: true, + uid: 1000, + ask: async () => assert.fail("must not prompt"), + run: runner({ + "node --version": { stdout: "v20.0.0\n" }, + "npm --version": { stdout: "10.0.0\n" }, + "sudo -n true": Object.assign(new Error("sudo failed"), { code: 1 }) + }), + install: async () => assert.fail("must not install") + }), + /root|passwordless sudo|non-interactive/i + ); +}); + +test("non-interactive install passes --non-interactive to install-dependency.sh", async () => { + const installs = []; + let nodeCalls = 0; + await ensureDependencies({ + datalinkEnabled: false, + nonInteractive: true, + uid: 0, + ask: async () => assert.fail("must not prompt"), + run: async (command, args = []) => { + const key = [command, ...args].join(" "); + if (key === "node --version") { + nodeCalls += 1; + return nodeCalls === 1 ? { stdout: "v20.0.0\n" } : { stdout: "v22.14.0\n" }; + } + if (key === "npm --version") return { stdout: "10.9.0\n" }; + throw new Error(`missing mock for ${key}`); + }, + install: async (action, installOptions) => { + installs.push({ action, installOptions }); + } + }); + assert.equal(installs.length, 1); + assert.equal(installs[0].action, "node"); + assert.equal(installs[0].installOptions.nonInteractive, true); +}); diff --git a/scripts/deploy/health.mjs b/scripts/deploy/health.mjs new file mode 100644 index 00000000..29a7b387 --- /dev/null +++ b/scripts/deploy/health.mjs @@ -0,0 +1,107 @@ +import net from "node:net"; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function probeHttp(url, options = {}) { + const timeoutMs = options.timeoutMs ?? 5000; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal, redirect: "manual" }); + return { ok: response.status >= 200 && response.status < 400, status: response.status }; + } catch (error) { + return { ok: false, status: null, error: error?.message ?? String(error) }; + } finally { + clearTimeout(timer); + } +} + +export async function probeTcp(host, port, options = {}) { + const timeoutMs = options.timeoutMs ?? 5000; + return new Promise((resolve) => { + const socket = net.connect({ host, port }); + const timer = setTimeout(() => { + socket.destroy(); + resolve({ ok: false, error: "timeout" }); + }, timeoutMs); + socket.once("connect", () => { + clearTimeout(timer); + socket.end(); + resolve({ ok: true }); + }); + socket.once("error", (error) => { + clearTimeout(timer); + resolve({ ok: false, error: error.message }); + }); + }); +} + +async function resolveProcessAlive(config) { + if (typeof config.checkProcessAlive === "function") { + return Boolean(await config.checkProcessAlive()); + } + return Boolean(config.processAlive); +} + +export async function collectDeploymentHealth(config, options = {}) { + const probeTimeoutMs = options.probeTimeoutMs ?? 5000; + const processAlive = await resolveProcessAlive(config); + const processStatus = processAlive ? "running" : "stopped"; + + const health = await probeHttp(`${config.apiBaseUrl}/healthz`, { timeoutMs: probeTimeoutMs }); + const ready = await probeHttp(`${config.apiBaseUrl}/ready`, { timeoutMs: probeTimeoutMs }); + const web = await probeHttp(config.webUrl, { timeoutMs: probeTimeoutMs }); + + let datalinkRest = "disabled"; + let datalinkMcp = "disabled"; + if (config.datalinkEnabled) { + const rest = await probeHttp(`${config.datalinkRestUrl}/healthz`, { timeoutMs: probeTimeoutMs }); + datalinkRest = rest.ok && rest.status === 200 ? "healthy" : "unhealthy"; + const mcp = await probeTcp(config.datalinkMcpHost, config.datalinkMcpPort, { + timeoutMs: probeTimeoutMs + }); + datalinkMcp = mcp.ok ? "healthy" : "unhealthy"; + } + + const apiHealth = health.ok && health.status === 200 ? "healthy" : "unhealthy"; + const apiReady = ready.ok && ready.status === 200 ? "ready" : "unhealthy"; + const webStatus = web.ok ? "reachable" : "unreachable"; + const ok = + processStatus === "running" && + apiHealth === "healthy" && + apiReady === "ready" && + webStatus === "reachable" && + (datalinkRest === "disabled" || datalinkRest === "healthy") && + (datalinkMcp === "disabled" || datalinkMcp === "healthy"); + + return { + process: processStatus, + apiHealth, + apiReady, + web: webStatus, + datalinkRest, + datalinkMcp, + ok + }; +} + +export async function waitForDeployment(config, options = {}) { + const intervalMs = options.intervalMs ?? 1000; + const timeoutMs = options.timeoutMs ?? 60_000; + const deadline = Date.now() + timeoutMs; + let summary; + + while (Date.now() <= deadline) { + summary = await collectDeploymentHealth(config, options); + if (summary.ok) return summary; + await sleep(intervalMs); + } + + const error = new Error( + `Deployment health check timed out: process=${summary.process} apiHealth=${summary.apiHealth} apiReady=${summary.apiReady} web=${summary.web} datalinkRest=${summary.datalinkRest} datalinkMcp=${summary.datalinkMcp}` + ); + error.summary = summary; + throw error; +} diff --git a/scripts/deploy/health.test.mjs b/scripts/deploy/health.test.mjs new file mode 100644 index 00000000..1defdf74 --- /dev/null +++ b/scripts/deploy/health.test.mjs @@ -0,0 +1,179 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import net from "node:net"; +import test from "node:test"; +import { + collectDeploymentHealth, + probeHttp, + probeTcp, + waitForDeployment +} from "./health.mjs"; + +function listen(server) { + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address().port)); + }); +} + +test("probeHttp and probeTcp succeed against ephemeral servers", async () => { + const server = http.createServer((_req, res) => { + res.writeHead(200); + res.end("ok"); + }); + const port = await listen(server); + try { + const result = await probeHttp(`http://127.0.0.1:${port}/`); + assert.equal(result.ok, true); + assert.equal(result.status, 200); + const tcp = await probeTcp("127.0.0.1", port); + assert.equal(tcp.ok, true); + } finally { + server.close(); + } +}); + +test("collectDeploymentHealth reports healthy stack without DataLink", async () => { + const api = http.createServer((req, res) => { + if (req.url === "/healthz" || req.url === "/ready") { + res.writeHead(200); + res.end("ok"); + return; + } + res.writeHead(404); + res.end(); + }); + const web = http.createServer((_req, res) => { + res.writeHead(302); + res.end(); + }); + const apiPort = await listen(api); + const webPort = await listen(web); + try { + const summary = await collectDeploymentHealth({ + processAlive: true, + apiBaseUrl: `http://127.0.0.1:${apiPort}`, + webUrl: `http://127.0.0.1:${webPort}`, + datalinkEnabled: false + }); + assert.deepEqual(summary, { + process: "running", + apiHealth: "healthy", + apiReady: "ready", + web: "reachable", + datalinkRest: "disabled", + datalinkMcp: "disabled", + ok: true + }); + } finally { + api.close(); + web.close(); + } +}); + +test("waitForDeployment retries until ready then times out with summary", async () => { + let ready = false; + const api = http.createServer((req, res) => { + if (req.url === "/healthz") { + res.writeHead(200); + res.end("ok"); + return; + } + if (req.url === "/ready") { + res.writeHead(ready ? 200 : 503); + res.end(ready ? "ok" : "starting"); + return; + } + res.writeHead(404); + res.end(); + }); + const web = http.createServer((_req, res) => { + res.writeHead(200); + res.end("web"); + }); + const apiPort = await listen(api); + const webPort = await listen(web); + const config = { + processAlive: true, + apiBaseUrl: `http://127.0.0.1:${apiPort}`, + webUrl: `http://127.0.0.1:${webPort}`, + datalinkEnabled: false + }; + + try { + setTimeout(() => { + ready = true; + }, 80); + const summary = await waitForDeployment(config, { + intervalMs: 40, + timeoutMs: 1000, + probeTimeoutMs: 200 + }); + assert.equal(summary.ok, true); + + await assert.rejects( + () => waitForDeployment({ ...config, processAlive: false }, { + intervalMs: 20, + timeoutMs: 60, + probeTimeoutMs: 50 + }), + /process|unhealthy|timeout/i + ); + } finally { + api.close(); + web.close(); + } +}); + +test("waitForDeployment rechecks processAlive each round via callback", async () => { + let aliveChecks = 0; + // Keep HTTP healthy so the only failing gate is processAlive, avoiding a race + // where /ready flips true while the callback still reports alive. + const api = http.createServer((req, res) => { + if (req.url === "/healthz" || req.url === "/ready") { + res.writeHead(200); + res.end("ok"); + return; + } + res.writeHead(404); + res.end("missing"); + }); + const web = http.createServer((_req, res) => { + res.writeHead(200); + res.end("web"); + }); + const apiPort = await listen(api); + const webPort = await listen(web); + + try { + await assert.rejects( + () => + waitForDeployment( + { + checkProcessAlive: async () => { + aliveChecks += 1; + return false; + }, + apiBaseUrl: `http://127.0.0.1:${apiPort}`, + webUrl: `http://127.0.0.1:${webPort}`, + datalinkEnabled: false + }, + { intervalMs: 20, timeoutMs: 120, probeTimeoutMs: 50 } + ), + /process=stopped|timed out/i + ); + assert.ok(aliveChecks >= 2); + } finally { + api.close(); + web.close(); + } +}); + +test("TCP probe fails for closed ports", async () => { + const server = net.createServer(); + const port = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address().port)); + }); + await new Promise((resolve) => server.close(resolve)); + const result = await probeTcp("127.0.0.1", port, { timeoutMs: 100 }); + assert.equal(result.ok, false); +}); diff --git a/scripts/deploy/install-dependency.sh b/scripts/deploy/install-dependency.sh new file mode 100755 index 00000000..b7659bd3 --- /dev/null +++ b/scripts/deploy/install-dependency.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ACTION="${1:-}" +NON_INTERACTIVE=0 +if [[ "${2:-}" == "--non-interactive" ]] || [[ "${DEPLOY_NON_INTERACTIVE:-}" == "1" ]]; then + NON_INTERACTIVE=1 +fi + +usage() { + echo "Usage: install-dependency.sh [--non-interactive]" >&2 + exit 2 +} + +case "${ACTION}" in + node|python|uv) ;; + *) usage ;; +esac + +run_privileged() { + if [[ "$(id -u)" -eq 0 ]]; then + "$@" + elif [[ "${NON_INTERACTIVE}" -eq 1 ]]; then + sudo -n "$@" + else + sudo "$@" + fi +} + +install_python() { + run_privileged apt-get update + run_privileged apt-get install -y python3 python3-venv + local version + version="$(python3 --version 2>&1 || true)" + if [[ ! "${version}" =~ Python\ (3)\.([0-9]+) ]]; then + echo "Python 3.10+ is required after installation; found ${version:-none}." >&2 + exit 1 + fi + local minor="${BASH_REMATCH[2]}" + if [[ "${BASH_REMATCH[1]}" -lt 3 || "${minor}" -lt 10 ]]; then + echo "Python 3.10+ is required after installation; found ${version}." >&2 + exit 1 + fi +} + +install_uv() { + local installer + installer="$(mktemp)" + trap 'rm -f "${installer}"' RETURN + curl -fsSL "https://astral.sh/uv/install.sh" -o "${installer}" + sh "${installer}" + rm -f "${installer}" + trap - RETURN + export PATH="${HOME}/.local/bin:${PATH}" + if ! command -v uv >/dev/null 2>&1; then + echo "uv installation did not produce a working uv command." >&2 + exit 1 + fi + uv --version >/dev/null +} + +install_node() { + local setup + local nodesource_url="https://deb.nodesource.com/setup_22.x" + setup="$(mktemp)" + trap 'rm -f "${setup}"' RETURN + curl -fsSL "${nodesource_url}" -o "${setup}" + run_privileged bash "${setup}" + run_privileged apt-get install -y nodejs + rm -f "${setup}" + trap - RETURN + node --version >/dev/null + npm --version >/dev/null +} + +case "${ACTION}" in + python) install_python ;; + uv) install_uv ;; + node) install_node ;; +esac diff --git a/scripts/deploy/lifecycle.test.mjs b/scripts/deploy/lifecycle.test.mjs new file mode 100644 index 00000000..64aa4351 --- /dev/null +++ b/scripts/deploy/lifecycle.test.mjs @@ -0,0 +1,358 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createDeploymentController } from "./controller.mjs"; +import { redactSensitiveText } from "./config.mjs"; +import { formatDeploymentFailure } from "./cli.mjs"; + +function createFakeDeployDeps(overrides = {}) { + const calls = []; + const storage = { sentinel: Buffer.from("storage-sentinel-bytes") }; + let running = Boolean(overrides.running); + let statusState = { process: "stopped", apiHealth: "unhealthy", apiReady: "unhealthy", web: "unreachable", datalinkRest: "disabled", ok: false }; + + const deps = { + calls, + storage, + async loadConfiguration() { + calls.push("load-config"); + return { env: { DATALINK_ENABLED: "false", WEB_PORT: "3000", API_PORT: "8787" }, envText: "", webText: "" }; + }, + async preflight(_config, options) { + calls.push("preflight"); + if (options?.command === "start" && overrides.missingBuild) { + throw new Error("请先运行 ./deploy.sh deploy"); + } + }, + async configure(config) { + calls.push("configure"); + return config; + }, + async checkDependencies() { + calls.push("check-dependencies"); + }, + async selectPorts(config) { + calls.push("select-ports"); + return config; + }, + async writeConfiguration() { + calls.push("write-config"); + }, + async isRunning() { + return running; + }, + async stop() { + calls.push("stop"); + running = false; + statusState = { ...statusState, process: "stopped", ok: false }; + }, + async installProject() { + calls.push("install"); + }, + async buildTypeScript() { + calls.push("build-typescript"); + if (overrides.failBuild) { + const error = new Error("TypeScript build failed"); + error.logPath = "storage/logs/deploy-20260722-143000.log"; + throw error; + } + }, + async buildWeb() { + calls.push("build-web"); + if (overrides.failWebBuild) { + const error = new Error("Web build failed"); + error.logPath = "storage/logs/deploy-20260722-143000.log"; + throw error; + } + }, + async buildTui() { + calls.push("build-tui"); + if (overrides.failTuiBuild) { + const error = new Error("TUI build failed"); + error.logPath = "storage/logs/deploy-20260722-143000.log"; + throw error; + } + }, + async ensureTuiReady() { + calls.push("ensure-tui-ready"); + }, + async checkApiForTui() { + calls.push("check-api-for-tui"); + if (overrides.tuiApiUnhealthy) { + throw new Error("API is not healthy at http://127.0.0.1:8787/healthz"); + } + }, + async startTui() { + calls.push("start-tui"); + }, + async installDataLink() { + calls.push("install-datalink"); + }, + + async verifyPorts() { + calls.push("verify-ports-again"); + if (overrides.portConflictAfterBuild) { + throw new Error("Web port 3000 is already in use"); + } + }, + async start() { + calls.push("start"); + running = true; + statusState = { + process: "running", + apiHealth: "healthy", + apiReady: "ready", + web: "reachable", + datalinkRest: "disabled", + datalinkMcp: "disabled", + ok: true + }; + }, + async waitForHealth() { + calls.push("wait-for-health"); + }, + async markHealthy() { + calls.push("mark-healthy"); + }, + async status() { + calls.push("status"); + return statusState; + }, + async logs() { + calls.push("logs"); + }, + async doctor() { + calls.push("doctor"); + }, + async printHelp() { + calls.push("help"); + }, + async reportFailure(failure) { + calls.push("report-failure"); + deps.lastFailure = failure; + }, + ...overrides.deps + }; + return deps; +} + +test("fresh deploy runs configure through healthy state", async () => { + const deps = createFakeDeployDeps({ running: false }); + const controller = createDeploymentController(deps); + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + assert.ok(result.ok); + assert.deepEqual(result.calls, [ + "load-config", + "preflight", + "configure", + "check-dependencies", + "select-ports", + "write-config", + "install", + "build-typescript", + "build-web", + "build-tui", + "verify-ports-again", + "start", + "wait-for-health", + "mark-healthy" + ]); + assert.ok(!result.calls.includes("stop-old")); + assert.ok(!result.calls.includes("install-datalink")); + assert.ok(!result.calls.includes("start-tui")); +}); + +test("update deploy stops only after preflight and before install/build", async () => { + const deps = createFakeDeployDeps({ running: true }); + const controller = createDeploymentController(deps); + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + const stopIndex = result.calls.indexOf("stop-old"); + const installIndex = result.calls.indexOf("install"); + assert.ok(stopIndex > result.calls.indexOf("preflight")); + assert.ok(stopIndex > result.calls.indexOf("write-config")); + assert.ok(installIndex > stopIndex); + assert.ok(result.calls.indexOf("build-typescript") > stopIndex); +}); + +test("build failure keeps storage intact and reports maintenance window", async () => { + const deps = createFakeDeployDeps({ running: true, failWebBuild: true }); + const controller = createDeploymentController(deps); + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /Web build failed/ + ); + assert.equal(deps.storage.sentinel.toString(), "storage-sentinel-bytes"); + assert.equal(deps.lastFailure.maintenanceWindow, true); + assert.equal(deps.lastFailure.oldServiceRunning, false); + assert.equal(deps.lastFailure.configurationChanged, true); + assert.equal(deps.lastFailure.stage, "build-web"); + assert.equal(deps.lastFailure.retryCommand, "./deploy.sh deploy"); + assert.equal(deps.lastFailure.doctorCommand, "./deploy.sh doctor"); + assert.ok(!deps.calls.includes("start")); + const formatted = formatDeploymentFailure({ + ...deps.lastFailure, + summary: "Web build failed" + }); + assert.match(formatted, /现有数据未被修改/); + assert.match(formatted, /维护窗口/); + assert.doesNotMatch(formatted, /已恢复|rolled back|restored/i); +}); + +test("port conflict after build prevents start and never kills listeners", async () => { + const deps = createFakeDeployDeps({ running: false, portConflictAfterBuild: true }); + let killed = false; + deps.killListener = () => { + killed = true; + }; + const controller = createDeploymentController(deps); + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /already in use/ + ); + assert.ok(!deps.calls.includes("start")); + assert.equal(killed, false); +}); + +test("start with missing build artifact fails with deploy hint", async () => { + const deps = createFakeDeployDeps({ missingBuild: true }); + const controller = createDeploymentController(deps); + await assert.rejects(() => controller.run({ command: "start" }), /请先运行 \.\/deploy\.sh deploy/); + assert.ok(!deps.calls.includes("install")); + assert.ok(!deps.calls.includes("build-typescript")); +}); + +test("start with incomplete config fails with deploy hint and never starts", async () => { + const deps = createFakeDeployDeps(); + deps.preflight = async (_config, options) => { + deps.calls.push("preflight"); + if (options?.command === "start") { + throw new Error("缺少合法完整的 .env 配置，请先运行 ./deploy.sh deploy"); + } + }; + const controller = createDeploymentController(deps); + await assert.rejects(() => controller.run({ command: "start" }), /缺少合法完整的 \.env|请先运行 \.\/deploy\.sh deploy/); + assert.ok(!deps.calls.includes("start")); +}); + +test("status distinguishes stale, unhealthy, unreachable, and disabled DataLink", async () => { + const deps = createFakeDeployDeps(); + deps.status = async () => ({ + process: "stopped", + apiHealth: "unhealthy", + apiReady: "unhealthy", + web: "unreachable", + datalinkRest: "disabled", + datalinkMcp: "disabled", + ok: false + }); + const controller = createDeploymentController(deps); + const summary = await controller.run({ command: "status" }); + assert.equal(summary.process, "stopped"); + assert.equal(summary.apiHealth, "unhealthy"); + assert.equal(summary.web, "unreachable"); + assert.equal(summary.datalinkRest, "disabled"); +}); + +test("restart does not install or build", async () => { + const deps = createFakeDeployDeps({ running: true }); + const controller = createDeploymentController(deps); + await controller.run({ command: "restart" }); + assert.ok(deps.calls.includes("stop")); + assert.ok(deps.calls.includes("start")); + assert.ok(!deps.calls.includes("install")); + assert.ok(!deps.calls.includes("build-typescript")); + assert.ok(!deps.calls.includes("build-web")); +}); + +test("stop twice succeeds both times", async () => { + const deps = createFakeDeployDeps({ running: true }); + const controller = createDeploymentController(deps); + await controller.run({ command: "stop" }); + await controller.run({ command: "stop" }); + assert.equal(deps.calls.filter((call) => call === "stop").length, 2); +}); + +test("runtime and deploy logs redact fixture secrets", () => { + const text = redactSensitiveText( + "AUTH_SESSION_SECRET=fixture-deploy-secret-at-least-32-chars\nLLM_API_KEY=sk-test\nWEB_PORT=3000\n" + ); + assert.doesNotMatch(text, /fixture-deploy-secret-at-least-32-chars/); + assert.doesNotMatch(text, /sk-test/); + assert.match(text, /WEB_PORT=3000/); +}); + +test("deploy failure uses beginDeployLog path", async () => { + let failure; + const controller = createDeploymentController({ + async beginDeployLog() { + return "storage/logs/deploy-20260722143000.log"; + }, + async loadConfiguration() { + return { env: { DATALINK_ENABLED: "false" } }; + }, + async preflight() {}, + async configure(config) { + return config; + }, + async checkDependencies() {}, + async selectPorts(config) { + return config; + }, + async writeConfiguration() {}, + async isRunning() { + return false; + }, + async stop() {}, + async installProject() {}, + async buildTypeScript() { + throw new Error("TypeScript build failed"); + }, + async buildWeb() {}, + async buildTui() {}, + async installDataLink() {}, + async verifyPorts() {}, + async start() {}, + async waitForHealth() {}, + async markHealthy() {}, + async reportFailure(details) { + failure = details; + } + }); + + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /TypeScript build failed/ + ); + assert.equal(failure.logPath, "storage/logs/deploy-20260722143000.log"); +}); + +test("fresh deploy with DataLink installs datalink after builds", async () => { + const deps = createFakeDeployDeps({ running: false }); + deps.loadConfiguration = async () => { + deps.calls.push("load-config"); + return { env: { DATALINK_ENABLED: "true" }, envText: "", webText: "" }; + }; + const controller = createDeploymentController(deps); + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + assert.ok(result.calls.includes("install-datalink")); + assert.ok(result.calls.indexOf("install-datalink") > result.calls.indexOf("build-tui")); + assert.ok(result.calls.indexOf("build-tui") > result.calls.indexOf("build-web")); +}); + +test("deploy never auto-starts TUI client", async () => { + const deps = createFakeDeployDeps({ running: false }); + const controller = createDeploymentController(deps); + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + assert.ok(result.calls.includes("build-tui")); + assert.ok(!result.calls.includes("ensure-tui-ready")); + assert.ok(!result.calls.includes("check-api-for-tui")); + assert.ok(!result.calls.includes("start-tui")); +}); + +test("optional tui command requires healthy API before start", async () => { + const deps = createFakeDeployDeps({ tuiApiUnhealthy: true }); + const controller = createDeploymentController(deps); + await assert.rejects(() => controller.run({ command: "tui", runtimeUrl: null }), /API is not healthy/); + assert.ok(deps.calls.includes("ensure-tui-ready")); + assert.ok(deps.calls.includes("check-api-for-tui")); + assert.ok(!deps.calls.includes("start-tui")); +}); diff --git a/scripts/deploy/ports.mjs b/scripts/deploy/ports.mjs new file mode 100644 index 00000000..f36b840b --- /dev/null +++ b/scripts/deploy/ports.mjs @@ -0,0 +1,127 @@ +import { execFile } from "node:child_process"; +import net from "node:net"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export function parsePort(value, label) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) { + throw new Error(`${label} port must be an integer between 1 and 65535`); + } + return parsed; +} + +export async function describePortOwner(port) { + if (process.platform !== "linux") return "unknown process"; + try { + const { stdout } = await execFileAsync("ss", ["-ltnp"], { + encoding: "utf8", + maxBuffer: 2 * 1024 * 1024 + }); + const needle = `:${port}`; + for (const line of stdout.split(/\r?\n/u)) { + if (!line.includes(needle) || !/\bLISTEN\b/u.test(line)) continue; + const users = /users:\(\("([^"]+)",pid=(\d+)/u.exec(line); + if (users) return `${users[1]} pid=${users[2]}`; + return "unknown process"; + } + } catch { + // best effort + } + return "unknown process"; +} + +export async function probePort(host, port, options = {}) { + const describe = options.describeOwner ?? describePortOwner; + const available = await new Promise((resolve) => { + const server = net.createServer(); + server.once("error", () => resolve(false)); + server.listen({ host, port, exclusive: true }, () => { + server.close(() => resolve(true)); + }); + }); + if (available) return { available: true, owner: null }; + return { available: false, owner: await describe(port) }; +} + +function defaultPrint(message) { + process.stdout.write(`${message}\n`); +} + +export async function selectDeploymentPort(options) { + const { + label, + defaultPort, + reserved = new Set(), + managedPorts = new Set(), + nonInteractive = false, + ask, + probe = (port) => probePort("0.0.0.0", port), + print = defaultPrint + } = options; + + let candidate = parsePort(defaultPort, label); + let explicitChoice = false; + + while (true) { + if (reserved.has(candidate)) { + const message = `${label} port ${candidate} is already selected for another service`; + if (nonInteractive) throw new Error(message); + print(message); + const answer = String(await ask(`请为 ${label} 输入其他端口，或输入 q 退出：`)).trim(); + if (answer.toLowerCase() === "q") throw new Error("port selection cancelled"); + candidate = parsePort(answer, label); + explicitChoice = true; + continue; + } + + const result = await probe(candidate); + const managed = managedPorts.has(candidate); + + if (result.available || managed) { + if (nonInteractive || explicitChoice) return candidate; + + print(`端口 ${candidate} 当前可用，请选择：`); + print(`1. 使用端口 ${candidate}`); + print("2. 指定其他端口"); + const choice = String((await ask("请选择 [1]：")) ?? "").trim() || "1"; + if (choice === "1") return candidate; + if (choice === "2") { + const answer = String(await ask(`请输入 ${label} 端口：`)).trim(); + candidate = parsePort(answer, label); + explicitChoice = true; + continue; + } + print("请输入 1 或 2"); + continue; + } + + if (nonInteractive) { + throw new Error(`${label} port ${candidate} is already in use`); + } + + const owner = result.owner || "unknown process"; + print(`端口 ${candidate} 已被未知进程占用（${owner}）。DataFoundry 不会结束该进程。`); + const answer = String(await ask("请输入其他端口，或输入 q 退出：")).trim(); + if (answer.toLowerCase() === "q") throw new Error("port selection cancelled"); + candidate = parsePort(answer, label); + explicitChoice = true; + } +} + +export async function verifySelectedPorts(services, options = {}) { + const { + managedPorts = new Set(), + probe = (port) => probePort("0.0.0.0", port) + } = options; + + for (const service of services) { + const port = parsePort(service.port, service.label); + if (managedPorts.has(port)) continue; + const result = await probe(port); + if (!result.available) { + throw new Error(`${service.label} port ${port} is already in use`); + } + } +} diff --git a/scripts/deploy/ports.test.mjs b/scripts/deploy/ports.test.mjs new file mode 100644 index 00000000..49bd3a3f --- /dev/null +++ b/scripts/deploy/ports.test.mjs @@ -0,0 +1,136 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + parsePort, + selectDeploymentPort, + verifySelectedPorts +} from "./ports.mjs"; + +test("parsePort accepts valid integers", () => { + assert.equal(parsePort("3000", "Web"), 3000); + assert.equal(parsePort(8787, "API"), 8787); +}); + +test("parsePort rejects invalid values", () => { + assert.throws(() => parsePort("0", "Web"), /Web/); + assert.throws(() => parsePort("70000", "API"), /API/); + assert.throws(() => parsePort("abc", "Web"), /Web/); +}); + +test("loops until the requested alternative is available", async () => { + const answers = ["2", "3001", "3002"]; + const selected = await selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set([8787]), + managedPorts: new Set(), + ask: async () => answers.shift(), + probe: async (port) => ({ available: port === 3002, owner: port === 3001 ? "node pid=42" : null }) + }); + assert.equal(selected, 3002); +}); + +test("accepts available default without choosing alternative", async () => { + const selected = await selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set(), + managedPorts: new Set(), + ask: async () => "1", + probe: async () => ({ available: true, owner: null }), + print: () => {} + }); + assert.equal(selected, 3000); +}); + +test("rejects duplicate reserved ports", async () => { + await assert.rejects( + selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set([3000]), + managedPorts: new Set(), + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + probe: async () => ({ available: true, owner: null }) + }), + /already selected|reserved|冲突|duplicate/i + ); +}); + +test("non-interactive mode fails on an unknown listener", async () => { + await assert.rejects( + selectDeploymentPort({ + label: "API", + defaultPort: 8787, + reserved: new Set(), + managedPorts: new Set(), + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + probe: async () => ({ available: false, owner: "python pid=99" }) + }), + /API port 8787 is already in use/ + ); +}); + +test("quitting from occupied-port prompt exits safely", async () => { + await assert.rejects( + selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set(), + managedPorts: new Set(), + ask: async () => "q", + probe: async () => ({ available: false, owner: "node pid=42" }), + print: () => {} + }), + /cancelled|退出|abort/i + ); +}); + +test("managed ports may be selected during update preflight", async () => { + const selected = await selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set(), + managedPorts: new Set([3000]), + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + probe: async () => ({ available: false, owner: "datafoundry pid=1" }) + }); + assert.equal(selected, 3000); +}); + +test("verifySelectedPorts fails when a port becomes occupied", async () => { + await assert.rejects( + verifySelectedPorts( + [{ label: "Web", port: 3000 }], + { + managedPorts: new Set(), + probe: async () => ({ available: false, owner: "nginx pid=7" }) + } + ), + /Web port 3000 is already in use/ + ); +}); + +test("verifySelectedPorts skips ports still held by the managed stack", async () => { + let probed = false; + await verifySelectedPorts( + [ + { label: "Web", port: 3000 }, + { label: "API", port: 8787 } + ], + { + managedPorts: new Set([3000]), + probe: async (port) => { + if (port === 3000) { + probed = true; + return { available: false, owner: "datafoundry pid=1" }; + } + return { available: true, owner: null }; + } + } + ); + assert.equal(probed, false); +}); diff --git a/scripts/deploy/process-state.mjs b/scripts/deploy/process-state.mjs new file mode 100644 index 00000000..fcbb811a --- /dev/null +++ b/scripts/deploy/process-state.mjs @@ -0,0 +1,242 @@ +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const SENSITIVE_KEY = /KEY|SECRET|TOKEN|PASSWORD|COOKIE|AUTHORIZATION/i; + +export function deploymentPaths(root) { + return { + logsDir: path.join(root, "storage/logs"), + runDir: path.join(root, "storage/run"), + runtimeLog: path.join(root, "storage/logs/datafoundry.log"), + pidFile: path.join(root, "storage/run/datafoundry.pid"), + deploymentJson: path.join(root, "storage/run/deployment.json") + }; +} + +export function validateDeploymentState(state) { + if (!state || typeof state !== "object" || Array.isArray(state)) { + throw new Error("deployment state must be an object"); + } + for (const key of Object.keys(state)) { + if (SENSITIVE_KEY.test(key)) { + throw new Error(`sensitive field ${key}`); + } + } + return state; +} + +async function writeAtomicJson(filePath, value) { + await mkdir(path.dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.${process.pid}.tmp`; + await writeFile(tempPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); + await rename(tempPath, filePath); +} + +export async function writeDeploymentState(root, state) { + validateDeploymentState(state); + const paths = deploymentPaths(root); + await mkdir(paths.runDir, { recursive: true }); + await writeAtomicJson(paths.deploymentJson, state); + await writeFile(paths.pidFile, `${state.pid}\n`, { encoding: "utf8", mode: 0o600 }); +} + +export async function readDeploymentState(root) { + const filePath = deploymentPaths(root).deploymentJson; + try { + const raw = await readFile(filePath, "utf8"); + return validateDeploymentState(JSON.parse(raw)); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +export function isProcessAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +export async function rotateRuntimeLog(logPath, options = {}) { + const maxBytes = options.maxBytes ?? 20 * 1024 * 1024; + const retain = options.retain ?? 5; + await mkdir(path.dirname(logPath), { recursive: true }); + try { + const info = await stat(logPath); + if (info.size <= maxBytes) return; + } catch (error) { + if (error?.code === "ENOENT") { + await writeFile(logPath, "", { encoding: "utf8", mode: 0o600 }); + return; + } + throw error; + } + + await rm(`${logPath}.${retain}`, { force: true }); + for (let i = retain - 1; i >= 1; i -= 1) { + const from = `${logPath}.${i}`; + const to = `${logPath}.${i + 1}`; + if (existsSync(from)) await rename(from, to); + } + await rename(logPath, `${logPath}.1`); + await writeFile(logPath, "", { encoding: "utf8", mode: 0o600 }); +} + +async function readLaunchIdFromProcAsync(pid) { + try { + const environ = await readFile(`/proc/${pid}/environ`); + const match = /DATAFOUNDRY_LAUNCH_ID=([^\0]+)/.exec(environ.toString("utf8")); + return match?.[1] ?? null; + } catch { + return null; + } +} + +export async function verifyManagedProcessForStop(pid, expectedLaunchId, options = {}) { + if (process.platform !== "linux") { + return { allowed: true }; + } + + if (!expectedLaunchId) { + return { allowed: false, reason: "missing-expected-launch-id" }; + } + + const readLaunchId = options.readLaunchId ?? readLaunchIdFromProcAsync; + const launchId = await readLaunchId(pid); + + if (launchId === expectedLaunchId) { + return { allowed: true }; + } + if (launchId && launchId !== expectedLaunchId) { + return { allowed: false, reason: "launch-id-mismatch" }; + } + + // Never fall back to cmdline heuristics — an unverifiable launch marker must refuse stop. + return { allowed: false, reason: "launch-id-unverified" }; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function startManagedStack(root, options = {}) { + const existing = await readDeploymentState(root); + if (existing?.pid && isProcessAlive(existing.pid)) { + throw new Error(`DataFoundry is already running with pid ${existing.pid}`); + } + + const paths = deploymentPaths(root); + await mkdir(paths.logsDir, { recursive: true }); + await mkdir(paths.runDir, { recursive: true }); + await rotateRuntimeLog(paths.runtimeLog); + + const launchId = options.launchId ?? randomBytes(16).toString("hex"); + const command = options.command ?? "npm"; + const args = options.args ?? ["run", "start"]; + const env = { + ...(options.env ?? process.env), + DATAFOUNDRY_LAUNCH_ID: launchId + }; + + const logFd = await open(paths.runtimeLog, "a", 0o600); + let child; + try { + child = spawn(command, args, { + cwd: root, + env, + detached: true, + stdio: ["ignore", logFd.fd, logFd.fd] + }); + } finally { + await logFd.close(); + } + + child.unref(); + + const state = { + pid: child.pid, + pgid: child.pid, + launchId, + status: "starting", + startedAt: new Date().toISOString(), + commitSha: options.commitSha ?? null, + ports: options.ports ?? null, + datalinkEnabled: Boolean(options.datalinkEnabled) + }; + await writeDeploymentState(root, state); + return state; +} + +export async function stopManagedStack(root, options = {}) { + const state = await readDeploymentState(root); + if (!state?.pid) return { stopped: false, reason: "not-running" }; + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: false, reason: "stale" }; + } + + if (process.platform === "linux") { + const verification = await verifyManagedProcessForStop(state.pid, state.launchId); + if (!verification.allowed) { + throw new Error( + `Refusing to signal pid ${state.pid}: launch marker does not identify DataFoundry` + ); + } + } + + try { + process.kill(-state.pid, "SIGTERM"); + } catch { + process.kill(state.pid, "SIGTERM"); + } + + const timeoutMs = options.timeoutMs ?? 15_000; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: true, reason: "terminated" }; + } + await sleep(100); + } + + try { + process.kill(-state.pid, "SIGKILL"); + } catch { + try { + process.kill(state.pid, "SIGKILL"); + } catch { + // process may have exited between checks + } + } + + const killTimeoutMs = options.killTimeoutMs ?? 5_000; + const killDeadline = Date.now() + killTimeoutMs; + while (Date.now() < killDeadline) { + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: true, reason: "killed" }; + } + await sleep(100); + } + + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: true, reason: "killed" }; + } + + throw new Error(`Timed out waiting for pid ${state.pid} to exit after SIGKILL`); +} + +async function clearDeploymentState(root) { + const paths = deploymentPaths(root); + await rm(paths.deploymentJson, { force: true }); + await rm(paths.pidFile, { force: true }); +} diff --git a/scripts/deploy/process-state.test.mjs b/scripts/deploy/process-state.test.mjs new file mode 100644 index 00000000..254764c2 --- /dev/null +++ b/scripts/deploy/process-state.test.mjs @@ -0,0 +1,160 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, stat, writeFile, mkdir } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + deploymentPaths, + isProcessAlive, + readDeploymentState, + rotateRuntimeLog, + startManagedStack, + stopManagedStack, + validateDeploymentState, + verifyManagedProcessForStop, + writeDeploymentState +} from "./process-state.mjs"; + +test("validateDeploymentState rejects sensitive fields", () => { + assert.throws( + () => validateDeploymentState({ pid: 123, API_KEY: "must-not-leak" }), + /sensitive field API_KEY/ + ); +}); + +test("writeDeploymentState persists non-sensitive state atomically", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-state-")); + const state = { + pid: 12345, + launchId: "launch-1", + status: "starting", + startedAt: "2026-07-22T00:00:00.000Z", + ports: { web: 3000, api: 8787 }, + datalinkEnabled: false + }; + await writeDeploymentState(root, state); + assert.deepEqual(await readDeploymentState(root), state); + const raw = await readFile(deploymentPaths(root).deploymentJson, "utf8"); + assert.doesNotMatch(raw, /SECRET|TOKEN|PASSWORD|KEY=/i); +}); + +test("isProcessAlive detects live and stale pids", () => { + assert.equal(isProcessAlive(process.pid), true); + assert.equal(isProcessAlive(99999999), false); +}); + +test("rotateRuntimeLog keeps five archives at 20 MiB", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-log-")); + const logPath = path.join(root, "datafoundry.log"); + await writeFile(logPath, "x".repeat(20 * 1024 * 1024 + 1)); + for (let i = 1; i <= 5; i += 1) { + await writeFile(`${logPath}.${i}`, `old-${i}`); + } + await rotateRuntimeLog(logPath, { maxBytes: 20 * 1024 * 1024, retain: 5 }); + await assert.rejects(() => readFile(`${logPath}.6`, "utf8"), /ENOENT/); + assert.match(await readFile(`${logPath}.1`, "utf8"), /x/); + assert.equal(await readFile(logPath, "utf8"), ""); +}); + +test("start and stop managed stack are idempotent and refuse foreign pids", async (t) => { + if (process.platform === "win32") { + t.skip("process-group semantics are Unix-only"); + return; + } + + const root = await mkdtemp(path.join(os.tmpdir(), "df-proc-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + + const started = await startManagedStack(root, { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"], + env: { ...process.env }, + ports: { web: 3000, api: 8787 }, + datalinkEnabled: false, + commitSha: "deadbeef" + }); + assert.equal(isProcessAlive(started.pid), true); + assert.equal((await readDeploymentState(root)).status, "starting"); + assert.equal((await readDeploymentState(root)).launchId, started.launchId); + const runtimeLogMode = (await stat(deploymentPaths(root).runtimeLog)).mode & 0o777; + assert.equal(runtimeLogMode, 0o600); + + await assert.rejects( + () => startManagedStack(root, { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"] + }), + /already running/i + ); + + const foreignRoot = await mkdtemp(path.join(os.tmpdir(), "df-foreign-")); + await writeDeploymentState(foreignRoot, { + pid: process.pid, + launchId: "not-this-process", + status: "running", + startedAt: new Date().toISOString(), + ports: { web: 3000, api: 8787 }, + datalinkEnabled: false + }); + await assert.rejects(() => stopManagedStack(foreignRoot), /does not identify DataFoundry|launch marker/i); + + await stopManagedStack(root); + assert.equal(isProcessAlive(started.pid), false); + assert.equal(await readDeploymentState(root), null); + + await stopManagedStack(root); +}); + +test("verifyManagedProcessForStop refuses stop when launch id is unreadable", async (t) => { + if (process.platform !== "linux") { + t.skip("proc verification is Linux-only"); + return; + } + + const unverified = await verifyManagedProcessForStop(12345, "launch-1", { + readLaunchId: async () => null + }); + assert.equal(unverified.allowed, false); + assert.equal(unverified.reason, "launch-id-unverified"); + + const mismatch = await verifyManagedProcessForStop(process.pid, "expected-launch", { + readLaunchId: async () => "other-launch" + }); + assert.equal(mismatch.allowed, false); + assert.equal(mismatch.reason, "launch-id-mismatch"); + + const matched = await verifyManagedProcessForStop(12345, "launch-1", { + readLaunchId: async () => "launch-1" + }); + assert.equal(matched.allowed, true); + + const missingExpected = await verifyManagedProcessForStop(12345, "", { + readLaunchId: async () => "launch-1" + }); + assert.equal(missingExpected.allowed, false); + assert.equal(missingExpected.reason, "missing-expected-launch-id"); +}); + +test("stopManagedStack SIGKILLs and clears state when SIGTERM times out", async (t) => { + if (process.platform === "win32") { + t.skip("process-group semantics are Unix-only"); + return; + } + + const root = await mkdtemp(path.join(os.tmpdir(), "df-kill-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + + const started = await startManagedStack(root, { + command: "bash", + args: ["-c", "trap '' TERM; while true; do sleep 1; done"], + env: { ...process.env }, + ports: { web: 3000, api: 8787 }, + datalinkEnabled: false + }); + assert.equal(isProcessAlive(started.pid), true); + + const result = await stopManagedStack(root, { timeoutMs: 200, killTimeoutMs: 5_000 }); + assert.equal(result.stopped, true); + assert.equal(isProcessAlive(started.pid), false); + assert.equal(await readDeploymentState(root), null); +}); diff --git a/scripts/deploy/smoke-native-deploy.mjs b/scripts/deploy/smoke-native-deploy.mjs new file mode 100644 index 00000000..e5f65b5d --- /dev/null +++ b/scripts/deploy/smoke-native-deploy.mjs @@ -0,0 +1,226 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createServer } from "node:net"; +import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createDeployLogWriter } from "./cli.mjs"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const FIXTURE_SECRET = "fixture-deploy-secret-at-least-32-chars"; + +async function reservePort() { + return new Promise((resolve, reject) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : null; + server.close((error) => { + if (error) reject(error); + else resolve(port); + }); + }); + server.on("error", reject); + }); +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + encoding: "utf8", + timeout: options.timeout ?? 20 * 60_000 + }); + if (result.status !== 0) { + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + throw new Error(`${command} ${args.join(" ")} failed (${result.status}):\n${output}`); + } + return result; +} + +async function httpGet(url) { + const response = await fetch(url, { redirect: "manual" }); + return { status: response.status, ok: response.status >= 200 && response.status < 400 }; +} + +function shouldCopy(source) { + const relative = path.relative(ROOT, source); + if (!relative || relative.startsWith("..")) return true; + const parts = relative.split(path.sep); + const blocked = new Set([ + ".git", + "node_modules", + "storage", + ".next", + "dist", + ".venv", + "venv", + ".worktrees", + "coverage" + ]); + if (parts.some((part) => blocked.has(part))) return false; + if (relative === ".env" || relative === path.join("apps", "web", ".env.local")) return false; + // Stale tsbuildinfo from the source tree can make tsc skip emit in the temp checkout (no dist/). + if (path.basename(source).endsWith(".tsbuildinfo")) return false; + return true; +} + +async function copyCheckout(destination) { + await cp(ROOT, destination, { + recursive: true, + filter: (source) => shouldCopy(source) + }); + await mkdir(path.join(destination, "storage"), { recursive: true }); + await writeFile(path.join(destination, "storage", "sentinel.txt"), "native-deploy-sentinel\n"); +} + +function scanForSecrets(text) { + if (text.includes(FIXTURE_SECRET)) { + throw new Error("Fixture secret leaked into logs or status output"); + } +} + +async function preserveFailureLogs(tempRoot) { + const sourceDir = path.join(tempRoot, "storage/logs"); + const targetDir = path.join(ROOT, "storage/logs"); + try { + const entries = await readdir(sourceDir); + await mkdir(targetDir, { recursive: true }); + for (const entry of entries) { + const content = await readFile(path.join(sourceDir, entry), "utf8"); + scanForSecrets(content); + await writeFile(path.join(targetDir, entry), content, { encoding: "utf8", mode: 0o600 }); + } + } catch (error) { + if (error?.code !== "ENOENT") { + console.error(`[smoke:native-deploy] failed to preserve logs: ${error.message}`); + } + } +} + +async function main() { + const webPort = await reservePort(); + const apiPort = await reservePort(); + const tempRoot = await mkdtemp(path.join(os.tmpdir(), "datafoundry-native-deploy-")); + let stopped = false; + + const env = { + ...process.env, + WEB_HOST: "127.0.0.1", + WEB_PORT: String(webPort), + API_HOST: "127.0.0.1", + API_PORT: String(apiPort), + AUTH_PUBLIC_BASE_URL: `http://127.0.0.1:${webPort}`, + AUTH_SESSION_SECRET: FIXTURE_SECRET, + SECRET_MASTER_KEY: FIXTURE_SECRET, + DATALINK_ENABLED: "false", + CI: "true", + SKIP_POSTINSTALL_BUILD: "1" + }; + + try { + console.log(`[smoke:native-deploy] checkout=${tempRoot}`); + console.log(`[smoke:native-deploy] web=${webPort} api=${apiPort}`); + await copyCheckout(tempRoot); + + // Inject fixture secret into the same redacting deploy-log path used in production. + const redactProbe = await createDeployLogWriter(tempRoot, { timestamp: "smoke-redact-probe" }); + await redactProbe.append( + [ + `AUTH_SESSION_SECRET=${FIXTURE_SECRET}`, + `Authorization: Bearer ${FIXTURE_SECRET}`, + `{"apiKey":"${FIXTURE_SECRET}"}`, + `https://user:${FIXTURE_SECRET}@example.com/callback`, + "npm ci output", + "" + ].join("\n") + ); + await redactProbe.finalize(); + const probeLog = await readFile( + path.join(tempRoot, "storage/logs/deploy-smoke-redact-probe.log"), + "utf8" + ); + scanForSecrets(probeLog); + assert.match(probeLog, /AUTH_SESSION_SECRET=\*+/); + assert.match(probeLog, /Authorization: Bearer \*+/i); + + run("bash", ["./deploy.sh", "deploy", "--non-interactive"], { + cwd: tempRoot, + env, + timeout: 25 * 60_000 + }); + + const status = run("bash", ["./deploy.sh", "status"], { cwd: tempRoot, env }); + scanForSecrets(`${status.stdout}\n${status.stderr}`); + assert.match(status.stdout, /进程\s+running/); + assert.doesNotMatch(status.stdout, /DataLink\s+healthy/); + + const healthz = await httpGet(`http://127.0.0.1:${apiPort}/healthz`); + const ready = await httpGet(`http://127.0.0.1:${apiPort}/ready`); + const web = await httpGet(`http://127.0.0.1:${webPort}/`); + assert.equal(healthz.status, 200); + assert.equal(ready.status, 200); + assert.ok(web.ok, `Web should be reachable, got ${web.status}`); + + const envText = await readFile(path.join(tempRoot, ".env"), "utf8"); + assert.match(envText, /DATALINK_ENABLED=false/); + assert.match(envText, new RegExp(`^AUTH_SESSION_SECRET=${FIXTURE_SECRET}$`, "m")); + assert.doesNotMatch(envText, /^LLM_API_KEY=.+$/m); + assert.doesNotMatch(envText, /^LLM_MODEL=.+$/m); + + run("bash", ["./deploy.sh", "restart"], { cwd: tempRoot, env, timeout: 10 * 60_000 }); + const statusAfterRestart = run("bash", ["./deploy.sh", "status"], { cwd: tempRoot, env }); + scanForSecrets(`${statusAfterRestart.stdout}\n${statusAfterRestart.stderr}`); + assert.match(statusAfterRestart.stdout, /进程\s+running/); + + const sentinel = await readFile(path.join(tempRoot, "storage", "sentinel.txt"), "utf8"); + assert.equal(sentinel, "native-deploy-sentinel\n"); + + run("bash", ["./deploy.sh", "stop"], { cwd: tempRoot, env }); + run("bash", ["./deploy.sh", "stop"], { cwd: tempRoot, env }); + stopped = true; + + const finalStatus = run("bash", ["./deploy.sh", "status"], { cwd: tempRoot, env }); + scanForSecrets(`${finalStatus.stdout}\n${finalStatus.stderr}`); + assert.match(finalStatus.stdout, /进程\s+stopped/); + assert.equal( + await readFile(path.join(tempRoot, "storage", "sentinel.txt"), "utf8"), + "native-deploy-sentinel\n" + ); + + for (const relative of [ + "storage/logs/datafoundry.log", + "storage/logs/deploy-latest.log", + "storage/logs/deploy-smoke-redact-probe.log" + ]) { + try { + const logs = await readFile(path.join(tempRoot, relative), "utf8"); + scanForSecrets(logs); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + } + + console.log("[smoke:native-deploy] ok"); + } catch (error) { + await preserveFailureLogs(tempRoot); + throw error; + } finally { + if (!stopped) { + try { + spawnSync("bash", ["./deploy.sh", "stop"], { + cwd: tempRoot, + env, + encoding: "utf8", + timeout: 60_000 + }); + } catch { + // best effort + } + } + await rm(tempRoot, { recursive: true, force: true }); + } +} + +await main(); diff --git a/scripts/dev.mjs b/scripts/dev.mjs index 995bccfb..4bfe109f 100755 --- a/scripts/dev.mjs +++ b/scripts/dev.mjs @@ -1,119 +1,4 @@ #!/usr/bin/env node -/** - * Build workspace packages and start API + web dev servers (Linux, Windows, macOS). - * - * Usage: - * npm run dev # start both - * npm run dev -- --api # API only - * npm run dev -- --web # web only - */ -import { spawn, execSync } from "node:child_process"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { runStack } from "./stack-runner.mjs"; -const root = join(dirname(fileURLToPath(import.meta.url)), ".."); -const args = process.argv.slice(2); -const apiOnly = args.includes("--api"); -const webOnly = args.includes("--web"); -const startApi = !webOnly || apiOnly; -const startWeb = !apiOnly || webOnly; - -execSync("node scripts/ensure-dev-environment.mjs", { - cwd: root, - stdio: "inherit", - env: process.env, - shell: true, -}); - -for (const port of [8787, 3000]) { - try { - freePort(port); - } catch { - // Port may already be free; dev servers will fail loudly if it stays busy. - } -} - -/** @type {import('node:child_process').ChildProcess[]} */ -const children = []; - -if (startApi) { - children.push(spawnNpm(["--workspace", "@datafoundry/api", "run", "dev"])); -} -if (startWeb) { - children.push(spawnNpm(["--workspace", "@datafoundry/web", "run", "dev"])); -} - -if (children.length === 0) { - console.error("Nothing to start. Use --api and/or --web."); - process.exit(1); -} - -console.log( - "\n[dev] " + - (startApi ? "API → http://127.0.0.1:8787 " : "") + - (startWeb ? "Web → http://localhost:3000/data-tasks" : "") + - "\n", -); - -function shutdown(signal) { - for (const child of children) { - if (!child.killed) child.kill(signal); - } -} - -process.on("SIGINT", () => shutdown("SIGINT")); -process.on("SIGTERM", () => shutdown("SIGTERM")); - -for (const child of children) { - child.on("exit", (code, signal) => { - if (signal) return; - if (code && code !== 0) { - shutdown("SIGTERM"); - process.exit(code); - } - }); -} - -function spawnNpm(args) { - return spawn("npm", args, { - cwd: root, - stdio: "inherit", - env: process.env, - shell: true, - }); -} - -function freePort(port) { - if (process.platform === "win32") { - let output = ""; - try { - output = execSync(`netstat -ano | findstr :${port}`, { - encoding: "utf8", - shell: true, - stdio: ["ignore", "pipe", "ignore"], - }); - } catch { - return; - } - - const pids = new Set(); - for (const line of output.split(/\r?\n/u)) { - if (!/\bLISTENING\b/u.test(line)) continue; - const pid = line.trim().split(/\s+/u).at(-1); - if (pid && /^\d+$/u.test(pid) && pid !== "0") { - pids.add(pid); - } - } - - for (const pid of pids) { - execSync(`taskkill /F /PID ${pid}`, { stdio: "ignore", shell: true }); - } - return; - } - - execSync(`fuser -k ${port}/tcp 2>/dev/null || true`, { - cwd: root, - stdio: "ignore", - shell: true, - }); -} +await runStack({ mode: "development", args: process.argv.slice(2) }); diff --git a/scripts/lib/authenticated-test-client.mjs b/scripts/lib/authenticated-test-client.mjs new file mode 100644 index 00000000..3546be94 --- /dev/null +++ b/scripts/lib/authenticated-test-client.mjs @@ -0,0 +1,222 @@ +import { randomUUID } from "node:crypto"; + +const UNSAFE_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); + +export function resolveApiUrl(baseUrl, relativePath) { + const base = new URL(baseUrl); + const prefix = base.pathname.replace(/\/?$/, "/"); + // Preserve query/hash from relativePath; strip only base URL search/hash. + const resolved = new URL(String(relativePath).replace(/^\/+/, ""), `https://resolve.invalid${prefix}`); + base.pathname = resolved.pathname; + base.search = resolved.search; + base.hash = ""; + return base; +} + +function createCookieJar() { + /** @type {Record} */ + const store = Object.create(null); + + return { + replace(cookies) { + for (const key of Object.keys(store)) { + delete store[key]; + } + for (const [name, value] of Object.entries(cookies ?? {})) { + store[name] = String(value); + } + }, + absorbSetCookie(headers) { + const values = + typeof headers.getSetCookie === "function" + ? headers.getSetCookie() + : splitSetCookieHeader(headers.get("set-cookie")); + for (const cookie of values) { + const pair = String(cookie).split(";", 1)[0]; + const eq = pair.indexOf("="); + if (eq <= 0) { + continue; + } + const name = pair.slice(0, eq).trim(); + const value = decodeURIComponent(pair.slice(eq + 1)); + store[name] = value; + } + }, + headerValue() { + const parts = Object.entries(store).map( + ([name, value]) => `${name}=${encodeURIComponent(value)}` + ); + return parts.length > 0 ? parts.join("; ") : undefined; + }, + csrfToken() { + return store.df_csrf; + }, + snapshot() { + return { ...store }; + }, + clear() { + for (const key of Object.keys(store)) { + delete store[key]; + } + } + }; +} + +function splitSetCookieHeader(value) { + if (!value) { + return []; + } + if (Array.isArray(value)) { + return value; + } + return [value]; +} + +function normalizeHeaders(initHeaders) { + if (!initHeaders) { + return new Headers(); + } + return initHeaders instanceof Headers ? new Headers(initHeaders) : new Headers(initHeaders); +} + +function createAuthError(status, code, message) { + const error = new Error(message || `HTTP ${status}`); + error.name = "AuthenticatedTestClientError"; + error.status = status; + error.code = code ?? "HTTP_ERROR"; + return error; +} + +async function readJsonSafe(response) { + const contentType = response.headers.get("content-type") ?? ""; + if (!contentType.includes("application/json")) { + return undefined; + } + try { + return await response.json(); + } catch { + return undefined; + } +} + +export function createAuthenticatedTestClient({ baseUrl, fetchImpl = fetch }) { + const cookies = createCookieJar(); + + async function authenticatedFetch(path, init = {}) { + const { expectOk = false, ...requestInit } = init; + const method = String(requestInit.method ?? "GET").toUpperCase(); + const headers = normalizeHeaders(requestInit.headers); + const cookieHeader = cookies.headerValue(); + if (cookieHeader) { + headers.set("cookie", cookieHeader); + } + if (UNSAFE_METHODS.has(method)) { + const csrf = cookies.csrfToken(); + if (csrf && !headers.has("x-csrf-token")) { + headers.set("x-csrf-token", csrf); + } + } + + const url = resolveApiUrl(baseUrl, path); + const response = await fetchImpl(url, { + ...requestInit, + method, + headers + }); + cookies.absorbSetCookie(response.headers); + + if (expectOk && (response.status < 200 || response.status >= 300)) { + const body = await readJsonSafe(response.clone()); + throw createAuthError( + response.status, + body?.error?.code, + body?.error?.message ?? `Request failed with status ${response.status}` + ); + } + + return response; + } + + async function fetchJson(path, init = {}) { + const response = await authenticatedFetch(path, { ...init, expectOk: true }); + return { + response, + body: await readJsonSafe(response) + }; + } + + async function verifyCurrentUser() { + const { body } = await fetchJson("/api/v1/me"); + const user = body?.data?.user; + const workspace = body?.data?.workspace; + if (!user?.id || !workspace?.id) { + throw createAuthError(500, "INVALID_ME_RESPONSE", "GET /api/v1/me returned incomplete identity."); + } + return { user, workspace }; + } + + async function registerAndLogin(input = {}) { + const email = input.email ?? `${randomUUID()}@example.test`; + const password = input.password ?? `pw-${randomUUID()}`; + const displayName = input.displayName ?? "Authenticated Test User"; + + const registered = await fetchJson("/api/v1/auth/register", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ email, password, displayName }) + }); + const verificationToken = registered.body?.data?.verificationToken; + if (!verificationToken) { + throw createAuthError(500, "MISSING_VERIFICATION_TOKEN", "Register response missing verificationToken."); + } + + await fetchJson("/api/v1/auth/verify-email", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ token: verificationToken }) + }); + + const loggedIn = await fetchJson("/api/v1/auth/login", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + email, + password, + ...(input.client ? { client: input.client } : {}) + }) + }); + + const me = await verifyCurrentUser(); + const userId = me.user.id ?? loggedIn.body?.data?.user?.id; + const workspaceId = me.workspace.id ?? loggedIn.body?.data?.workspace?.id; + if (!userId || !workspaceId) { + throw createAuthError(500, "INVALID_LOGIN_IDENTITY", "Login/me did not return user and workspace ids."); + } + + return { + userId, + workspaceId, + email: me.user.email ?? email, + cookies: cookies.snapshot(), + user: me.user, + workspace: me.workspace + }; + } + + async function logout() { + await authenticatedFetch("/api/v1/auth/logout", { + method: "POST", + expectOk: true + }); + cookies.clear(); + } + + return { + cookies, + fetch: authenticatedFetch, + fetchJson, + registerAndLogin, + verifyCurrentUser, + logout + }; +} diff --git a/scripts/lib/authenticated-test-client.test.mjs b/scripts/lib/authenticated-test-client.test.mjs new file mode 100644 index 00000000..6f90696a --- /dev/null +++ b/scripts/lib/authenticated-test-client.test.mjs @@ -0,0 +1,220 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { randomUUID } from "node:crypto"; + +import { + createAuthenticatedTestClient, + resolveApiUrl +} from "./authenticated-test-client.mjs"; + +test("resolveApiUrl keeps deployment path prefix", () => { + const url = resolveApiUrl("https://example.com/datafoundry", "api/v1/me"); + assert.equal(url.href, "https://example.com/datafoundry/api/v1/me"); +}); + +test("resolveApiUrl strips search and hash from base", () => { + const url = resolveApiUrl("https://example.com/datafoundry?x=1#frag", "/api/v1/me"); + assert.equal(url.href, "https://example.com/datafoundry/api/v1/me"); +}); + +test("resolveApiUrl preserves query from relative path", () => { + const url = resolveApiUrl("https://example.com/datafoundry", "/api/v1/sessions/s1/conversation?limit=10"); + assert.equal(url.href, "https://example.com/datafoundry/api/v1/sessions/s1/conversation?limit=10"); +}); + +test("adds cookie and csrf to unsafe requests", async () => { + const calls = []; + const client = createAuthenticatedTestClient({ + baseUrl: "http://127.0.0.1:8787", + fetchImpl: async (url, init) => { + calls.push({ url, init }); + return new Response("{}", { status: 200 }); + } + }); + client.cookies.replace({ + df_session: "session-secret", + df_csrf: "csrf-secret" + }); + + await client.fetch("/api/v1/config", { method: "POST", body: "{}" }); + + assert.equal(calls[0].init.headers.get("cookie"), "df_session=session-secret; df_csrf=csrf-secret"); + assert.equal(calls[0].init.headers.get("x-csrf-token"), "csrf-secret"); +}); + +test("does not add csrf to safe GET requests", async () => { + const calls = []; + const client = createAuthenticatedTestClient({ + baseUrl: "http://127.0.0.1:8787", + fetchImpl: async (url, init) => { + calls.push({ url, init }); + return new Response("{}", { status: 200 }); + } + }); + client.cookies.replace({ + df_session: "session-secret", + df_csrf: "csrf-secret" + }); + + await client.fetch("/api/v1/me"); + + assert.equal(calls[0].init.headers.get("cookie"), "df_session=session-secret; df_csrf=csrf-secret"); + assert.equal(calls[0].init.headers.get("x-csrf-token"), null); +}); + +test("absorbs Set-Cookie from multiple headers", async () => { + const client = createAuthenticatedTestClient({ + baseUrl: "http://127.0.0.1:8787", + fetchImpl: async () => { + const headers = new Headers(); + headers.append("Set-Cookie", "df_session=session-a; Path=/; HttpOnly"); + headers.append("Set-Cookie", "df_csrf=csrf-a; Path=/"); + return new Response("{}", { status: 200, headers }); + } + }); + + await client.fetch("/api/v1/auth/login", { method: "POST", body: "{}" }); + assert.deepEqual(client.cookies.snapshot(), { + df_session: "session-a", + df_csrf: "csrf-a" + }); +}); + +test("auth errors omit cookie secrets", async () => { + const client = createAuthenticatedTestClient({ + baseUrl: "http://127.0.0.1:8787", + fetchImpl: async () => + new Response(JSON.stringify({ error: { code: "UNAUTHORIZED", message: "nope" } }), { + status: 401, + headers: { "Content-Type": "application/json" } + }) + }); + client.cookies.replace({ + df_session: "session-secret", + df_csrf: "csrf-secret" + }); + + await assert.rejects( + () => client.fetch("/api/v1/me", { expectOk: true }), + (error) => { + const text = String(error); + assert.equal(error.status, 401); + assert.equal(error.code, "UNAUTHORIZED"); + assert.doesNotMatch(text, /session-secret|csrf-secret/); + return true; + } + ); +}); + +test("verifyCurrentUser calls GET /api/v1/me only", async () => { + const calls = []; + const client = createAuthenticatedTestClient({ + baseUrl: "http://127.0.0.1:8787/datafoundry", + fetchImpl: async (url, init) => { + calls.push({ url: String(url), method: init?.method ?? "GET" }); + return new Response( + JSON.stringify({ + data: { + user: { id: "u1", email: "a@example.test" }, + workspace: { id: "w1" } + } + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ); + } + }); + client.cookies.replace({ df_session: "s", df_csrf: "c" }); + + const me = await client.verifyCurrentUser(); + assert.equal(me.user.id, "u1"); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, "http://127.0.0.1:8787/datafoundry/api/v1/me"); + assert.equal(calls[0].method, "GET"); + assert.ok(!calls.some((call) => call.url.includes("/api/v1/auth/me"))); +}); + +test("registerAndLogin walks register verify login and me", async () => { + const calls = []; + const email = `${randomUUID()}@example.test`; + const client = createAuthenticatedTestClient({ + baseUrl: "http://127.0.0.1:8787", + fetchImpl: async (url, init) => { + const href = String(url); + const method = init?.method ?? "GET"; + calls.push({ href, method, body: init?.body }); + if (href.endsWith("/api/v1/auth/register") && method === "POST") { + return new Response( + JSON.stringify({ + data: { + verificationToken: "verify-token", + user: { id: "user-1", email } + } + }), + { + status: 201, + headers: { "Content-Type": "application/json" } + } + ); + } + if (href.endsWith("/api/v1/auth/verify-email") && method === "POST") { + return new Response(JSON.stringify({ data: { ok: true } }), { + status: 200, + headers: { "Content-Type": "application/json" } + }); + } + if (href.endsWith("/api/v1/auth/login") && method === "POST") { + const headers = new Headers({ "Content-Type": "application/json" }); + headers.append("Set-Cookie", "df_session=session-1; Path=/; HttpOnly"); + headers.append("Set-Cookie", "df_csrf=csrf-1; Path=/"); + return new Response( + JSON.stringify({ + data: { + user: { id: "user-1", email }, + workspace: { id: "personal-user-1" } + } + }), + { status: 200, headers } + ); + } + if (href.endsWith("/api/v1/me") && method === "GET") { + return new Response( + JSON.stringify({ + data: { + user: { id: "user-1", email }, + workspace: { id: "personal-user-1" } + } + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ); + } + return new Response(JSON.stringify({ error: { code: "NOT_FOUND", message: href } }), { + status: 404, + headers: { "Content-Type": "application/json" } + }); + } + }); + + const result = await client.registerAndLogin({ + email, + password: "correct-horse-battery", + displayName: "Test User" + }); + + assert.equal(result.userId, "user-1"); + assert.equal(result.workspaceId, "personal-user-1"); + assert.equal(result.email, email); + assert.deepEqual(result.cookies, { + df_session: "session-1", + df_csrf: "csrf-1" + }); + assert.deepEqual( + calls.map((call) => `${call.method} ${new URL(call.href).pathname}`), + [ + "POST /api/v1/auth/register", + "POST /api/v1/auth/verify-email", + "POST /api/v1/auth/login", + "GET /api/v1/me" + ] + ); + assert.ok(!calls.some((call) => call.href.includes("/api/v1/auth/me"))); +}); diff --git a/scripts/run-dacomp6-complex-case.mjs b/scripts/run-dacomp6-complex-case.mjs index 0974b9b6..22caffd5 100644 --- a/scripts/run-dacomp6-complex-case.mjs +++ b/scripts/run-dacomp6-complex-case.mjs @@ -3,8 +3,12 @@ import assert from "node:assert/strict"; import { EventType } from "@ag-ui/client"; import { dacomp6ComplexCases, findDacomp6Case } from "./dacomp6-complex-cases.mjs"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const baseUrl = (process.env.DATAFOUNDRY_BASE_URL ?? "http://127.0.0.1:8787").replace(/\/$/u, ""); + +const client = createAuthenticatedTestClient({ baseUrl }); +await client.registerAndLogin({ displayName: "DACOMP Smoke" }); const caseId = process.argv[2] ?? "profit-root-cause"; if (caseId === "--list") { @@ -54,7 +58,7 @@ console.log(`Open in DataFoundry: http://localhost:3000/data-tasks?thread=${enco process.exit(0); async function runAgent(input) { - const response = await fetch(`${baseUrl}/api/copilotkit`, { + const response = await client.fetch("/api/copilotkit", { method: "POST", headers: requestHeaders("text/event-stream"), body: JSON.stringify({ @@ -90,8 +94,8 @@ async function runAgent(input) { async function waitForTraceSections(sessionId) { const deadline = Date.now() + 180_000; while (Date.now() < deadline) { - const response = await fetch( - `${baseUrl}/api/v1/sessions/${encodeURIComponent(sessionId)}/trace-dag`, + const response = await client.fetch( + `/api/v1/sessions/${encodeURIComponent(sessionId)}/trace-dag`, { headers: requestHeaders("application/json") }, ); if (response.ok) { @@ -125,8 +129,6 @@ function parseEventStream(text) { function requestHeaders(accept) { return { Accept: accept, - Authorization: "Bearer dev-token", "Content-Type": "application/json", - "X-Workspace-Id": "default", }; } diff --git a/scripts/seed-dtc-growth-demo.mjs b/scripts/seed-dtc-growth-demo.mjs index c5a912bf..afa07df8 100644 --- a/scripts/seed-dtc-growth-demo.mjs +++ b/scripts/seed-dtc-growth-demo.mjs @@ -7,6 +7,7 @@ import { mkdirSync, rmSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { DatabaseSync } from "node:sqlite"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const fixturesRoot = resolve(repoRoot, "storage/fixtures"); @@ -217,8 +218,18 @@ function insertRows(database, table, rows) { } } +const client = createAuthenticatedTestClient({ baseUrl: apiBase }); +let authenticated = false; +async function ensureAuth() { + if (!authenticated) { + await client.registerAndLogin({ displayName: "Seed DTC Growth" }); + authenticated = true; + } +} + async function requestJson(path, init = {}) { - const response = await fetch(`${apiBase}${path}`, init); + await ensureAuth(); + const response = await client.fetch(path, init); const body = await response.json().catch(() => ({})); return { body, response }; } diff --git a/scripts/seed-local-fixtures.mjs b/scripts/seed-local-fixtures.mjs index 0e3ee880..bed7d026 100644 --- a/scripts/seed-local-fixtures.mjs +++ b/scripts/seed-local-fixtures.mjs @@ -8,6 +8,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { DatabaseSync } from "node:sqlite"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const fixturesRoot = resolve(repoRoot, "storage/fixtures"); @@ -62,8 +63,18 @@ const datasources = [ }, ]; +const client = createAuthenticatedTestClient({ baseUrl: apiBase }); +let authenticated = false; +async function ensureAuth() { + if (!authenticated) { + await client.registerAndLogin({ displayName: "Seed Local Fixtures" }); + authenticated = true; + } +} + async function requestJson(path, init = {}) { - const response = await fetch(`${apiBase}${path}`, init); + await ensureAuth(); + const response = await client.fetch(path, init); const body = await response.json(); return { body, response }; } @@ -130,7 +141,7 @@ async function verifyDatasource(id) { } try { - const health = await fetch(`${apiBase}/healthz`); + const health = await fetch(`${apiBase}/healthz`); // public if (!health.ok) { throw new Error(`API not reachable at ${apiBase} — start with: npm run dev:api`); } diff --git a/scripts/smoke-agent-protocol-deepseek.mjs b/scripts/smoke-agent-protocol-deepseek.mjs index 32a5a94e..de3f1726 100644 --- a/scripts/smoke-agent-protocol-deepseek.mjs +++ b/scripts/smoke-agent-protocol-deepseek.mjs @@ -6,6 +6,7 @@ import { createRunProtocolBoundary } from "../packages/agent-runtime/dist/testing.js"; import { createModelProvider } from "../packages/providers/dist/index.js"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const provider = createModelProvider(process.env); assert.notEqual(provider.kind, "mock", "DeepSeek credentials are required; fake LLM is not allowed"); @@ -139,14 +140,22 @@ async function runApiScenarios(baseUrl) { ); } + +const clientsByBase = new Map(); +async function createClientForBase(baseUrl) { + const client = createAuthenticatedTestClient({ baseUrl }); + await client.registerAndLogin({ displayName: "Deepseek Protocol Smoke" }); + clientsByBase.set(baseUrl, client); + return client; +} + async function runAgent(baseUrl, input) { - const response = await fetch(`${baseUrl}/api/copilotkit`, { + const client = clientsByBase.get(baseUrl) ?? await createClientForBase(baseUrl); + const response = await client.fetch("/api/copilotkit", { method: "POST", headers: { Accept: "text/event-stream", - Authorization: "Bearer dev-token", - "Content-Type": "application/json", - "X-Workspace-Id": "default" + "Content-Type": "application/json" }, body: JSON.stringify({ method: "agent/run", diff --git a/scripts/smoke-ask-user-interrupt.mjs b/scripts/smoke-ask-user-interrupt.mjs index 404ee403..da4099d9 100644 --- a/scripts/smoke-ask-user-interrupt.mjs +++ b/scripts/smoke-ask-user-interrupt.mjs @@ -4,6 +4,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { randomUUID } from "node:crypto"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const envPath = join(process.cwd(), ".env"); try { @@ -26,6 +27,8 @@ if (!process.env.LLM_API_KEY) { } const apiBase = `http://127.0.0.1:${process.env.API_PORT ?? "8787"}`; +const client = createAuthenticatedTestClient({ baseUrl: apiBase }); +await client.registerAndLogin({ displayName: "Ask User Smoke" }); const threadId = `thread-ask-smoke-${Date.now()}`; const runId = randomUUID(); @@ -50,7 +53,7 @@ const payload = { }, }; -const response = await fetch(`${apiBase}/api/copilotkit`, { +const response = await client.fetch("/api/copilotkit", { method: "POST", headers: { "Content-Type": "application/json", Accept: "text/event-stream" }, body: JSON.stringify(payload), diff --git a/scripts/smoke-auth.mjs b/scripts/smoke-auth.mjs index 7d972755..7992f20a 100644 --- a/scripts/smoke-auth.mjs +++ b/scripts/smoke-auth.mjs @@ -5,12 +5,14 @@ import { join } from "node:path"; import { createServer as createApiServer } from "../apps/api/dist/server.js"; import { createMetadataStore } from "../packages/metadata/dist/index.js"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const root = mkdtempSync(join(tmpdir(), "datafoundry-auth-smoke-")); process.env.DATAFOUNDRY_AUTH_MODE = "password"; process.env.AUTH_SESSION_SECRET = "smoke-session-secret-with-at-least-32-bytes"; process.env.AUTH_PUBLIC_BASE_URL = "http://127.0.0.1"; process.env.AUTH_EMAIL_DELIVERY = "test"; +process.env.AUTH_REGISTRATION_MODE = "open"; process.env.EMBEDDING_API_KEY = ""; process.env.MASTRA_STORAGE_PATH = join(root, "mastra.sqlite"); process.env.STORAGE_ROOT_DIR = root; @@ -32,33 +34,10 @@ const closeServer = async () => { }); }; -const cookieJar = []; -const rememberCookies = (response) => { - const setCookie = response.headers.getSetCookie?.() ?? []; - for (const cookie of setCookie) { - const pair = cookie.split(";", 1)[0]; - const name = pair.split("=", 1)[0]; - const index = cookieJar.findIndex((item) => item.startsWith(`${name}=`)); - if (index >= 0) { - cookieJar.splice(index, 1, pair); - } else { - cookieJar.push(pair); - } - } -}; -const cookieHeader = () => cookieJar.join("; "); -const csrfCookie = () => { - const entry = cookieJar.find((item) => item.startsWith("df_csrf=")); - return entry ? decodeURIComponent(entry.slice("df_csrf=".length)) : undefined; -}; +const client = createAuthenticatedTestClient({ baseUrl }); const requestJson = async (path, init = {}) => { - const headers = { - ...(cookieJar.length > 0 ? { Cookie: cookieHeader() } : {}), - ...init.headers - }; - const response = await fetch(`${baseUrl}${path}`, { ...init, headers }); - rememberCookies(response); + const response = await client.fetch(path, init); const body = await response.json(); return { body, response }; }; @@ -140,7 +119,7 @@ try { const missingCsrf = await requestJson("/api/v1/datasources", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", "X-CSRF-Token": "" }, body: JSON.stringify({ id: "alice-sqlite", name: "Alice SQLite", @@ -148,14 +127,15 @@ try { settings: { filePath: join(root, "alice.sqlite") } }) }); + // Clear empty override so subsequent unsafe calls auto-attach cookie CSRF. assert.equal(missingCsrf.response.status, 403); - assert.equal(missingCsrf.body.error.code, "FORBIDDEN"); + assert.equal(missingCsrf.body.error.code, "CSRF_INVALID"); - const csrf = csrfCookie(); + const csrf = client.cookies.csrfToken(); assert(csrf, "Expected login to set a readable CSRF cookie"); const aliceDatasource = await requestJson("/api/v1/datasources", { method: "POST", - headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ id: "alice-sqlite", name: "Alice SQLite", @@ -186,7 +166,7 @@ try { const revokedMe = await requestJson("/api/v1/me"); assert.equal(revokedMe.response.status, 401); - cookieJar.splice(0, cookieJar.length); + client.cookies.clear(); const relogged = await requestJson("/api/v1/auth/login", { method: "POST", headers: { "Content-Type": "application/json" }, @@ -195,8 +175,7 @@ try { assert.equal(relogged.response.status, 200); const logout = await requestJson("/api/v1/auth/logout", { - method: "POST", - headers: { "X-CSRF-Token": csrfCookie() } + method: "POST" }); assert.equal(logout.response.status, 200); diff --git a/scripts/smoke-config-api.mjs b/scripts/smoke-config-api.mjs index df84c154..33b890f2 100644 --- a/scripts/smoke-config-api.mjs +++ b/scripts/smoke-config-api.mjs @@ -14,6 +14,7 @@ import { resolveEffectiveRunConfig } from "../apps/api/dist/run-input.js"; import { createTaskStateRuntime } from "../packages/agent-runtime/dist/index.js"; import { LocalDataGateway } from "../packages/data-gateway/dist/index.js"; import { RunEventWriter, createMetadataStore } from "../packages/metadata/dist/index.js"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const root = mkdtempSync(join(tmpdir(), "open-data-foundry-config-smoke-")); const datasourcePath = join(root, "source.sqlite"); @@ -26,6 +27,11 @@ source.exec(` `); source.close(); +process.env.DATAFOUNDRY_AUTH_MODE = "password"; +process.env.AUTH_SESSION_SECRET = "config-smoke-session-secret-32bytes!!"; +process.env.AUTH_PUBLIC_BASE_URL = "http://127.0.0.1:3000"; +process.env.AUTH_EMAIL_DELIVERY = "test"; +process.env.AUTH_REGISTRATION_MODE = "open"; process.env.EMBEDDING_API_KEY = ""; process.env.MASTRA_STORAGE_PATH = join(root, "mastra.sqlite"); process.env.STORAGE_ROOT_DIR = root; @@ -131,20 +137,21 @@ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const address = server.address(); assert(address && typeof address === "object"); const baseUrl = `http://127.0.0.1:${address.port}`; -metadataStore.users.upsertDevUser({ - id: "tenant-user", - email: "tenant@example.com", - display_name: "Tenant User", - dev_token: "tenant-token" -}); -const requestJson = async (path, init = {}) => { - const response = await fetch(`${baseUrl}${path}`, init); +const client = createAuthenticatedTestClient({ baseUrl }); +const identity = await client.registerAndLogin({ displayName: "Config Smoke User" }); +const { userId, workspaceId } = identity; + +const tenantClient = createAuthenticatedTestClient({ baseUrl }); +const tenantIdentity = await tenantClient.registerAndLogin({ displayName: "Tenant User" }); + +const requestJson = async (path, init = {}, authClient = client) => { + const response = await authClient.fetch(path, init); const body = await response.json(); return { body, response }; }; -const requestRaw = async (path, init = {}) => fetch(`${baseUrl}${path}`, init); +const requestRaw = async (path, init = {}, authClient = client) => authClient.fetch(path, init); const closeHttpServer = async (httpServer) => { await new Promise((resolve, reject) => { @@ -207,78 +214,55 @@ try { ); } - const invalidToken = await requestJson("/api/v1/workspace-config", { - headers: { "Authorization": "Bearer invalid-token" } - }); - assert.equal(invalidToken.response.status, 401); - assert.equal(invalidToken.body.error.code, "UNAUTHORIZED"); + const anonymousClient = createAuthenticatedTestClient({ baseUrl }); + const unauthenticated = await requestJson("/api/v1/workspace-config", {}, anonymousClient); + assert.equal(unauthenticated.response.status, 401); + assert.equal(unauthenticated.body.error.code, "UNAUTHORIZED"); const defaultMe = await requestJson("/api/v1/me"); assert.equal(defaultMe.response.status, 200); - assert.equal(defaultMe.body.data.user.id, "dev-user"); - assert.equal(defaultMe.body.data.workspace.id, "default"); + assert.equal(defaultMe.body.data.user.id, userId); + assert.equal(defaultMe.body.data.workspace.id, workspaceId); - const identitiesBeforeCreate = await requestJson("/api/v1/dev/identities"); - assert.equal(identitiesBeforeCreate.response.status, 200); - assert(identitiesBeforeCreate.body.data.users.some((user) => user.id === "dev-user")); - - const createdDevUser = await requestJson("/api/v1/dev/users", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - id: "analyst-user", - email: "analyst@example.com", - displayName: "Analyst User" - }) - }); - assert.equal(createdDevUser.response.status, 201, JSON.stringify(createdDevUser.body)); - assert.equal(createdDevUser.body.data.user.id, "analyst-user"); - assert.equal(createdDevUser.body.data.user.devToken, "dev-token-analyst-user"); - const analystMe = await requestJson("/api/v1/me", { - headers: { "Authorization": "Bearer dev-token-analyst-user" } + const analystClient = createAuthenticatedTestClient({ baseUrl }); + const analystIdentity = await analystClient.registerAndLogin({ + email: `analyst-${Date.now()}@example.test`, + displayName: "Analyst User" }); + const analystMe = await requestJson("/api/v1/me", {}, analystClient); assert.equal(analystMe.response.status, 200); - assert.equal(analystMe.body.data.user.id, "analyst-user"); - assert.equal(analystMe.body.data.workspace.id, "default"); + assert.equal(analystMe.body.data.user.id, analystIdentity.userId); + assert.equal(analystMe.body.data.workspace.id, analystIdentity.workspaceId); + assert.notEqual(analystIdentity.userId, userId); - const tenantHeaders = { - "Authorization": "Bearer tenant-token", - "Content-Type": "application/json", - "X-Workspace-Id": "tenant-workspace" - }; const tenantDatasource = await requestJson("/api/v1/datasources", { method: "POST", - headers: tenantHeaders, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ id: "local-sqlite", name: "Tenant SQLite", type: "sqlite", settings: { filePath: datasourcePath } }) - }); + }, tenantClient); assert.equal(tenantDatasource.response.status, 201, JSON.stringify(tenantDatasource.body)); const devDatasourceListBeforeTenant = await requestJson("/api/v1/datasources"); assert.equal(devDatasourceListBeforeTenant.body.data.some((item) => item.id === "local-sqlite"), false); const tenantKnowledge = await requestJson("/api/v1/knowledge-bases", { method: "POST", - headers: tenantHeaders, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ id: "tenant-docs", name: "Tenant Docs", defaultEnabled: true, retrievalTopK: 2 }) - }); + }, tenantClient); assert.equal(tenantKnowledge.response.status, 201); const devTenantKnowledge = await requestJson("/api/v1/knowledge-bases/tenant-docs"); assert.equal(devTenantKnowledge.response.status, 404); - const tenantOtherWorkspaceKnowledge = await requestJson("/api/v1/knowledge-bases/tenant-docs", { - headers: { - "Authorization": "Bearer tenant-token", - "X-Workspace-Id": "other-workspace" - } - }); - assert.equal(tenantOtherWorkspaceKnowledge.response.status, 404); + const tenantVisibleKnowledge = await requestJson("/api/v1/knowledge-bases/tenant-docs", {}, tenantClient); + assert.equal(tenantVisibleKnowledge.response.status, 200); const uploadForm = new FormData(); uploadForm.append("sessionId", "session-smoke-upload"); @@ -324,9 +308,9 @@ try { const conversationSessionId = "conversation-api-session"; const conversationRunId = "conversation-api-run"; - metadataStore.sessions.create({ user_id: "dev-user", id: conversationSessionId, title: "conversation API" }); + metadataStore.sessions.create({ user_id: userId, id: conversationSessionId, title: "conversation API" }); metadataStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: conversationRunId, session_id: conversationSessionId, request_fingerprint: "conversation-api-fingerprint", @@ -334,7 +318,7 @@ try { status: "completed" }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, run_id: conversationRunId, id: `${conversationRunId}:user`, @@ -345,7 +329,7 @@ try { content: { text: "inspect orders" } }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, run_id: conversationRunId, id: `${conversationRunId}:assistant`, @@ -356,7 +340,7 @@ try { content: { text: "orders has 2 columns" } }); metadataStore.conversationSummaries.create({ - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, id: `summary:${conversationSessionId}:1-1`, source_run_id: conversationRunId, @@ -366,19 +350,19 @@ try { }); const conversationWriter = new RunEventWriter(metadataStore.runEvents); conversationWriter.write({ - user_id: "dev-user", + user_id: userId, run_id: conversationRunId, session_id: conversationSessionId, event: { type: "TOOL_CALL_START", toolCallId: "call_schema", toolCallName: "inspect_schema" } }); conversationWriter.write({ - user_id: "dev-user", + user_id: userId, run_id: conversationRunId, session_id: conversationSessionId, event: { type: "TOOL_CALL_END", toolCallId: "call_schema", toolCallName: "inspect_schema" } }); conversationWriter.write({ - user_id: "dev-user", + user_id: userId, run_id: conversationRunId, session_id: conversationSessionId, event: { @@ -442,7 +426,7 @@ try { // R-018: pending HITL without TOOL_CALL_START still synthesizes toolCalls + checkpoint. const hitlOnlyRunId = "conversation-hitl-only-run"; metadataStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: hitlOnlyRunId, session_id: conversationSessionId, user_input: "ask the user", @@ -450,7 +434,7 @@ try { }); metadataStore.interactions.request({ id: "interaction-hitl-only", - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, run_id: hitlOnlyRunId, tool_call_id: "call_ask_missing_start", @@ -487,14 +471,14 @@ try { const branchOriginalRunId = "conversation-branch-original-run"; const branchPendingRunId = "conversation-branch-pending-run"; metadataStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: branchBaseRunId, session_id: conversationSessionId, user_input: "summarize orders", status: "completed" }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, run_id: branchBaseRunId, id: `${branchBaseRunId}:user`, @@ -505,7 +489,7 @@ try { content: { text: "summarize orders" } }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, run_id: branchBaseRunId, id: `${branchBaseRunId}:assistant`, @@ -516,14 +500,14 @@ try { content: { text: "orders summary" } }); metadataStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: branchOriginalRunId, session_id: conversationSessionId, user_input: "make a chart", status: "completed" }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, run_id: branchOriginalRunId, id: `${branchOriginalRunId}:user`, @@ -534,7 +518,7 @@ try { content: { text: "make a chart" } }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: conversationSessionId, run_id: branchOriginalRunId, id: `${branchOriginalRunId}:assistant`, @@ -545,7 +529,7 @@ try { content: { text: "original chart answer" } }); metadataStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: branchPendingRunId, session_id: conversationSessionId, user_input: "still running", @@ -573,7 +557,7 @@ try { const branchRunId = "conversation-branch-child-run"; metadataStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: branchRunId, session_id: branchSessionId, parent_run_id: branchOriginalRunId, @@ -581,7 +565,7 @@ try { status: "completed" }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: branchSessionId, run_id: branchRunId, id: `${branchRunId}:user`, @@ -592,7 +576,7 @@ try { content: { text: "make a table instead" } }); metadataStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: branchSessionId, run_id: branchRunId, id: `${branchRunId}:assistant`, @@ -711,12 +695,18 @@ try { const policyGateway = new LocalDataGateway(metadataStore, { defaultLimit: 100, maxLimit: 1000, - timeoutMs: 10000 + timeoutMs: 10000, + workspaceId + }); + const policySchema = await policyGateway.inspectSchema({ + user_id: userId, + workspace_id: workspaceId, + datasource_id: "local-sqlite" }); - const policySchema = await policyGateway.inspectSchema({ user_id: "dev-user", datasource_id: "local-sqlite" }); assert.deepEqual(policySchema.tables.map((table) => table.name), ["metrics"]); const preview = await policyGateway.previewTable({ - user_id: "dev-user", + user_id: userId, + workspace_id: workspaceId, datasource_id: "local-sqlite", table: "metrics", limit: 10 @@ -724,7 +714,8 @@ try { assert.equal(preview.row_count, 1, "samplePolicy.maxSampleRows should cap preview rows"); assert.equal(preview.rows[0]?.[preview.columns.indexOf("secret")], "[MASKED]"); const maskedSql = await policyGateway.runSqlReadonly({ - user_id: "dev-user", + user_id: userId, + workspace_id: workspaceId, datasource_id: "local-sqlite", sql: "SELECT name, secret FROM metrics", limit: 10 @@ -732,7 +723,8 @@ try { assert.equal(maskedSql.rows[0]?.[maskedSql.columns.indexOf("secret")], "[MASKED]"); await assert.rejects( () => policyGateway.previewTable({ - user_id: "dev-user", + user_id: userId, + workspace_id: workspaceId, datasource_id: "local-sqlite", table: "private_metrics", limit: 1 @@ -741,7 +733,8 @@ try { ); await assert.rejects( () => policyGateway.runSqlReadonly({ - user_id: "dev-user", + user_id: userId, + workspace_id: workspaceId, datasource_id: "local-sqlite", sql: "SELECT * FROM private_metrics", limit: 1 @@ -759,7 +752,8 @@ try { assert.equal(sampleDisabledPatch.response.status, 200); await assert.rejects( () => policyGateway.previewTable({ - user_id: "dev-user", + user_id: userId, + workspace_id: workspaceId, datasource_id: "local-sqlite", table: "metrics", limit: 1 @@ -880,7 +874,7 @@ try { // Light upload → promote → list(scope=workspace) path for workspace assets. const workspaceSessionId = "config-smoke-workspace-promote"; metadataStore.sessions.create({ - user_id: "dev-user", + user_id: userId, id: workspaceSessionId, title: "config smoke workspace promote" }); @@ -1106,9 +1100,9 @@ try { } } }, - userId: "dev-user", + userId: userId, userInput: "inspect metrics", - workspaceId: "default" + workspaceId }); assert.equal(resolvedModelRun.modelSettings?.topP, 0.75); assert.equal(resolvedModelRun.modelSettings?.frequencyPenalty, 0.25); @@ -1138,9 +1132,9 @@ try { } } }, - userId: "dev-user", + userId: userId, userInput: "use smoke skill to inspect schema", - workspaceId: "default" + workspaceId }); assert.equal(skillBoundRun.effectiveRunConfig.activeDatasourceId, "local-sqlite"); assert.equal(skillBoundRun.effectiveRunConfig.activeLlmProfileId, "smoke-openai-compatible"); @@ -1152,20 +1146,20 @@ try { assert.deepEqual(skillBoundRun.mcpRuntime.servers[0]?.toolAllowlist, ["echo"]); const currentMetricsDocsResource = metadataStore.configResources.get({ id: "metrics-docs", - workspace_id: "default", - user_id: "dev-user", + workspace_id: workspaceId, + user_id: userId, kind: "knowledge-base" }); const currentMcpResource = metadataStore.configResources.get({ id: "smoke-mcp", - workspace_id: "default", - user_id: "dev-user", + workspace_id: workspaceId, + user_id: userId, kind: "mcp-server" }); const currentModelProfileResource = metadataStore.configResources.get({ id: "smoke-openai-compatible", - workspace_id: "default", - user_id: "dev-user", + workspace_id: workspaceId, + user_id: userId, kind: "model-profile" }); assert.equal( @@ -1203,9 +1197,9 @@ try { } } }, - userId: "dev-user", + userId: userId, userInput: "use smoke skill to inspect schema", - workspaceId: "default" + workspaceId }); assert.equal(explicitResourceRun.effectiveRunConfig.activeDatasourceId, "dtc-growth-demo"); assert(explicitResourceRun.effectiveRunConfig.enabledDatasourceIds.includes("local-sqlite")); @@ -1269,7 +1263,7 @@ try { }], state: {}, forwardedProps: {} - }, metadataStore, "dev-user", "dtc-growth-demo"); + }, metadataStore, userId, "dtc-growth-demo", workspaceId); assert.equal(effective.activeSkillId, undefined); assert.equal(effective.resourceRevisions["datasource:local-sqlite"], sampleDisabledPatch.body.data.revision); assert.equal(effective.resourceRevisions["model-profile:server-default"] > 0, true); @@ -1290,18 +1284,55 @@ try { const defaults = await requestJson("/api/v1/run-defaults"); assert.equal(defaults.body.data.enabledKnowledgeIds.includes("metrics-docs"), false); - metadataStore.sessions.create({ user_id: "dev-user", id: "session-smoke", title: "config smoke" }); + metadataStore.sessions.create({ user_id: userId, id: "session-smoke", title: "config smoke" }); metadataStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: "run-smoke", session_id: "session-smoke", request_fingerprint: "config-smoke", user_input: "config smoke", status: "completed" }); + metadataStore.contextPackageSnapshots.create({ + user_id: userId, + session_id: "session-smoke", + run_id: "run-smoke", + package_id: "context-smoke", + revision: 0, + payload: {} + }); + metadataStore.protocolStates.compareAndSetWithEvents({ + user_id: userId, + run_id: "run-smoke", + segment_id: "segment-smoke", + expected_revision: -1, + state: { + protocolId: "general-task", + protocolVersion: "1", + runId: "run-smoke", + segmentId: "segment-smoke", + revision: 0, + phase: "work", + status: "completed", + contextPackageRef: { packageId: "context-smoke", revision: 0 }, + actions: [], + completionRejections: 0, + domain: {} + } + }, [{ + eventId: "protocol-event-smoke", + type: "protocol.run.completed", + runId: "run-smoke", + segmentId: "segment-smoke", + revision: 0 + }]); + assert.equal(metadataStore.protocolStates.pendingEvents({ + user_id: userId, + run_id: "run-smoke" + }).length, 1); const artifact = metadataStore.artifacts.create({ id: "artifact-smoke", - user_id: "dev-user", + user_id: userId, session_id: "session-smoke", run_id: "run-smoke", type: "table", @@ -1309,16 +1340,33 @@ try { preview_json: { columns: ["name", "value"], rows: [{ name: "revenue", value: 42 }] } }); assert.equal(artifact.id, "artifact-smoke"); - const download = await fetch(`${baseUrl}/api/v1/artifacts/artifact-smoke/download`); + const download = await client.fetch("/api/v1/artifacts/artifact-smoke/download"); assert.equal(download.headers.get("content-type"), "text/csv; charset=utf-8"); assert.equal((await download.text()).includes("revenue,42"), true); + const deletedSession = await requestJson("/api/v1/sessions/session-smoke", { method: "DELETE" }); + assert.equal(deletedSession.response.status, 200, JSON.stringify(deletedSession.body)); + assert.deepEqual(deletedSession.body.data, { + sessionId: "session-smoke", + deleted: true, + deletedSessionIds: ["session-smoke"] + }); + assert.equal(metadataStore.protocolStates.find({ + user_id: userId, + run_id: "run-smoke", + segment_id: "segment-smoke" + }), undefined); + assert.deepEqual(metadataStore.protocolStates.pendingEvents({ + user_id: userId, + run_id: "run-smoke" + }), []); + const builtinDelete = await requestJson("/api/v1/datasources/dtc-growth-demo", { method: "DELETE" }); assert.equal(builtinDelete.response.status, 200); assert.equal(builtinDelete.body.data.deleted, true); metadataStore.sqlAuditLogs.create({ - user_id: "dev-user", + user_id: userId, id: "audit-before-delete", datasource_id: "local-sqlite", sql_text: "SELECT 1", @@ -1330,13 +1378,13 @@ try { assert.equal(datasourceList.body.data.some((item) => item.id === "local-sqlite"), false); assert.throws(() => metadataStore.secrets.get({ ref: createdDatasource.body.data.secretRef, - workspace_id: "default", - user_id: "dev-user" + workspace_id: workspaceId, + user_id: userId }), /SECRET_NOT_FOUND/u); console.log( "Config API smoke OK: secrets, datasource/types/policies, chat upload, datasource upload, conversation, revision, " - + "KB, MCP, model profile, skill binding, defaults, artifact, tombstone" + + "KB, MCP, model profile, skill binding, defaults, artifact, session delete, tombstone" ); } finally { await closeHttpServer(server); diff --git a/scripts/smoke-copilotkit-run.mjs b/scripts/smoke-copilotkit-run.mjs index 48adf617..58b00836 100644 --- a/scripts/smoke-copilotkit-run.mjs +++ b/scripts/smoke-copilotkit-run.mjs @@ -6,12 +6,18 @@ import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { EventType } from "@ag-ui/core"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const root = mkdtempSync(join(tmpdir(), "open-data-foundry-copilotkit-run-")); const metadataPath = join(root, "metadata.sqlite"); const mastraStoragePath = join(root, "mastra.sqlite"); const workspaceRoot = join(root, "workspaces"); +process.env.DATAFOUNDRY_AUTH_MODE = "password"; +process.env.AUTH_SESSION_SECRET = "copilotkit-run-session-secret-32b!!!"; +process.env.AUTH_PUBLIC_BASE_URL = "http://127.0.0.1:3000"; +process.env.AUTH_EMAIL_DELIVERY = "test"; +process.env.AUTH_REGISTRATION_MODE = "open"; process.env.EMBEDDING_API_KEY = ""; process.env.LLM_PROVIDER = "openai-compatible"; process.env.LLM_MODEL = "copilotkit-smoke-model"; @@ -142,7 +148,30 @@ const address = server.address(); assert(address && typeof address === "object"); const baseUrl = `http://127.0.0.1:${address.port}`; +const client = createAuthenticatedTestClient({ baseUrl }); +const identity = await client.registerAndLogin({ displayName: "CopilotKit Run Smoke" }); +const { userId } = identity; + try { + const demoDbPath = join(root, "api-duckdb-demo.sqlite"); + const demoDb = new DatabaseSync(demoDbPath); + demoDb.exec(` + CREATE TABLE orders (id INTEGER, amount REAL); + INSERT INTO orders VALUES (1, 10.5), (2, 20); + `); + demoDb.close(); + const demoDatasource = await client.fetch("/api/v1/datasources", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + id: "api-duckdb-demo", + name: "CopilotKit Smoke Demo", + type: "sqlite", + settings: { filePath: demoDbPath } + }) + }); + assert.equal(demoDatasource.status, 201, await demoDatasource.text()); + const skillForm = new FormData(); skillForm.set("file", new Blob([ "---\n", @@ -154,12 +183,12 @@ try { "---\n", "Use this skill to prove Mastra skill tool loading works in the AG-UI runtime.\n" ], { type: "text/markdown" }), "SKILL.md"); - const skillUploadResponse = await fetch(`${baseUrl}/api/v1/skills`, { method: "POST", body: skillForm }); + const skillUploadResponse = await client.fetch("/api/v1/skills", { method: "POST", body: skillForm }); assert.equal(skillUploadResponse.status, 201); const skillUpload = await skillUploadResponse.json(); assert.equal(skillUpload.data.validationStatus, "valid"); - const timeoutProfileResponse = await fetch(`${baseUrl}/api/v1/model-profiles`, { + const timeoutProfileResponse = await client.fetch("/api/v1/model-profiles", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -204,8 +233,8 @@ try { "Missing model profile should emit RUN_ERROR" ); assertRunStatusDelta(badModelEvents, "failed"); - const badModelConversationResponse = await fetch( - `${baseUrl}/api/v1/sessions/${badModelThreadId}/conversation?limit=10` + const badModelConversationResponse = await client.fetch( + `/api/v1/sessions/${badModelThreadId}/conversation?limit=10` ); assert.equal(badModelConversationResponse.status, 200); const badModelConversation = await badModelConversationResponse.json(); @@ -270,19 +299,19 @@ try { const branchSeedStore = createMetadataStore({ database_path: metadataPath }); try { branchSeedStore.sessions.create({ - user_id: "dev-user", + user_id: userId, id: branchParentSessionId, title: "CopilotKit branch smoke" }); branchSeedStore.runs.create({ - user_id: "dev-user", + user_id: userId, id: branchParentRunId, session_id: branchParentSessionId, user_input: "请分析订单。", status: "completed" }); branchSeedStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: branchParentSessionId, run_id: branchParentRunId, id: `${branchParentRunId}:user`, @@ -293,7 +322,7 @@ try { content: { text: "请分析订单。" } }); branchSeedStore.conversationMessages.append({ - user_id: "dev-user", + user_id: userId, session_id: branchParentSessionId, run_id: branchParentRunId, id: `${branchParentRunId}:assistant`, @@ -304,13 +333,13 @@ try { content: { text: "订单分析已完成。" } }); branchSeedStore.sessions.touchLastMessage({ - user_id: "dev-user", + user_id: userId, session_id: branchParentSessionId }); } finally { branchSeedStore.close(); } - const branchResponse = await fetch(`${baseUrl}/api/v1/sessions/${branchParentSessionId}/branches`, { + const branchResponse = await client.fetch(`/api/v1/sessions/${branchParentSessionId}/branches`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ runId: branchParentRunId }) @@ -421,7 +450,7 @@ try { const store = createMetadataStore({ database_path: metadataPath }); try { - const persisted = store.runEvents.listByRun({ user_id: "dev-user", run_id: runId }); + const persisted = store.runEvents.listByRun({ user_id: userId, run_id: runId }); assert(persisted.length > 0, "DataFoundryAgUiAgent.run should persist AG-UI events"); const persistedEvents = persisted.map((item) => JSON.parse(item.payload_json)); assert.equal(persistedEvents[persistedEvents.length - 1]?.type, EventType.RUN_FINISHED); @@ -477,10 +506,10 @@ try { const suspendedStore = createMetadataStore({ database_path: metadataPath }); try { - const suspendedRun = suspendedStore.runs.get({ user_id: "dev-user", run_id: suspendedRunId }); + const suspendedRun = suspendedStore.runs.get({ user_id: userId, run_id: suspendedRunId }); assert.equal(suspendedRun.status, "suspended"); const suspendedPersistedEvents = suspendedStore.runEvents - .listByRun({ user_id: "dev-user", run_id: suspendedRunId }) + .listByRun({ user_id: userId, run_id: suspendedRunId }) .map((item) => JSON.parse(item.payload_json)); assert.equal( suspendedPersistedEvents.some((event) => event.type === EventType.RUN_FINISHED), @@ -491,7 +520,7 @@ try { ); assertRunStatusDelta(suspendedPersistedEvents, "suspended"); const suspendedMessages = suspendedStore.conversationMessages.listRecent({ - user_id: "dev-user", + user_id: userId, session_id: suspendedThreadId, limit: 20 }); @@ -535,8 +564,8 @@ function writeStreamDone(response, model, finishReason) { response.end("data: [DONE]\n\n"); } -async function runCopilotKitAgent(baseUrl, input) { - return readAgUiEventStream(await fetch(`${baseUrl}/api/copilotkit`, { +async function runCopilotKitAgent(_baseUrl, input) { + return readAgUiEventStream(await client.fetch("/api/copilotkit", { method: "POST", headers: { "Accept": "text/event-stream", @@ -550,8 +579,8 @@ async function runCopilotKitAgent(baseUrl, input) { })); } -async function connectCopilotKitAgent(baseUrl, threadId) { - return readAgUiEventStream(await fetch(`${baseUrl}/api/copilotkit`, { +async function connectCopilotKitAgent(_baseUrl, threadId) { + return readAgUiEventStream(await client.fetch("/api/copilotkit", { method: "POST", headers: { "Accept": "text/event-stream", diff --git a/scripts/smoke-copilotkit.mjs b/scripts/smoke-copilotkit.mjs index b0089e07..076e3ba3 100644 --- a/scripts/smoke-copilotkit.mjs +++ b/scripts/smoke-copilotkit.mjs @@ -1,5 +1,6 @@ import { spawn } from "node:child_process"; import { setTimeout as delay } from "node:timers/promises"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const apiPort = process.env.API_PORT ?? "8798"; const apiBaseUrl = `http://127.0.0.1:${apiPort}`; @@ -10,7 +11,12 @@ const child = spawn("npm", ["--workspace", "@datafoundry/api", "run", "dev"], { ...process.env, API_HOST: "127.0.0.1", API_PORT: apiPort, - METADATA_DB_PATH: metadataDbPath + METADATA_DB_PATH: metadataDbPath, + DATAFOUNDRY_AUTH_MODE: "password", + AUTH_SESSION_SECRET: process.env.AUTH_SESSION_SECRET ?? "copilotkit-smoke-session-secret-32b!", + AUTH_PUBLIC_BASE_URL: process.env.AUTH_PUBLIC_BASE_URL ?? "http://127.0.0.1:3000", + AUTH_EMAIL_DELIVERY: process.env.AUTH_EMAIL_DELIVERY ?? "test", + AUTH_REGISTRATION_MODE: process.env.AUTH_REGISTRATION_MODE ?? "open" }, stdio: ["ignore", "pipe", "pipe"] }); @@ -25,8 +31,10 @@ child.stderr.on("data", (chunk) => { try { await waitForHealth(apiBaseUrl); + const client = createAuthenticatedTestClient({ baseUrl: apiBaseUrl }); + await client.registerAndLogin({ displayName: "CopilotKit Smoke" }); - const optionsResponse = await fetch(`${apiBaseUrl}/api/copilotkit`, { + const optionsResponse = await client.fetch("/api/copilotkit", { method: "OPTIONS", headers: { Origin: "http://127.0.0.1:3000", @@ -38,7 +46,7 @@ try { throw new Error(`Unexpected CopilotKit OPTIONS status: ${optionsResponse.status}`); } - const postResponse = await fetch(`${apiBaseUrl}/api/copilotkit`, { + const postResponse = await client.fetch("/api/copilotkit", { method: "POST", headers: { "Content-Type": "application/json" diff --git a/scripts/smoke-interaction-run-id.mjs b/scripts/smoke-interaction-run-id.mjs index 7b845d60..62bea0e7 100644 --- a/scripts/smoke-interaction-run-id.mjs +++ b/scripts/smoke-interaction-run-id.mjs @@ -4,6 +4,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { randomUUID } from "node:crypto"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const envPath = join(process.cwd(), ".env"); try { @@ -26,6 +27,8 @@ if (!process.env.LLM_API_KEY) { } const apiBase = `http://127.0.0.1:${process.env.API_PORT ?? "8787"}`; +const client = createAuthenticatedTestClient({ baseUrl: apiBase }); +await client.registerAndLogin({ displayName: "Interaction RunId Smoke" }); const threadId = `thread-run-id-smoke-${Date.now()}`; const originalRunId = randomUUID(); const mismatchedResumeRunId = randomUUID(); @@ -116,7 +119,7 @@ if (runErrors.length > 0) { console.log("interaction run-id smoke OK"); async function runAgent(body) { - const response = await fetch(`${apiBase}/api/copilotkit`, { + const response = await client.fetch("/api/copilotkit", { method: "POST", headers: { "Content-Type": "application/json", Accept: "text/event-stream" }, body: JSON.stringify({ diff --git a/scripts/smoke-password-frontend-isolation.mjs b/scripts/smoke-password-frontend-isolation.mjs index 8d1580e7..3521981f 100644 --- a/scripts/smoke-password-frontend-isolation.mjs +++ b/scripts/smoke-password-frontend-isolation.mjs @@ -7,74 +7,14 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -const baseUrl = process.env.SMOKE_BASE_URL ?? "http://localhost:3000"; -const password = "correct horse battery staple"; - -const cookieJar = []; -const rememberCookies = (response) => { - const setCookie = response.headers.getSetCookie?.() ?? []; - for (const cookie of setCookie) { - const pair = cookie.split(";", 1)[0]; - const name = pair.split("=", 1)[0]; - const index = cookieJar.findIndex((item) => item.startsWith(`${name}=`)); - if (index >= 0) cookieJar.splice(index, 1, pair); - else cookieJar.push(pair); - } -}; -const cookieHeader = () => cookieJar.join("; "); -const csrfCookie = () => { - const entry = cookieJar.find((item) => item.startsWith("df_csrf=")); - return entry ? decodeURIComponent(entry.slice("df_csrf=".length)) : undefined; -}; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; -const requestJson = async (path, init = {}) => { - const headers = { - ...(cookieJar.length > 0 ? { Cookie: cookieHeader() } : {}), - ...init.headers, - }; - const response = await fetch(`${baseUrl}${path}`, { ...init, headers }); - rememberCookies(response); - const text = await response.text(); - let body; - try { - body = text ? JSON.parse(text) : null; - } catch { - throw new Error(`Non-JSON ${response.status} for ${path}: ${text.slice(0, 200)}`); - } - return { body, response }; -}; +const baseUrl = process.env.SMOKE_BASE_URL ?? "http://localhost:3000"; -const registerVerifyLogin = async (email, displayName) => { - cookieJar.splice(0, cookieJar.length); - const registered = await requestJson("/api/v1/auth/register", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ email, password, displayName }), - }); - assert.equal(registered.response.status, 201, JSON.stringify(registered.body)); - const token = registered.body.data.verificationToken; - assert.equal(typeof token, "string"); - const verified = await requestJson("/api/v1/auth/verify-email", { +const createDatasource = async (client, id, name, filePath) => + client.fetchJson("/api/v1/datasources", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ token }), - }); - assert.equal(verified.response.status, 200, JSON.stringify(verified.body)); - const loggedIn = await requestJson("/api/v1/auth/login", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ email, password }), - }); - assert.equal(loggedIn.response.status, 200, JSON.stringify(loggedIn.body)); - return { ...loggedIn.body.data, email }; -}; - -const createDatasource = async (id, name, filePath) => { - const csrf = csrfCookie(); - assert(csrf, "Expected CSRF cookie after login"); - return requestJson("/api/v1/datasources", { - method: "POST", - headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, body: JSON.stringify({ id, name, @@ -82,72 +22,79 @@ const createDatasource = async (id, name, filePath) => { settings: { filePath }, }), }); -}; -const patchSessionTitle = async (sessionId, title) => { - const csrf = csrfCookie(); - return requestJson(`/api/v1/sessions/${encodeURIComponent(sessionId)}`, { +const patchSessionTitle = async (client, sessionId, title) => + client.fetchJson(`/api/v1/sessions/${encodeURIComponent(sessionId)}`, { method: "PATCH", - headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ title }), }); -}; const stamp = Date.now(); -const aliceEmail = `alice-front-${stamp}@example.com`; -const bobEmail = `bob-front-${stamp}@example.com`; const root = mkdtempSync(join(tmpdir(), "df-password-front-")); const aliceDb = join(root, "alice.sqlite"); const bobDb = join(root, "bob.sqlite"); try { - const alice = await registerVerifyLogin(aliceEmail, "Alice Front"); - const aliceDs = await createDatasource("alice-front-db", "Alice Front DB", aliceDb); + const aliceClient = createAuthenticatedTestClient({ baseUrl }); + const alice = await aliceClient.registerAndLogin({ + email: `alice-front-${stamp}@example.com`, + displayName: "Alice Front", + }); + const aliceDs = await createDatasource(aliceClient, "alice-front-db", "Alice Front DB", aliceDb); assert.equal(aliceDs.response.status, 201, JSON.stringify(aliceDs.body)); const aliceSessionId = crypto.randomUUID(); - const aliceTitle = await patchSessionTitle(aliceSessionId, "Alice isolated session"); + const aliceTitle = await patchSessionTitle(aliceClient, aliceSessionId, "Alice isolated session"); assert.equal(aliceTitle.response.status, 200, JSON.stringify(aliceTitle.body)); - const aliceList = await requestJson("/api/v1/datasources"); + const aliceList = await aliceClient.fetchJson("/api/v1/datasources"); assert.equal(aliceList.response.status, 200); assert.equal(aliceList.body.data.some((item) => item.id === "alice-front-db"), true); - const aliceSessions = await requestJson("/api/v1/sessions?limit=20"); + const aliceSessions = await aliceClient.fetchJson("/api/v1/sessions?limit=20"); assert.equal(aliceSessions.response.status, 200); assert.equal( aliceSessions.body.data.sessions.some((item) => item.title === "Alice isolated session"), true, ); - const bob = await registerVerifyLogin(bobEmail, "Bob Front"); - const bobDs = await createDatasource("bob-front-db", "Bob Front DB", bobDb); + const bobClient = createAuthenticatedTestClient({ baseUrl }); + const bob = await bobClient.registerAndLogin({ + email: `bob-front-${stamp}@example.com`, + displayName: "Bob Front", + }); + const bobDs = await createDatasource(bobClient, "bob-front-db", "Bob Front DB", bobDb); assert.equal(bobDs.response.status, 201, JSON.stringify(bobDs.body)); - const bobList = await requestJson("/api/v1/datasources"); + const bobList = await bobClient.fetchJson("/api/v1/datasources"); assert.equal(bobList.response.status, 200); assert.equal(bobList.body.data.some((item) => item.id === "bob-front-db"), true); assert.equal(bobList.body.data.some((item) => item.id === "alice-front-db"), false); - const bobSessions = await requestJson("/api/v1/sessions?limit=20"); + const bobSessions = await bobClient.fetchJson("/api/v1/sessions?limit=20"); assert.equal(bobSessions.response.status, 200); assert.equal( bobSessions.body.data.sessions.some((item) => item.title === "Alice isolated session"), false, ); - const missingCsrf = await requestJson("/api/v1/datasources/bob-front-db", { + // Intentionally omit X-CSRF-Token while keeping the session cookie. + const missingCsrf = await fetch(`${baseUrl}/api/v1/datasources/bob-front-db`, { method: "DELETE", + headers: { + Cookie: `df_session=${encodeURIComponent(bob.cookies.df_session)}; df_csrf=${encodeURIComponent(bob.cookies.df_csrf)}`, + }, }); - assert.equal(missingCsrf.response.status, 403); - assert.equal(missingCsrf.body.error.code, "FORBIDDEN"); + const missingCsrfBody = await missingCsrf.json(); + assert.equal(missingCsrf.status, 403); + assert.equal(missingCsrfBody.error.code, "CSRF_INVALID"); - const me = await requestJson("/api/v1/me"); - assert.equal(me.response.status, 200); - assert.equal(me.body.data.user.email, bobEmail); - assert.notEqual(me.body.data.user.id, alice.user.id); + const me = await bobClient.verifyCurrentUser(); + assert.equal(me.user.email, bob.email); + assert.notEqual(me.user.id, alice.userId); console.log( - `Password frontend isolation smoke OK via ${baseUrl}: alice=${alice.user.id.slice(0, 8)} bob=${bob.user.id.slice(0, 8)}`, + `Password frontend isolation smoke OK via ${baseUrl}: alice=${alice.userId.slice(0, 8)} bob=${bob.userId.slice(0, 8)}`, ); } catch (error) { console.error(error); diff --git a/scripts/smoke-server-datasources-e2e.mjs b/scripts/smoke-server-datasources-e2e.mjs index 592dd4d5..32fe3421 100644 --- a/scripts/smoke-server-datasources-e2e.mjs +++ b/scripts/smoke-server-datasources-e2e.mjs @@ -7,8 +7,14 @@ import { createServer as createApiServer } from "../apps/api/dist/server.js"; import { createTaskStateRuntime } from "../packages/agent-runtime/dist/index.js"; import { LocalDataGateway } from "../packages/data-gateway/dist/index.js"; import { createMetadataStore } from "../packages/metadata/dist/index.js"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const root = mkdtempSync(join(tmpdir(), "open-data-foundry-server-datasources-e2e-")); +process.env.DATAFOUNDRY_AUTH_MODE = "password"; +process.env.AUTH_SESSION_SECRET = "server-datasources-e2e-session-secret-32b!"; +process.env.AUTH_PUBLIC_BASE_URL = "http://127.0.0.1:3000"; +process.env.AUTH_EMAIL_DELIVERY = "test"; +process.env.AUTH_REGISTRATION_MODE = "open"; process.env.STORAGE_ROOT_DIR = root; process.env.MASTRA_STORAGE_PATH = join(root, "mastra.sqlite"); process.env.EMBEDDING_API_KEY = ""; @@ -45,6 +51,9 @@ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const address = server.address(); assert(address && typeof address === "object"); const baseUrl = `http://127.0.0.1:${address.port}`; +const client = createAuthenticatedTestClient({ baseUrl }); +const identity = await client.registerAndLogin({ displayName: "Server Datasources E2E" }); +const { userId, workspaceId } = identity; try { for (const target of targets) { @@ -84,11 +93,12 @@ async function verifyTarget(target) { assert.equal(schemaResponse.response.status, 200, JSON.stringify(schemaResponse.body)); assert(Array.isArray(schemaResponse.body.data.tables), `${target.type} schema tables missing`); - const schema = await dataGateway.inspectSchema({ user_id: "dev-user", datasource_id: datasourceId }); + const schema = await dataGateway.inspectSchema({ user_id: userId, workspace_id: workspaceId, datasource_id: datasourceId }); assert(Array.isArray(schema.tables), `${target.type} inspectSchema failed`); const result = await dataGateway.runSqlReadonly({ - user_id: "dev-user", + user_id: userId, + workspace_id: workspaceId, datasource_id: datasourceId, sql: target.sql, limit: 10 @@ -123,7 +133,7 @@ function serverTarget(type, prefix, overrides = {}) { } async function requestJson(path, init = {}) { - const response = await fetch(`${baseUrl}${path}`, init); + const response = await client.fetch(path, init); const body = await response.json(); return { body, response }; } diff --git a/scripts/stack-runner.mjs b/scripts/stack-runner.mjs new file mode 100644 index 00000000..3f0f5cbd --- /dev/null +++ b/scripts/stack-runner.mjs @@ -0,0 +1,153 @@ +import { execFileSync, execSync, spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { loadEnvFile } from "node:process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { datalinkEnabled } from "./datalink-stack-config.mjs"; +import { + formatStackEndpoints, + resolveStackRuntimeConfig, + webProcessEnvironment, +} from "./stack-runtime-config.mjs"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); + +export async function runStack({ mode, args = [] }) { + loadRootEnv(); + const apiOnly = args.includes("--api"); + const webOnly = args.includes("--web"); + const startApi = !webOnly || apiOnly; + const startWeb = !apiOnly || webOnly; + if (datalinkEnabled()) { + throw new Error( + "DATALINK_ENABLED=true is no longer supported: the bundled DataLink service was removed. Set DATALINK_ENABLED=false (or unset it).", + ); + } + const startDatalink = false; + const runtimeConfig = resolveStackRuntimeConfig(); + const datalinkEnv = runtimeConfig; + + if (mode === "development") { + execSync("node scripts/ensure-dev-environment.mjs", { + cwd: root, + stdio: "inherit", + env: process.env, + shell: true, + }); + const ports = [ + ...(startApi ? [Number(runtimeConfig.API_PORT)] : []), + ...(startWeb ? [Number(runtimeConfig.WEB_PORT)] : []), + ...(startDatalink + ? [Number(datalinkEnv.DATALINK_MCP_PORT), Number(datalinkEnv.DATALINK_API_PORT)] + : []), + ]; + for (const port of ports) freePort(port); + } + + if (startDatalink) ensureUvAvailable(); + + const children = []; + if (startDatalink) { + children.push(spawnProcess("DataLink MCP", "uv", [ + "run", "--project", "services/datalink", "datalink", "serve", + "--host", datalinkEnv.DATALINK_MCP_HOST, + "--port", datalinkEnv.DATALINK_MCP_PORT, + "--transport", "streamable-http", + "--db", datalinkEnv.DATALINK_GRAPH_DB_PATH, + ], datalinkEnv)); + children.push(spawnProcess("DataLink REST", "uv", [ + "run", "--project", "services/datalink", "datalink", "api", + "--host", datalinkEnv.DATALINK_API_HOST, + "--port", datalinkEnv.DATALINK_API_PORT, + ], datalinkEnv)); + } + if (startApi) { + const command = mode === "development" + ? ["--workspace", "@datafoundry/api", "run", "dev"] + : ["--prefix", "apps/api", "run", "start"]; + children.push(spawnProcess("DataFoundry API", "npm", command, datalinkEnv)); + } + if (startWeb) { + const webScript = mode === "development" ? "dev" : "start"; + const command = mode === "development" + ? ["--workspace", "@datafoundry/web", "run", webScript] + : ["--prefix", "apps/web", "run", webScript]; + const webEnv = { ...datalinkEnv, ...webProcessEnvironment(runtimeConfig) }; + children.push(spawnProcess("DataFoundry Web", "npm", command, webEnv)); + } + + if (children.length === 0) { + throw new Error("Nothing to start. Use --api and/or --web."); + } + + console.log(formatStackEndpoints(runtimeConfig, { startApi, startDatalink, startWeb })); + let shuttingDown = false; + const shutdown = (signal) => { + if (shuttingDown) return; + shuttingDown = true; + for (const { child } of children) { + if (!child.killed) child.kill(signal); + } + }; + + process.on("SIGINT", () => shutdown("SIGINT")); + process.on("SIGTERM", () => shutdown("SIGTERM")); + for (const { child, label } of children) { + child.on("exit", (code, signal) => { + if (shuttingDown || signal) return; + console.error(`[stack] ${label} exited with code ${code ?? "unknown"}.`); + shutdown("SIGTERM"); + process.exitCode = code && code !== 0 ? code : 1; + }); + } +} + +function loadRootEnv() { + const envPath = join(root, ".env"); + if (existsSync(envPath)) loadEnvFile(envPath); +} + +function ensureUvAvailable() { + try { + execFileSync("uv", ["--version"], { stdio: "ignore" }); + } catch { + throw new Error( + "DATALINK_ENABLED=true requires uv and Python 3.10+. Install uv, then run `npm run install:datalink`.", + ); + } +} + +function spawnProcess(label, command, args, env) { + const child = spawn(command, args, { + cwd: root, + stdio: "inherit", + env, + shell: process.platform === "win32", + }); + child.on("error", (error) => console.error(`[stack] Unable to start ${label}: ${error.message}`)); + return { child, label }; +} + +function freePort(port) { + try { + if (process.platform === "win32") { + const output = execSync(`netstat -ano | findstr :${port}`, { + encoding: "utf8", + shell: true, + stdio: ["ignore", "pipe", "ignore"], + }); + const pids = new Set(); + for (const line of output.split(/\r?\n/u)) { + if (!/\bLISTENING\b/u.test(line)) continue; + const pid = line.trim().split(/\s+/u).at(-1); + if (pid && /^\d+$/u.test(pid) && pid !== "0") pids.add(pid); + } + for (const pid of pids) execSync(`taskkill /F /PID ${pid}`, { stdio: "ignore", shell: true }); + return; + } + execSync(`fuser -k ${port}/tcp 2>/dev/null || true`, { cwd: root, stdio: "ignore", shell: true }); + } catch { + // The port was already free. + } +} diff --git a/scripts/stack-runtime-config.mjs b/scripts/stack-runtime-config.mjs new file mode 100644 index 00000000..a3dca474 --- /dev/null +++ b/scripts/stack-runtime-config.mjs @@ -0,0 +1,34 @@ +import { resolveDatalinkEnv } from "./datalink-stack-config.mjs"; + +function port(value, fallback, name) { + const parsed = Number(value ?? fallback); + if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) { + throw new Error(`${name} must be an integer between 1 and 65535`); + } + return String(parsed); +} + +export function resolveStackRuntimeConfig(env = process.env) { + return { + ...resolveDatalinkEnv(process.cwd(), env), + API_HOST: env.API_HOST?.trim() || "127.0.0.1", + API_PORT: port(env.API_PORT, 8787, "API_PORT"), + WEB_HOST: env.WEB_HOST?.trim() || "0.0.0.0", + WEB_PORT: port(env.WEB_PORT, 3000, "WEB_PORT") + }; +} + +export function webProcessEnvironment(config) { + return { HOSTNAME: config.WEB_HOST, PORT: config.WEB_PORT }; +} + +export function formatStackEndpoints(config, enabled) { + const lines = ["DataFoundry endpoints:"]; + if (enabled.startWeb) lines.push(` Web: http://127.0.0.1:${config.WEB_PORT}`); + if (enabled.startApi) lines.push(` API: http://${config.API_HOST}:${config.API_PORT}`); + if (enabled.startDatalink) { + lines.push(` DataLink MCP: http://${config.DATALINK_MCP_HOST}:${config.DATALINK_MCP_PORT}/mcp`); + lines.push(` DataLink REST: http://${config.DATALINK_API_HOST}:${config.DATALINK_API_PORT}`); + } + return lines.join("\n"); +} diff --git a/scripts/stack-runtime-config.test.mjs b/scripts/stack-runtime-config.test.mjs new file mode 100644 index 00000000..654a6a74 --- /dev/null +++ b/scripts/stack-runtime-config.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + formatStackEndpoints, + resolveStackRuntimeConfig, + webProcessEnvironment +} from "./stack-runtime-config.mjs"; + +test("uses native deployment defaults", () => { + const config = resolveStackRuntimeConfig({ DATALINK_ENABLED: "false" }); + assert.equal(config.API_HOST, "127.0.0.1"); + assert.equal(config.API_PORT, "8787"); + assert.equal(config.WEB_HOST, "0.0.0.0"); + assert.equal(config.WEB_PORT, "3000"); +}); + +test("projects configured host and port into the Next.js child", () => { + const config = resolveStackRuntimeConfig({ WEB_HOST: "127.0.0.1", WEB_PORT: "3310" }); + assert.deepEqual(webProcessEnvironment(config), { HOSTNAME: "127.0.0.1", PORT: "3310" }); +}); + +test("prints actual configured endpoints", () => { + const config = resolveStackRuntimeConfig({ API_PORT: "8877", WEB_PORT: "3310" }); + const output = formatStackEndpoints(config, { startApi: true, startDatalink: false, startWeb: true }); + assert.match(output, /http:\/\/127\.0\.0\.1:8877/); + assert.match(output, /http:\/\/127\.0\.0\.1:3310/); +}); + +test("rejects invalid ports", () => { + assert.throws(() => resolveStackRuntimeConfig({ WEB_PORT: "70000" }), /WEB_PORT/); +}); diff --git a/scripts/start.mjs b/scripts/start.mjs new file mode 100755 index 00000000..f05efd57 --- /dev/null +++ b/scripts/start.mjs @@ -0,0 +1,4 @@ +#!/usr/bin/env node +import { runStack } from "./stack-runner.mjs"; + +await runStack({ mode: "production", args: process.argv.slice(2) }); diff --git a/scripts/test-builtin-dtc-growth-datasource.mjs b/scripts/test-builtin-dtc-growth-datasource.mjs index aae7a342..456b7b9b 100644 --- a/scripts/test-builtin-dtc-growth-datasource.mjs +++ b/scripts/test-builtin-dtc-growth-datasource.mjs @@ -14,6 +14,7 @@ import { import { createServer as createApiServer } from "../apps/api/dist/server.js"; import { LocalDataGateway } from "../packages/data-gateway/dist/index.js"; import { createMetadataStore } from "../packages/metadata/dist/index.js"; +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const repoFixture = join(repoRoot, "storage/fixtures/dtc-growth-demo.sqlite"); @@ -130,6 +131,11 @@ test("ensureBuiltinDtcGrowthDatasource copies fixture and registers datasource i test("createServer auto-provisions DTC Growth Review and test-connect works", async () => { assert.ok(existsSync(repoFixture), `fixture missing: ${repoFixture}`); const root = mkdtempSync(join(tmpdir(), "dtc-growth-server-")); + process.env.DATAFOUNDRY_AUTH_MODE = "password"; + process.env.AUTH_SESSION_SECRET = "dtc-growth-server-session-secret-32b!"; + process.env.AUTH_PUBLIC_BASE_URL = "http://127.0.0.1:3000"; + process.env.AUTH_EMAIL_DELIVERY = "test"; + process.env.AUTH_REGISTRATION_MODE = "open"; process.env.STORAGE_ROOT_DIR = root; process.env.WORKSPACE_ROOT = join(root, "workspaces"); process.env.MASTRA_STORAGE_PATH = join(root, "mastra.sqlite"); @@ -140,15 +146,18 @@ test("createServer auto-provisions DTC Growth Review and test-connect works", as database_path: join(root, "metadata.sqlite"), secret_master_key: "dtc-growth-server-test-key" }); - const dataGateway = new LocalDataGateway(metadataStore, { workspaceId: "default" }); + const dataGateway = new LocalDataGateway(metadataStore); const server = await createApiServer({ metadataStore }); await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); const address = server.address(); assert(address && typeof address === "object"); const baseUrl = `http://127.0.0.1:${address.port}`; + const client = createAuthenticatedTestClient({ baseUrl }); + const identity = await client.registerAndLogin({ displayName: "Builtin DTC Growth" }); + const { userId, workspaceId } = identity; try { - const listed = await fetch(`${baseUrl}/api/v1/datasources`); + const listed = await client.fetch("/api/v1/datasources"); assert.equal(listed.status, 200); const listedBody = await listed.json(); const items = listedBody.data ?? listedBody; @@ -159,7 +168,7 @@ test("createServer auto-provisions DTC Growth Review and test-connect works", as assert.equal(dtc.builtin, true); assert.ok(typeof dtc.config?.path === "string" && existsSync(dtc.config.path)); - const testConnect = await fetch(`${baseUrl}/api/v1/datasources/${DTC_GROWTH_DATASOURCE_ID}/test`, { + const testConnect = await client.fetch(`/api/v1/datasources/${DTC_GROWTH_DATASOURCE_ID}/test`, { method: "POST" }); const testBody = await testConnect.json(); @@ -168,8 +177,8 @@ test("createServer auto-provisions DTC Growth Review and test-connect works", as // Gateway query against provisioned path. const schema = await dataGateway.inspectSchema({ - user_id: "dev-user", - workspace_id: "default", + user_id: userId, + workspace_id: workspaceId, datasource_id: DTC_GROWTH_DATASOURCE_ID }); const tableNames = schema.tables.map((table) => table.name).sort(); @@ -177,7 +186,7 @@ test("createServer auto-provisions DTC Growth Review and test-connect works", as assert.ok(tableNames.includes("ad_spend")); // Second list remains single datasource (idempotent via memo + ensure). - const listedAgain = await fetch(`${baseUrl}/api/v1/datasources`); + const listedAgain = await client.fetch("/api/v1/datasources"); const againBody = await listedAgain.json(); const againItems = againBody.data ?? againBody; const dtcCount = (Array.isArray(againItems) ? againItems : []).filter( diff --git a/scripts/verify-token-usage-display.mjs b/scripts/verify-token-usage-display.mjs index a2a5dac0..40b2926b 100644 --- a/scripts/verify-token-usage-display.mjs +++ b/scripts/verify-token-usage-display.mjs @@ -2,6 +2,7 @@ * Live API + frontend state verification for token_usage display. * Replays AG-UI events through live-run-state and prints overview/detail token stats. */ +import { createAuthenticatedTestClient } from "./lib/authenticated-test-client.mjs"; import { createInitialLiveRun, deriveRunUsage, @@ -10,10 +11,12 @@ import { } from "../apps/web/src/app/data-tasks/live-run-state.ts"; const API_BASE = process.env.API_BASE_URL ?? "http://127.0.0.1:8787"; +const client = createAuthenticatedTestClient({ baseUrl: API_BASE }); +await client.registerAndLogin({ displayName: "Token Usage Verify" }); const threadId = `token-verify-${Date.now()}`; const runId = `token-verify-run-${Date.now()}`; -const response = await fetch(`${API_BASE}/api/copilotkit`, { +const response = await client.fetch("/api/copilotkit", { method: "POST", headers: { Accept: "text/event-stream",