feat: integrate DataLink semantic service

.env.example

@@ -44,6 +44,24 @@ EMBEDDING_OUTPUT_TYPE=dense
 EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
 EMBEDDING_API_KEY=
 
+# First-party DataLink semantic service. Disabled by default so existing
+# DataFoundry deployments do not require Python or uv. Set true to let
+# `npm run dev` / `npm run start` launch REST + MCP with the main stack.
+DATALINK_ENABLED=false
+DATALINK_CONFIG_PATH=services/datalink/datalink_config.json
+DATALINK_GRAPH_DB_PATH=storage/datalink/datalink.db
+DATALINK_API_HOST=127.0.0.1
+DATALINK_API_PORT=8081
+DATALINK_MCP_HOST=127.0.0.1
+DATALINK_MCP_PORT=8080
+# Optional overrides. Empty values fall back to LLM_* / EMBEDDING_* above.
+DATALINK_LLM_MODEL=
+DATALINK_LLM_BASE_URL=
+DATALINK_LLM_API_KEY=
+DATALINK_EMBEDDING_MODEL=
+DATALINK_EMBEDDING_BASE_URL=
+DATALINK_EMBEDDING_API_KEY=
+
 # Agent execute_command sandbox: Python venv with numpy/pandas/matplotlib/scikit-learn.
 # Default auto-detects repo `.venv` (also checks npm INIT_CWD). Override with absolute path:
 # WORKSPACE_PYTHON_VENV=/path/to/DataFoundry/.venv

.github/workflows/ci.yml

@@ -79,6 +79,12 @@ jobs:
       - name: Run configuration API regression
         run: node scripts/smoke-config-api.mjs
 
+      - name: Run built-in DataLink lifecycle regression
+        run: node scripts/smoke-builtin-datalink.mjs
+
+      - name: Run DataLink stack configuration tests
+        run: npm run test:datalink-stack
+
       - name: Run file and workspace regression
         run: node scripts/smoke-files.mjs
 
@@ -97,6 +103,59 @@ jobs:
       - name: Run built-in DTC Growth regression
         run: node --test scripts/test-builtin-dtc-growth-datasource.mjs
 
+  datalink:
+    name: DataLink Service
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install DataLink dependencies
+        run: uv sync --project services/datalink --locked
+
+      - name: Lint DataLink
+        run: npm run lint:datalink
+
+      - name: Test DataLink
+        run: npm run test:datalink
+
+      - name: Build DataLink package
+        run: npm run build:datalink
+
+      - name: Smoke DataLink REST and MCP
+        shell: bash
+        run: |
+          set -euo pipefail
+          export DATALINK_CONFIG_PATH="$RUNNER_TEMP/datalink_config.json"
+          export DATALINK_GRAPH_DB_PATH="$RUNNER_TEMP/datalink.db"
+          uv run --project services/datalink datalink api --host 127.0.0.1 --port 8081 &
+          rest_pid=$!
+          uv run --project services/datalink datalink serve --host 127.0.0.1 --port 8080 --transport streamable-http --db "$DATALINK_GRAPH_DB_PATH" &
+          mcp_pid=$!
+          trap 'kill "$rest_pid" "$mcp_pid" 2>/dev/null || true' EXIT
+          for _ in $(seq 1 30); do
+            curl --fail --silent http://127.0.0.1:8081/healthz && break
+            sleep 1
+          done
+          curl --fail --silent http://127.0.0.1:8081/healthz
+          for _ in $(seq 1 30); do
+            status=$(curl --silent --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8080/mcp || true)
+            if [ "$status" != "000" ]; then break; fi
+            sleep 1
+          done
+          test "$status" != "000"
+
   docs:
     name: Docs
     runs-on: ubuntu-latest

README.md

@@ -63,7 +63,7 @@ DataFoundry 0.2 turns the first usable workbench into a more complete, stateful
 
 - **Branchable, concurrent analysis** — Keep multiple sessions running, queue follow-up prompts, restore completed work, and branch from an earlier question or checkpoint without overwriting the original path.
 - **Evidence-first follow-ups** — Reference a complete output or a selected table/text region in the next question; resolved evidence is carried into the governed run context with diagnostics.
-- **Semantic trace and first-party Data Link** — Inspect checkpoint-backed run structure in a semantic Trace DAG. With our newly open-sourced [Data Link](https://github.com/datagallery-lab/datalink), connect tables and columns to business concepts, entities, joinable paths, and confidence-scored relationships for stronger agent grounding.
+- **Semantic trace and built-in DataLink** — Inspect checkpoint-backed run structure in a semantic Trace DAG. The first-party DataLink service now ships in `services/datalink`, connecting tables and columns to business concepts, entities, joinable paths, and confidence-scored relationships for stronger agent grounding.
 - **Reusable outputs and workspace assets** — Preview and export tables, charts, reports, SQL, and files; upload files into an active session, then promote supported files for reuse across sessions.
 - **Production-facing Web foundation** — Built-in password authentication, same-origin API proxying, bilingual UI, model connection tests, onboarding, and an auto-provisioned DTC growth analysis case.
 
@@ -96,6 +96,10 @@ LLM_MODEL=qwen-plus                # or deepseek-chat, gpt-4o, ...
 LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
 LLM_API_KEY=replace-with-your-key
 
+# Optional: launch the built-in semantic service with the main stack.
+# Requires Python 3.10+ and uv; then run `npm run install:datalink` once.
+DATALINK_ENABLED=true
+
 DATAFOUNDRY_AUTH_MODE=password
 AUTH_SESSION_SECRET=replace-with-at-least-32-random-characters
 AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000   # real production: https://your.domain
@@ -116,10 +120,11 @@ Build and start:
 ```bash
 npm run build
 npm run build:web
-npm run start:api    # :8787  — /healthz liveness, /ready readiness
-npm run start:web    # :3000  — password mode via same-origin BFF
+npm run start        # Web :3000 + API :8787 (+ DataLink :8080/:8081 when enabled)
 ```
 
+For split-process deployment, `start:api` and `start:web` remain available; start DataLink separately with `start:datalink:mcp` and `start:datalink:api` when enabled.
+
 Open `http://127.0.0.1:3000/login`, register or sign in, go to `/data-tasks`, and ask:
 
 ```text

README_zh.md

@@ -63,7 +63,7 @@ DataFoundry 0.2 在首个可用版本上，进一步补齐了有状态、可追
 
 - **可分支的并发分析** — 多个会话可同时运行，运行中可排队后续问题；恢复历史后，可从早期问题或 checkpoint 创建新分支，不覆盖原分析路径。
 - **证据驱动的追问** — 可将完整产出，或选中的表格区域、文本片段引用到下一个问题；证据解析结果和诊断信息会进入受控 run context。
-- **语义 Trace 与自研 Data Link** — 通过基于 checkpoint 的语义 Trace DAG 复盘执行结构；结合我们新近开源的 [Data Link](https://github.com/datagallery-lab/datalink)，把表和字段连接到业务概念、实体、可 JOIN 路径和带置信度的关系，为 Agent 提供更强语义支撑。
+- **语义 Trace 与内置 DataLink** — 通过基于 checkpoint 的语义 Trace DAG 复盘执行结构；自研 DataLink 服务现已随仓库内置于 `services/datalink`，把表和字段连接到业务概念、实体、可 JOIN 路径和带置信度的关系，为 Agent 提供更强语义支撑。
 - **可复用的产出与工作区资产** — 统一预览和导出表格、图表、报告、SQL 和文件；文件可先上传到当前会话，再提升为跨会话复用的工作区资产。
 - **面向正式部署的 Web 基础** — 内置密码认证、同源 API 代理、中英双语界面、模型连接测试、首次引导，并自动准备 DTC 增长分析案例。
 
@@ -96,6 +96,10 @@ LLM_MODEL=qwen-plus                # 或 deepseek-chat、gpt-4o……
 LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
 LLM_API_KEY=replace-with-your-key
 
+# 可选：随主应用启动内置语义服务。
+# 需要 Python 3.10+ 与 uv，并先执行一次 `npm run install:datalink`。
+DATALINK_ENABLED=true
+
 DATAFOUNDRY_AUTH_MODE=password
 AUTH_SESSION_SECRET=replace-with-at-least-32-random-characters
 AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000   # 真实生产改为 https://你的域名
@@ -116,10 +120,11 @@ API_PROXY_TARGET=http://127.0.0.1:8787
 ```bash
 npm run build
 npm run build:web
-npm run start:api    # :8787  — /healthz 存活，/ready 就绪
-npm run start:web    # :3000  — password 模式走同源 BFF
+npm run start        # Web :3000 + API :8787（启用后含 DataLink :8080/:8081）
 ```
 
+需要拆分进程部署时，仍可使用 `start:api` 与 `start:web`；开启 DataLink 后再分别运行 `start:datalink:mcp` 和 `start:datalink:api`。
+
 打开 `http://127.0.0.1:3000/login` 注册登录后进入 `/data-tasks`，提问：
 
 ```text

apps/api/src/builtin-datalink-server.ts

@@ -0,0 +1,80 @@
+import type { MetadataStore } from "@datafoundry/metadata";
+
+export const BUILTIN_DATALINK_SERVER_ID = "builtin-datalink";
+
+const DATALINK_TOOL_MANIFEST = [{ name: "datalink_explore" }] as const;
+
+export const isBuiltinDatalinkEnabled = (env: NodeJS.ProcessEnv = process.env): boolean =>
+  ["1", "true", "yes", "on"].includes((env.DATALINK_ENABLED ?? "").trim().toLowerCase());
+
+export const ensureBuiltinDatalinkServer = (input: {
+  env?: NodeJS.ProcessEnv;
+  metadataStore: MetadataStore;
+  userId: string;
+  workspaceId: string;
+}): "created" | "removed" | "skipped" | "updated" => {
+  const env = input.env ?? process.env;
+  const key = {
+    id: BUILTIN_DATALINK_SERVER_ID,
+    workspace_id: input.workspaceId,
+    user_id: input.userId,
+    kind: "mcp-server" as const,
+  };
+  const current = input.metadataStore.configResources.find(key);
+  if (!isBuiltinDatalinkEnabled(env)) {
+    if (current?.builtin) {
+      input.metadataStore.configResources.delete(key);
+      return "removed";
+    }
+    return "skipped";
+  }
+
+  // Never overwrite a user-owned resource that happens to use the reserved id.
+  if (current && !current.builtin) return "skipped";
+
+  const payload = builtinDatalinkPayload(env);
+  if (
+    current
+    && current.name === "DataLink"
+    && current.description === "First-party DataFoundry semantic graph service."
+    && JSON.stringify(current.payload) === JSON.stringify(payload)
+  ) {
+    return "skipped";
+  }
+
+  input.metadataStore.configResources.upsert({
+    ...key,
+    name: "DataLink",
+    description: "First-party DataFoundry semantic graph service.",
+    payload,
+    default_enabled: current?.default_enabled ?? true,
+    builtin: true,
+    status: "untested",
+    ...(current ? { expected_revision: current.revision } : {}),
+  });
+  return current ? "updated" : "created";
+};
+
+const builtinDatalinkPayload = (env: NodeJS.ProcessEnv): Record<string, unknown> => {
+  const apiHost = connectHost(env.DATALINK_API_HOST);
+  const mcpHost = connectHost(env.DATALINK_MCP_HOST);
+  const apiPort = port(env.DATALINK_API_PORT, 8081);
+  const mcpPort = port(env.DATALINK_MCP_PORT, 8080);
+  return {
+    apiUrl: `http://${apiHost}:${apiPort}`,
+    managed: true,
+    serverUrl: `http://${mcpHost}:${mcpPort}/mcp`,
+    toolManifest: DATALINK_TOOL_MANIFEST,
+    transport: "streamable-http",
+  };
+};
+
+const connectHost = (value: string | undefined): string => {
+  const host = value?.trim() || "127.0.0.1";
+  return host === "0.0.0.0" || host === "::" ? "127.0.0.1" : host;
+};
+
+const port = (value: string | undefined, fallback: number): number => {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65535 ? parsed : fallback;
+};

apps/api/src/config-api.ts

@@ -55,6 +55,7 @@ import { basename, join, resolve, sep } from "node:path";
 import writeXlsxFile, { type SheetData } from "write-excel-file/node";
 
 import { resolveEffectiveRunConfig } from "./run-input.js";
+import { BUILTIN_DATALINK_SERVER_ID } from "./builtin-datalink-server.js";
 import {
   llmEnvFingerprint,
   modelProfileConnectivityPayloadChanged,
@@ -796,7 +797,7 @@ const handleDatalinkRequest = async (
   const target = segments[0];
   const action = segments[1];
   if (target === "servers" && request.method === "GET") {
-    return ok({ servers: listDatalinkServers(context) });
+    return ok({ servers: await listDatalinkServers(context) });
   }
   if (!target) {
     return methodNotAllowed();
@@ -867,12 +868,20 @@ type DatalinkApiCallResult = {
   text: string;
 };
 
-const listDatalinkServers = (context: Required<ConfigApiContext>): Record<string, unknown>[] =>
-  context.metadataStore.configResources.list({
+const listDatalinkServers = async (
+  context: Required<ConfigApiContext>
+): Promise<Record<string, unknown>[]> => {
+  const resources = context.metadataStore.configResources.list({
     workspace_id: context.workspaceId,
     user_id: context.userId,
     kind: "mcp-server"
-  }).filter(isDatalinkServer).map(datalinkServerDto);
+  }).filter(isDatalinkServer).sort((left, right) =>
+    Number(right.id === BUILTIN_DATALINK_SERVER_ID) - Number(left.id === BUILTIN_DATALINK_SERVER_ID)
+  );
+  return Promise.all(resources.map(async (resource) =>
+    datalinkServerDto(resource, await builtinDatalinkHealthStatus(resource))
+  ));
+};
 
 const getDatalinkServer = (
   context: Required<ConfigApiContext>,
@@ -903,13 +912,17 @@ const isDatalinkServer = (resource: ConfigResourceRecord): boolean => {
 const isDatalinkManifestToolName = (toolName: string): boolean =>
   toolName.startsWith("datalink_") || toolName.startsWith("datagraph_");
 
-const datalinkServerDto = (resource: ConfigResourceRecord): Record<string, unknown> => {
+const datalinkServerDto = (
+  resource: ConfigResourceRecord,
+  healthStatus = resource.status
+): Record<string, unknown> => {
   const tools = datalinkToolManifest(resource);
   return {
     id: resource.id,
     name: datalinkServerDisplayName(resource),
     description: resource.description ?? "",
-    healthStatus: resource.status,
+    healthStatus,
+    managed: resource.builtin && resource.payload.managed === true,
     serverUrl: stringValue(resource.payload.serverUrl) ?? stringValue(resource.payload.url) ?? "",
     apiUrl: stringValue(resource.payload.apiUrl) ?? "",
     transport: stringValue(resource.payload.transport) ?? "streamable-http",
@@ -919,6 +932,21 @@ const datalinkServerDto = (resource: ConfigResourceRecord): Record<string, unkno
   };
 };
 
+const builtinDatalinkHealthStatus = async (resource: ConfigResourceRecord): Promise<string> => {
+  if (resource.id !== BUILTIN_DATALINK_SERVER_ID || resource.payload.managed !== true) {
+    return resource.status;
+  }
+  try {
+    const response = await fetch(datalinkApiEndpoint(resource, "/healthz"), {
+      headers: { Accept: "application/json" },
+      signal: AbortSignal.timeout(1_000)
+    });
+    return response.ok ? "connected" : "unavailable";
+  } catch {
+    return "unavailable";
+  }
+};
+
 const datalinkServerDisplayName = (resource: ConfigResourceRecord): string => {
   if (resource.name.toLowerCase() === "datagraph") {
     return "datalink";
@@ -1636,6 +1664,9 @@ const handleGenericResourceRequest = async (
       user_id: context.userId,
       kind
     });
+    if (current.builtin && current.id === BUILTIN_DATALINK_SERVER_ID) {
+      throw new Error(`BUILTIN_RESOURCE_READONLY:${id}`);
+    }
     if (kind === "knowledge-base") {
       context.metadataStore.db.prepare(
         "DELETE FROM knowledge_embeddings WHERE user_id = ? AND collection_id = ?"
@@ -1687,6 +1718,13 @@ const saveConfigResourceInTransaction = (
     user_id: context.userId,
     kind
   });
+  if (current?.builtin && current.id === BUILTIN_DATALINK_SERVER_ID) {
+    const mutableKeys = new Set(["defaultEnabled", "revision"]);
+    const readonlyKeys = Object.keys(body).filter((key) => !mutableKeys.has(key));
+    if (readonlyKeys.length > 0) {
+      throw new Error(`BUILTIN_RESOURCE_READONLY:${id}`);
+    }
+  }
   if (current?.builtin && kind === "skill") {
     const mutableKeys = new Set(["defaultEnabled", "revision"]);
     const readonlyKeys = Object.keys(body).filter((key) => !mutableKeys.has(key));

apps/api/src/server.ts

@@ -40,6 +40,7 @@ import { Observable } from "rxjs";
 import { handleConfigApiRequest } from "./config-api.js";
 import { createAsyncMemoByKey, createStartupTimer } from "./async-memo.js";
 import { ensureBuiltinDtcGrowthDatasource } from "./builtin-dtc-growth-datasource.js";
+import { ensureBuiltinDatalinkServer } from "./builtin-datalink-server.js";
 import { reclaimOrphanedQueuedAndRunningRuns } from "./stale-active-runs.js";
 import { loadPasswordAuthConfig, type PasswordAuthConfig } from "./auth/config.js";
 import { AuthService, type AuthIdentity } from "./auth/service.js";
@@ -1261,6 +1262,11 @@ const ensureBuiltinConfigResources = async (
     userId,
     workspaceId
   });
+  ensureBuiltinDatalinkServer({
+    metadataStore,
+    userId,
+    workspaceId
+  });
 
   for (const source of BUILTIN_SKILL_SOURCES) {
     const content = readFileSync(source.path);

docs/en/architecture/overview.md

@@ -109,10 +109,11 @@ The default path is formal mode (`password` + `build` / `start`); do not run `np
 
 ```bash
 npm run build && npm run build:web
-npm run start:api
-npm run start:web
+npm run start
 ```
 
+`npm run start` manages Web and API with the same local-process deployment used previously. With `DATALINK_ENABLED=true`, it additionally starts the bundled Python DataLink REST and MCP processes; with the default `false`, the original two-process topology is unchanged. Separate `start:*` commands remain available for external process supervisors.
+
 | Environment | Email | Public URL |
 | --- | --- | --- |
 | Formal test | `AUTH_EMAIL_DELIVERY=test` | Local / private |