From 2cfab6e47b036541e9b7cef62929bc37f382da5c Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Mon, 12 Oct 2026 11:08:51 -0700 Subject: [PATCH 1/2] feat: count template installs and tag runtime telemetry Send one unsampled event when an installation first reaches the runtime and one when its first agent run finishes with an answer, through the CopilotKit telemetry ingest. A random installation id is stored locally and both events respect DO_NOT_TRACK and COPILOTKIT_TELEMETRY_DISABLED. Also pass the same metadata to the runtime's own sampled telemetry through properties._copilotkit, so this template's runs can be told apart from other CopilotKit apps. --- .gitignore | 3 + apps/app/src/app/api/copilotkit/route.ts | 17 +- .../lib/__tests__/launch-telemetry.test.ts | 249 ++++++++++++++++++ apps/app/src/lib/launch-telemetry.ts | 243 +++++++++++++++++ 4 files changed, 510 insertions(+), 2 deletions(-) create mode 100644 apps/app/src/lib/__tests__/launch-telemetry.test.ts create mode 100644 apps/app/src/lib/launch-telemetry.ts diff --git a/.gitignore b/.gitignore index b352c66..cb30952 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,9 @@ bun.lockb # Turbo .turbo +# Telemetry installation id (docs/telemetry.md) +.open-intelligent-ui + # Tools .claude diff --git a/apps/app/src/app/api/copilotkit/route.ts b/apps/app/src/app/api/copilotkit/route.ts index ee3bfc3..f16873b 100644 --- a/apps/app/src/app/api/copilotkit/route.ts +++ b/apps/app/src/app/api/copilotkit/route.ts @@ -3,9 +3,10 @@ import { ExperimentalEmptyAdapter, copilotRuntimeNextJSAppRouterEndpoint, } from "@copilotkit/runtime"; -import { NextRequest } from "next/server"; +import { NextRequest, after } from "next/server"; import { readProviderHeaders, PROVIDER_KEY_INPUT_ERROR } from "@/lib/provider-keys"; import { buildRuntimeOptions } from "@/lib/copilotkit-runtime-options"; +import { getLaunchTelemetry, observeRunResult, runtimeMethod } from "@/lib/launch-telemetry"; // Simple in-memory sliding-window rate limiter (per IP) // Enable via RATE_LIMIT_ENABLED=true — off by default. @@ -55,11 +56,23 @@ export const POST = async (req: NextRequest) => { providerHeaders, }); + // Install counting (docs/telemetry.md). Only the first run's request body is inspected. + const telemetry = getLaunchTelemetry(); + const telemetryProperties = telemetry.runtimeProperties(); + const watchRun = !telemetry.activated && (await runtimeMethod(req)) === "agent/run"; + const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({ endpoint: "/api/copilotkit", serviceAdapter: new ExperimentalEmptyAdapter(), runtime: new CopilotRuntime(runtimeOptions), + ...(telemetryProperties && { properties: { _copilotkit: telemetryProperties } }), }); - return handleRequest(req); + const response = await handleRequest(req); + after(telemetry.recordOpened()); + if (!watchRun || !response.body) return response; + + const [client, observer] = response.body.tee(); + after(observeRunResult(observer).then((ok) => (ok ? telemetry.recordActivated() : undefined))); + return new Response(client, { status: response.status, statusText: response.statusText, headers: response.headers }); }; diff --git a/apps/app/src/lib/__tests__/launch-telemetry.test.ts b/apps/app/src/lib/__tests__/launch-telemetry.test.ts new file mode 100644 index 0000000..3a59db3 --- /dev/null +++ b/apps/app/src/lib/__tests__/launch-telemetry.test.ts @@ -0,0 +1,249 @@ +// @vitest-environment node +import { mkdtempSync, readFileSync, existsSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { LaunchTelemetry, observeRunResult, runtimeMethod, telemetryDistribution } from "../launch-telemetry"; + +type SentBody = { + event: string; + properties: Record; + package: Record; + global_properties: Record; +}; +type Sent = { identity: string; body: SentBody }; + +let dir: string; +let statePath: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "oiui-telemetry-")); + statePath = join(dir, "state", "telemetry.json"); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +function recorder() { + const sent: Sent[] = []; + const send = vi.fn(async (identity: string, body: Record) => { + sent.push({ identity, body: body as SentBody }); + }); + return { sent, send }; +} + +describe("LaunchTelemetry", () => { + it("sends the opened event once per installation, with closed metadata", async () => { + const { sent, send } = recorder(); + const telemetry = new LaunchTelemetry({ env: {}, statePath, send }); + + await telemetry.recordOpened(); + await telemetry.recordOpened(); + + expect(sent).toHaveLength(1); + const [{ identity, body }] = sent; + expect(body.event).toBe("oss.runtime.setup_step_viewed"); + expect(body.properties).toEqual({ kind: "step_viewed", step: "ready" }); + expect(body.package).toEqual({ name: "open-intelligent-ui", version: expect.any(String) }); + expect(body.global_properties).toMatchObject({ + accessibility_title: "Open Intelligent UI", + openintelligentui_distribution: "self_hosted", + runtime_env: "node", + sampleRate: 1, + installation_id: identity, + }); + expect(Object.keys(body.global_properties).sort()).toEqual( + [ + "accessibility_title", + "installation_id", + "openintelligentui_distribution", + "openintelligentui_version", + "runtime_env", + "sampleRate", + "sampleRateAdjustmentFactor", + "sampleWeight", + "server_arch", + "server_platform", + ].sort(), + ); + }); + + it("keeps the installation id across restarts", async () => { + const first = recorder(); + await new LaunchTelemetry({ env: {}, statePath, send: first.send }).recordOpened(); + const second = recorder(); + const restarted = new LaunchTelemetry({ env: {}, statePath, send: second.send }); + await restarted.recordOpened(); + await restarted.recordActivated(); + + expect(second.sent).toHaveLength(1); + expect(second.sent[0].body.event).toBe("oss.runtime.setup_activated"); + expect(second.sent[0].identity).toBe(first.sent[0].identity); + }); + + it("sends the activated event once", async () => { + const { sent, send } = recorder(); + const telemetry = new LaunchTelemetry({ env: {}, statePath, send }); + + await Promise.all([telemetry.recordActivated(), telemetry.recordActivated()]); + await telemetry.recordActivated(); + + expect(sent.map((s) => s.body.event)).toEqual(["oss.runtime.setup_activated"]); + expect(sent[0].body.properties).toEqual({ kind: "activated" }); + }); + + it("retries after a failed delivery and never throws", async () => { + const send = vi + .fn<(identity: string, body: Record) => Promise>() + .mockRejectedValueOnce(new Error("offline")) + .mockResolvedValue(undefined); + const telemetry = new LaunchTelemetry({ env: {}, statePath, send, report: () => {} }); + + await expect(telemetry.recordOpened()).resolves.toBeUndefined(); + await telemetry.recordOpened(); + await telemetry.recordOpened(); + + expect(send).toHaveBeenCalledTimes(2); + }); + + it.each([ + ["DO_NOT_TRACK", "1"], + ["DO_NOT_TRACK", "true"], + ["COPILOTKIT_TELEMETRY_DISABLED", "1"], + ["COPILOTKIT_TELEMETRY_DISABLED", "true"], + ["COPILOTKIT_TELEMETRY_SAMPLE_RATE", "0"], + ])("sends nothing and stores nothing when %s=%s", async (key, value) => { + const { send } = recorder(); + const telemetry = new LaunchTelemetry({ env: { [key]: value }, statePath, send }); + + await telemetry.recordOpened(); + await telemetry.recordActivated(); + + expect(send).not.toHaveBeenCalled(); + expect(existsSync(statePath)).toBe(false); + expect(telemetry.runtimeProperties()).toBeUndefined(); + }); + + it("deletes stored state after opting out", async () => { + await new LaunchTelemetry({ env: {}, statePath, send: recorder().send }).recordOpened(); + expect(existsSync(statePath)).toBe(true); + + new LaunchTelemetry({ env: { DO_NOT_TRACK: "1" }, statePath, send: recorder().send }); + + expect(existsSync(statePath)).toBe(false); + }); + + it("uses CPK_TELEMETRY_ID as the identity but keeps the installation id separate", async () => { + const { sent, send } = recorder(); + const telemetry = new LaunchTelemetry({ env: { CPK_TELEMETRY_ID: "cli-account-1" }, statePath, send }); + + await telemetry.recordOpened(); + + expect(sent[0].identity).toBe("cli-account-1"); + expect(sent[0].body.global_properties.installation_id).not.toBe("cli-account-1"); + }); + + it("stores only the installation id and delivery flags", async () => { + await new LaunchTelemetry({ env: {}, statePath, send: recorder().send }).recordOpened(); + + const saved = JSON.parse(readFileSync(statePath, "utf8")); + expect(Object.keys(saved).sort()).toEqual(["installationId", "sent"]); + expect(saved.sent).toEqual({ opened: true }); + }); + + it("tags the runtime's own telemetry with the same metadata", () => { + const telemetry = new LaunchTelemetry({ env: { OIUI_DISTRIBUTION: "hosted_demo" }, statePath, send: recorder().send }); + + expect(telemetry.runtimeProperties()).toMatchObject({ + template: "open-intelligent-ui", + openintelligentui_distribution: "hosted_demo", + installation_id: expect.any(String), + }); + }); + + it("still works when the state file can't be written", async () => { + const { sent, send } = recorder(); + const blocked = join(dir, "file"); + await import("node:fs").then((fs) => fs.writeFileSync(blocked, "not a directory")); + const telemetry = new LaunchTelemetry({ env: {}, statePath: join(blocked, "telemetry.json"), send, report: () => {} }); + + await expect(telemetry.recordOpened()).resolves.toBeUndefined(); + expect(sent).toHaveLength(1); + }); +}); + +describe("telemetryDistribution", () => { + it("accepts the known values and defaults to self_hosted", () => { + expect(telemetryDistribution({ OIUI_DISTRIBUTION: "hosted_demo" })).toBe("hosted_demo"); + expect(telemetryDistribution({ OIUI_DISTRIBUTION: "development" })).toBe("development"); + expect(telemetryDistribution({ OIUI_DISTRIBUTION: "anything else" })).toBe("self_hosted"); + expect(telemetryDistribution({})).toBe("self_hosted"); + }); +}); + +describe("runtimeMethod", () => { + it("reads the method without consuming the request body", async () => { + const req = new Request("http://localhost/api/copilotkit", { + method: "POST", + body: JSON.stringify({ method: "agent/run", params: { agentId: "default" } }), + }); + + await expect(runtimeMethod(req)).resolves.toBe("agent/run"); + await expect(req.json()).resolves.toMatchObject({ method: "agent/run" }); + }); + + it("returns undefined for bodies that aren't JSON", async () => { + const req = new Request("http://localhost/api/copilotkit", { method: "POST", body: "not json" }); + await expect(runtimeMethod(req)).resolves.toBeUndefined(); + }); +}); + +describe("observeRunResult", () => { + const encoder = new TextEncoder(); + function sse(...events: Record[]) { + return events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(""); + } + async function run(chunks: string[]) { + const stream = new ReadableStream({ + start(controller) { + chunks.forEach((c) => controller.enqueue(encoder.encode(c))); + controller.close(); + }, + }); + return observeRunResult(stream); + } + + it("reports success when an assistant answer streams and the run finishes", async () => { + const body = sse( + { type: "RUN_STARTED" }, + { type: "TEXT_MESSAGE_START", role: "assistant", messageId: "m1" }, + { type: "TEXT_MESSAGE_CONTENT", messageId: "m1", delta: "Hello" }, + { type: "RUN_FINISHED" }, + ); + // Split mid-token to prove chunk boundaries don't matter. + await expect(run([body.slice(0, 57), body.slice(57)])).resolves.toBe(true); + }); + + it("accepts streams of text chunks as well as bytes", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(sse({ type: "TEXT_MESSAGE_CONTENT", delta: "Hi" })); + controller.enqueue(sse({ type: "RUN_FINISHED" })); + controller.close(); + }, + }); + await expect(observeRunResult(stream)).resolves.toBe(true); + }); + + it("reports failure on a run error", async () => { + await expect( + run([sse({ type: "TEXT_MESSAGE_CONTENT", delta: "Hi" }, { type: "RUN_ERROR" }, { type: "RUN_FINISHED" })]), + ).resolves.toBe(false); + }); + + it("reports failure when the run finishes without any answer text", async () => { + await expect(run([sse({ type: "RUN_STARTED" }, { type: "RUN_FINISHED" })])).resolves.toBe(false); + }); +}); diff --git a/apps/app/src/lib/launch-telemetry.ts b/apps/app/src/lib/launch-telemetry.ts new file mode 100644 index 0000000..f58437b --- /dev/null +++ b/apps/app/src/lib/launch-telemetry.ts @@ -0,0 +1,243 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +import appPackage from "../../package.json"; + +// Counts how many people run this template, once per installation, through the +// same CopilotKit telemetry ingest that OpenDots uses. See docs/telemetry.md for +// exactly what is sent and how to turn it off. + +export const TELEMETRY_INGEST_URL = "https://telemetry.copilotkit.ai/ingest"; + +type Env = Record; +type Kind = "opened" | "activated"; +type Diagnostic = "delivery_failed" | "storage_failed"; +type Send = (identity: string, body: Record) => Promise; + +interface State { + installationId: string; + sent: Partial>; +} + +const DISTRIBUTIONS = ["self_hosted", "hosted_demo", "development"] as const; +export type Distribution = (typeof DISTRIBUTIONS)[number]; + +const EVENTS: Record }> = { + opened: { name: "oss.runtime.setup_step_viewed", properties: { kind: "step_viewed", step: "ready" } }, + activated: { name: "oss.runtime.setup_activated", properties: { kind: "activated" } }, +}; + +export function telemetryEnabled(env: Env): boolean { + const off = (key: string) => ["true", "1"].includes(env[key] ?? ""); + return !(off("DO_NOT_TRACK") || off("COPILOTKIT_TELEMETRY_DISABLED") || env.COPILOTKIT_TELEMETRY_SAMPLE_RATE === "0"); +} + +export function telemetryDistribution(env: Env): Distribution { + const value = env.OIUI_DISTRIBUTION as Distribution | undefined; + return value && DISTRIBUTIONS.includes(value) ? value : "self_hosted"; +} + +const serverPlatform = () => + process.platform === "darwin" ? "macos" : process.platform === "win32" ? "windows" : process.platform === "linux" ? "linux" : "other"; +const serverArch = () => (process.arch === "arm64" ? "aarch64" : process.arch === "x64" ? "x86_64" : "other"); + +const productionSend: Send = async (identity, body) => { + const response = await fetch(TELEMETRY_INGEST_URL, { + method: "POST", + headers: { "Content-Type": "application/json", "X-CopilotKit-Telemetry-Id": identity }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(1500), + }); + if (!response.ok) throw new Error("Telemetry delivery failed"); +}; + +export class LaunchTelemetry { + private readonly env: Env; + private readonly statePath: string; + private readonly send: Send; + private readonly report: (diagnostic: Diagnostic) => void; + private readonly enabled: boolean; + private state?: State; + private readonly inflight = new Map>(); + + constructor(options: { + env?: Env; + statePath?: string; + send?: Send; + report?: (diagnostic: Diagnostic) => void; + } = {}) { + this.env = options.env ?? process.env; + this.statePath = + options.statePath ?? this.env.OIUI_TELEMETRY_STATE_PATH ?? join(process.cwd(), ".open-intelligent-ui", "telemetry.json"); + this.send = options.send ?? productionSend; + this.report = + options.report ?? + ((diagnostic) => + console.error(JSON.stringify({ error: diagnostic, context: { component: "launch_telemetry" }, timestamp: new Date().toISOString() }))); + this.enabled = telemetryEnabled(this.env); + + if (!this.enabled) { + // Opting out also removes the stored installation id. + try { + rmSync(this.statePath, { force: true }); + } catch { + this.report("storage_failed"); + } + return; + } + this.state = this.load() ?? { installationId: randomUUID(), sent: {} }; + } + + /** Properties for the CopilotKit runtime's own telemetry (`properties._copilotkit`). */ + runtimeProperties(): Record | undefined { + if (!this.state) return undefined; + return { + template: "open-intelligent-ui", + openintelligentui_distribution: telemetryDistribution(this.env), + openintelligentui_version: appPackage.version, + installation_id: this.state.installationId, + }; + } + + /** Someone opened the app: its UI reached the runtime for the first time. */ + recordOpened(): Promise { + return this.record("opened"); + } + + /** The agent finished its first run with an answer. */ + recordActivated(): Promise { + return this.record("activated"); + } + + get activated(): boolean { + return !this.state || this.state.sent.activated === true; + } + + private record(kind: Kind): Promise { + const state = this.state; + if (!state || state.sent[kind]) return Promise.resolve(); + const pending = this.inflight.get(kind); + if (pending) return pending; + const delivery = this.deliver(kind, state).finally(() => this.inflight.delete(kind)); + this.inflight.set(kind, delivery); + return delivery; + } + + private async deliver(kind: Kind, state: State): Promise { + const event = EVENTS[kind]; + try { + await this.send(this.env.CPK_TELEMETRY_ID?.trim() || state.installationId, { + event: event.name, + event_id: randomUUID(), + properties: event.properties, + global_properties: this.metadata(state), + package: { name: "open-intelligent-ui", version: appPackage.version }, + ts: Math.floor(Date.now() / 1000), + }); + } catch { + this.report("delivery_failed"); + return; + } + state.sent[kind] = true; + this.persist(state); + } + + private metadata(state: State) { + return { + accessibility_title: "Open Intelligent UI", + openintelligentui_distribution: telemetryDistribution(this.env), + openintelligentui_version: appPackage.version, + runtime_env: "node", + server_platform: serverPlatform(), + server_arch: serverArch(), + installation_id: state.installationId, + sampleRate: 1, + sampleWeight: 1, + sampleRateAdjustmentFactor: 0, + }; + } + + private load(): State | undefined { + try { + const saved = JSON.parse(readFileSync(this.statePath, "utf8")); + if (typeof saved?.installationId === "string" && saved.installationId && typeof saved.sent === "object" && saved.sent) { + return { + installationId: saved.installationId, + sent: { ...(saved.sent.opened === true && { opened: true }), ...(saved.sent.activated === true && { activated: true }) }, + }; + } + } catch { + // Missing or unreadable state starts a new installation. + } + return undefined; + } + + private persist(state: State) { + try { + mkdirSync(dirname(this.statePath), { recursive: true }); + writeFileSync(this.statePath, JSON.stringify({ installationId: state.installationId, sent: state.sent })); + } catch { + this.report("storage_failed"); + } + } +} + +/** The single-route runtime method of a request (for example "agent/run"), read from a clone. */ +export async function runtimeMethod(req: Request): Promise { + try { + const body = await req.clone().json(); + return typeof body?.method === "string" ? body.method : undefined; + } catch { + return undefined; + } +} + +let shared: LaunchTelemetry | undefined; + +/** Lazily created so builds and imports never touch the filesystem. */ +export function getLaunchTelemetry(): LaunchTelemetry { + shared ??= new LaunchTelemetry(); + return shared; +} + +/** + * Reads an AG-UI event stream (server-sent events) and resolves true when the run + * finished with assistant text and no run error. No message content is retained. + */ +export async function observeRunResult(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let carry = ""; + let text = false; + let failed = false; + let finished = false; + + const inspect = (line: string) => { + if (!line.startsWith("data:")) return; + let event: { type?: string; delta?: string }; + try { + event = JSON.parse(line.slice(5)); + } catch { + return; + } + if (event.type === "TEXT_MESSAGE_CONTENT" && typeof event.delta === "string" && event.delta.trim()) text = true; + if (event.type === "RUN_ERROR") failed = true; + if (event.type === "RUN_FINISHED") finished = true; + }; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + carry += typeof value === "string" ? value : decoder.decode(value, { stream: true }); + const lines = carry.split("\n"); + carry = lines.pop() ?? ""; + lines.forEach((line) => inspect(line.trim())); + } + inspect(carry.trim()); + } catch { + return false; + } + return finished && text && !failed; +} From 5c27bcd693e7448b03976c1b0d9ec2104dcdad14 Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Mon, 12 Oct 2026 11:08:51 -0700 Subject: [PATCH 2/2] feat: add hosted-demo page analytics and document telemetry Load PostHog in the browser only when NEXT_PUBLIC_POSTHOG_KEY is set at build time, with session recording off because visitors type API keys into the app. Set OIUI_DISTRIBUTION=hosted_demo on Render so the demo is not counted as a self-hosted install, and document what is sent and how to turn it off. --- .env.example | 8 ++ README.md | 1 + apps/app/package.json | 1 + .../app/__tests__/posthog-analytics.test.tsx | 39 ++++++++ apps/app/src/app/posthog-analytics.tsx | 23 +++++ apps/app/src/app/providers.tsx | 2 + docs/README.md | 1 + docs/deployment.md | 5 +- docs/telemetry.md | 64 +++++++++++++ pnpm-lock.yaml | 94 +++++++++++++++++++ render.yaml | 6 ++ 11 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 apps/app/src/app/__tests__/posthog-analytics.test.tsx create mode 100644 apps/app/src/app/posthog-analytics.tsx create mode 100644 docs/telemetry.md diff --git a/.env.example b/.env.example index 351ad4e..29bbc80 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,14 @@ JEV_MODEL=jev-latest # LANGGRAPH_DEPLOYMENT_URL=http://localhost:8123 # MCP_SERVER_URL=http://localhost:3100/mcp +# Telemetry (docs/telemetry.md): anonymous install counts, on by default. +# Set either to turn it off. +# DO_NOT_TRACK=1 +# COPILOTKIT_TELEMETRY_DISABLED=1 +# OIUI_DISTRIBUTION=self_hosted +# Optional browser page analytics; leave unset unless you want them. +# NEXT_PUBLIC_POSTHOG_KEY= + # Rate limiting (per IP) — disabled by default RATE_LIMIT_ENABLED=false RATE_LIMIT_WINDOW_MS=60000 diff --git a/README.md b/README.md index 1f9055b..944bce4 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,7 @@ Built with CopilotKit. The `openGenerativeUI` runtime API and `generateSandboxed - [Documentation index](docs/README.md) - [Bring these patterns to your app](docs/bring-to-your-app.md) - [Source repository](https://github.com/CopilotKit/OpenIntelligentUI) +- [Telemetry and how to turn it off](docs/telemetry.md) ## Contributing diff --git a/apps/app/package.json b/apps/app/package.json index 4479587..ddb5dc0 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -20,6 +20,7 @@ "@jetbrains/websandbox": "1.2.1", "@repo/design-system": "workspace:*", "next": "16.1.6", + "posthog-js": "^1.438.8", "qrcode.react": "^4.2.0", "react": "^19.2.4", "react-dom": "^19.2.4", diff --git a/apps/app/src/app/__tests__/posthog-analytics.test.tsx b/apps/app/src/app/__tests__/posthog-analytics.test.tsx new file mode 100644 index 0000000..52ab334 --- /dev/null +++ b/apps/app/src/app/__tests__/posthog-analytics.test.tsx @@ -0,0 +1,39 @@ +import { render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const posthog = vi.hoisted(() => ({ __loaded: false, init: vi.fn(), register: vi.fn() })); +vi.mock("posthog-js", () => ({ default: posthog })); + +import { PostHogAnalytics } from "../posthog-analytics"; + +afterEach(() => { + vi.unstubAllEnvs(); + posthog.init.mockReset(); + posthog.register.mockReset(); + posthog.__loaded = false; +}); + +describe("PostHogAnalytics", () => { + it("loads nothing without a PostHog key", () => { + vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", ""); + render(); + expect(posthog.init).not.toHaveBeenCalled(); + }); + + it("starts PostHog with session recording off when a key is set", () => { + vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test"); + render(); + expect(posthog.init).toHaveBeenCalledWith( + "phc_test", + expect.objectContaining({ api_host: "https://eu.i.posthog.com", disable_session_recording: true }), + ); + expect(posthog.register).toHaveBeenCalledWith({ app: "open-intelligent-ui" }); + }); + + it("does not start twice", () => { + vi.stubEnv("NEXT_PUBLIC_POSTHOG_KEY", "phc_test"); + posthog.__loaded = true; + render(); + expect(posthog.init).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/app/src/app/posthog-analytics.tsx b/apps/app/src/app/posthog-analytics.tsx new file mode 100644 index 0000000..cc5ab17 --- /dev/null +++ b/apps/app/src/app/posthog-analytics.tsx @@ -0,0 +1,23 @@ +"use client"; + +import { useEffect } from "react"; +import posthog from "posthog-js"; + +// Page analytics for the hosted demo only. Nothing loads unless +// NEXT_PUBLIC_POSTHOG_KEY is set at build time, so clones of this template +// don't report page views. See docs/telemetry.md. +export function PostHogAnalytics() { + useEffect(() => { + const key = process.env.NEXT_PUBLIC_POSTHOG_KEY; + if (!key || posthog.__loaded) return; + posthog.init(key, { + api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST || "https://eu.i.posthog.com", + defaults: "2025-05-24", + person_profiles: "identified_only", + // Visitors type provider API keys into this app; never record sessions. + disable_session_recording: true, + }); + posthog.register({ app: "open-intelligent-ui" }); + }, []); + return null; +} diff --git a/apps/app/src/app/providers.tsx b/apps/app/src/app/providers.tsx index 595a6c6..ecb8e81 100644 --- a/apps/app/src/app/providers.tsx +++ b/apps/app/src/app/providers.tsx @@ -9,6 +9,7 @@ import { OpenGenUIPromptBridge } from "@/lib/sandbox/prompt-bridge"; import { tableCatalog } from "@/components/generative-ui/table-catalog"; import { ProviderKeysProvider, useProviderKeys } from "@/components/chat/provider-keys"; import { ThemeProvider } from "@/hooks/use-theme"; +import { PostHogAnalytics } from "./posthog-analytics"; const a2ui = { catalog: tableCatalog }; @@ -21,6 +22,7 @@ const openGenerativeUI = { export function Providers({ children }: { children: React.ReactNode }) { return ( + {children} diff --git a/docs/README.md b/docs/README.md index 11fcdb4..b5fc402 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ Local prerequisites: Node.js 22+, pnpm 9+, Python 3.12+, uv, an OpenAI key for t | [Visualization routing](visualization-routing.md) | Jev, A2UI tables, charts, diagrams, and maps | | [Architecture](architecture.md) | Agent, runtime, streaming renderer, and sandbox | | [Deployment](deployment.md) | Configuration and launch verification | +| [Telemetry](telemetry.md) | What usage data is sent and how to turn it off | | [Bring to your app](bring-to-your-app.md) | Adopt the integration patterns | | [Agent state](agent-state.md) | Shared state examples | | [Generative UI](generative-ui.md) | Native component examples | diff --git a/docs/deployment.md b/docs/deployment.md index 4dbb8a9..1bbf12d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -74,8 +74,9 @@ Use a reachable agent URL appropriate to your network. Expose the frontend throu 3. Check the chat layout at desktop and narrow widths. Exercise the composer, starter suggestions, and New chat. For generated tools, check keyboard controls, visible focus, numerical boundaries, and reset behavior. 4. Make real provider-backed requests for a direct text answer, an explainer, a comparison, and a working calculator. Verify streaming reaches the interactive final sandbox. Confirm current-data requests do not fabricate evidence. 5. Exercise a user-clicked follow-up with selected values. Confirm it starts a new turn/output, preserves earlier artifacts, and reports relevant errors. Check that ordinary controls do not send prompts. -6. Deploy the reviewed revision. Confirm both health endpoints return `{"status":"ok"}`, then repeat a real prompt against the deployed frontend. Health checks alone do not test provider authentication, streaming, or browser behavior. -7. Record the deployed revision, environment, checks actually performed, and remaining limitations. If validation fails, use the host's rollback workflow to restore a known working revision. +6. Check analytics before launch: set `OIUI_DISTRIBUTION` and `NEXT_PUBLIC_POSTHOG_KEY` on the frontend, then confirm a page view and the two install events arrive in PostHog (see [Telemetry](telemetry.md)). +7. Deploy the reviewed revision. Confirm both health endpoints return `{"status":"ok"}`, then repeat a real prompt against the deployed frontend. Health checks alone do not test provider authentication, streaming, or browser behavior. +8. Record the deployed revision, environment, checks actually performed, and remaining limitations. If validation fails, use the host's rollback workflow to restore a known working revision. A rendered chat interface alone is not a model smoke test; verify a real response. Do not describe local test success as a deployed launch. See [Getting started](getting-started.md) and [Architecture](architecture.md) for setup and runtime details. diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..c449c48 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,64 @@ +# Telemetry + +Open Intelligent UI reports anonymous usage so CopilotKit can see how many people run the template. It uses the same CopilotKit telemetry ingest as the CopilotKit runtime and OpenDots. You can turn it off with one environment variable. + +## Turning it off + +Set either variable on the frontend (`apps/app`) and restart: + +```bash +DO_NOT_TRACK=1 +# or +COPILOTKIT_TELEMETRY_DISABLED=1 +``` + +`COPILOTKIT_TELEMETRY_SAMPLE_RATE=0` also turns it off. Opting out deletes the stored installation id. + +## What is sent + +The frontend server sends at most two events per installation, each once: + +| Event | When | Properties | +| ------------------------------- | -------------------------------------------------------------------- | -------------------------------- | +| `oss.runtime.setup_step_viewed` | The app's UI first reaches the runtime (`/api/copilotkit`) | `kind=step_viewed`, `step=ready` | +| `oss.runtime.setup_activated` | The agent's first run finishes with an assistant answer and no error | `kind=activated` | + +Every event carries this metadata and nothing else: + +- `accessibility_title`: `Open Intelligent UI` +- `openintelligentui_distribution`: `self_hosted` by default, `hosted_demo` for CopilotKit's hosted demo, or `development` +- `openintelligentui_version`: the app's package version +- `runtime_env`: `node` +- `server_platform` (`macos`, `windows`, `linux`, `other`) and `server_arch` (`aarch64`, `x86_64`, `other`) +- `installation_id`: a random UUID created on first run + +No prompts, answers, API keys, URLs, file paths, model names, emails, or error details are sent. Activation is detected by reading the event types of the first run's response stream; no message text is kept. + +The installation id and two delivered/not-delivered flags are stored in `apps/app/.open-intelligent-ui/telemetry.json` (ignored by git). Set `OIUI_TELEMETRY_STATE_PATH` to store it elsewhere. If `CPK_TELEMETRY_ID` is set (by the CopilotKit CLI), it is used as the telemetry identity; the installation id stays separate. + +A failed delivery is retried on a later request. Delivery never blocks or fails a chat request. + +## Runtime telemetry + +The CopilotKit runtime already sends its own sampled, anonymous telemetry (see the CopilotKit docs). This app adds the same metadata (`template`, `openintelligentui_distribution`, `openintelligentui_version`, `installation_id`) to those events so they can be told apart from other CopilotKit apps. The opt-out variables above turn both off. + +## Hosted demo analytics + +The hosted demo loads PostHog page analytics in the browser. It only loads when `NEXT_PUBLIC_POSTHOG_KEY` is set at build time, so your own deployments and local runs don't load it unless you add a key. Session recording is always off, because visitors can type API keys into the app. `NEXT_PUBLIC_POSTHOG_HOST` defaults to `https://eu.i.posthog.com`. + +## Hosted demo setup + +On the Render frontend service (see `render.yaml`): + +- `OIUI_DISTRIBUTION=hosted_demo`, so the demo's runs are not counted as self-hosted installs. +- `NEXT_PUBLIC_POSTHOG_KEY`: the CopilotKit PostHog project key. It must be present when the app is built. + +## Checking delivery + +Unit tests intercept the transport and never send events: + +```bash +pnpm --filter @repo/app exec vitest run src/lib/__tests__/launch-telemetry.test.ts src/app/__tests__/posthog-analytics.test.tsx +``` + +To check live delivery, run the app with `OIUI_DISTRIBUTION=development`, open it and send a prompt, then look for the two events with that distribution in PostHog. HTTP acceptance by the ingest alone does not prove the events reached PostHog. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a8161fe..b159b96 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: next: specifier: 16.1.6 version: 16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + posthog-js: + specifier: ^1.438.8 + version: 1.438.8(@types/react@19.2.10)(react@19.2.4) qrcode.react: specifier: ^4.2.0 version: 4.2.0(react@19.2.4) @@ -1208,6 +1211,15 @@ packages: '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@posthog/browser-common@0.9.4': + resolution: {integrity: sha512-QlG0WZFG9iTp9cmXTypB4cgnxBAD3GImrj9AwmA84uh8uy5vIVgVLjoTTdOAW29XL+kh/eLn/5ZtBQt2JQfR+w==} + + '@posthog/core@1.58.0': + resolution: {integrity: sha512-vluXIlVF6WGeHoQbsQK8Y1ywXZlJDU7YHEKiXAArQU38yFBpgxxfD0kInbYHtura4Fr7Z7H8WiDFvtMpCGehGg==} + + '@posthog/types@1.415.2': + resolution: {integrity: sha512-BIIHPyXcPZuNTImrR1qLqLKFw7j9o8co6LG/9OpLtpWJ3RT0y89Lx6VCtd37N9/dkr/qW7y0uPjJxAxjsALHtw==} + '@preact/signals-core@1.14.2': resolution: {integrity: sha512-RZHdBj9ZF4n40Rp4jS052EHHjBWf96P9oNdXPfhQTovCuWY9iQn3Gq+gOTJSgBO9A/JBuPfMOWsSX/lIU9Pc/A==} @@ -2608,6 +2620,9 @@ packages: copy-to-clipboard@3.3.3: resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} + core-js@3.50.0: + resolution: {integrity: sha512-BRWgOLKkFeCgRudR6zrs8p9XJZcE14grzKMMssoYrk6krtuEZ7MTKPIY5RzOnqsEKIR9kst7wNzphttraT+Yqw==} + cors@2.8.6: resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} engines: {node: '>= 0.10'} @@ -2919,6 +2934,9 @@ packages: dompurify@3.3.1: resolution: {integrity: sha512-qkdCKzLNtrgPFP1Vo+98FRzJnBRGe4ffyCea9IwHB1fyxPOeNTHpLKYGd4Uk9xvNoH0ZoOjwZxNptyMwqrId1Q==} + dompurify@3.4.17: + resolution: {integrity: sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w==} + dset@3.1.4: resolution: {integrity: sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==} engines: {node: '>=4'} @@ -3232,6 +3250,9 @@ packages: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} + fflate@0.4.9: + resolution: {integrity: sha512-zdxgIEddhfsyCaWpJ2SdXEP8ZMrKJ6+5jl4OupODcywU0IhRk6gdXuVGcPICyfx2H97hVK7xmJtRLPjkxAX8Vw==} + file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} @@ -4746,6 +4767,25 @@ packages: resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} engines: {node: ^10 || ^12 || >=14} + posthog-js@1.438.8: + resolution: {integrity: sha512-nQpTyiVryD2C3hm3KWrslmqiEGSFqOclTq5bsmiBKHOsDAwvVQ2wgNCbpeCScYOn56h3/dgO/va+Zu1yOv2CCQ==} + peerDependencies: + '@types/react': '>=16.8.0' + react: '>=16.8.0' + peerDependenciesMeta: + '@types/react': + optional: true + react: + optional: true + + preact@10.29.8: + resolution: {integrity: sha512-ej2aVZ+vZ8WO7tvlQWRM9N63A0KzF9q4mWJfDUHgYaIofWY9hu74QdnQrjoPMmZi2/nZ5gN0bJCQF49xQqx09Q==} + peerDependencies: + preact-render-to-string: '>=5' + peerDependenciesMeta: + preact-render-to-string: + optional: true + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -4801,6 +4841,9 @@ packages: resolution: {integrity: sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==} engines: {node: '>=0.6'} + query-selector-shadow-dom@1.0.1: + resolution: {integrity: sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==} + queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -5814,6 +5857,12 @@ packages: resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} engines: {node: '>= 14'} + web-vitals@6.2.1: + resolution: {integrity: sha512-rLcLXA2sx6+9dE88NHFubwTtGxpK4yYBLj6qHPdFoCaLr0cXGb4efOqtKLlm4loGA4OEKHIQKMKzZkKyOh5ctw==} + + web-vitals@6.2.3: + resolution: {integrity: sha512-Dmy+vdO1UPMqXiAtZSNayhl4f794Bxfd94W4HIj2HlYtczEQfEOon969memlcpcHrTuOKLa5baFjX7WbgmOq+A==} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -7106,6 +7155,17 @@ snapshots: '@pinojs/redact@0.4.0': {} + '@posthog/browser-common@0.9.4': + dependencies: + '@posthog/core': 1.58.0 + '@posthog/types': 1.415.2 + + '@posthog/core@1.58.0': + dependencies: + '@posthog/types': 1.415.2 + + '@posthog/types@1.415.2': {} + '@preact/signals-core@1.14.2': {} '@protobuf-ts/protoc@2.11.1': {} @@ -8511,6 +8571,8 @@ snapshots: dependencies: toggle-selection: 1.0.6 + core-js@3.50.0: {} + cors@2.8.6: dependencies: object-assign: 4.1.1 @@ -8832,6 +8894,10 @@ snapshots: optionalDependencies: '@types/trusted-types': 2.0.7 + dompurify@3.4.17: + optionalDependencies: + '@types/trusted-types': 2.0.7 + dset@3.1.4: {} dunder-proto@1.0.1: @@ -9350,6 +9416,8 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 3.3.3 + fflate@0.4.9: {} + file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 @@ -11263,6 +11331,26 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + posthog-js@1.438.8(@types/react@19.2.10)(react@19.2.4): + dependencies: + '@posthog/browser-common': 0.9.4 + '@posthog/core': 1.58.0 + '@posthog/types': 1.415.2 + core-js: 3.50.0 + dompurify: 3.4.17 + fflate: 0.4.9 + preact: 10.29.8 + query-selector-shadow-dom: 1.0.1 + web-vitals: 6.2.3 + web-vitals-soft-navs: web-vitals@6.2.1 + optionalDependencies: + '@types/react': 19.2.10 + react: 19.2.4 + transitivePeerDependencies: + - preact-render-to-string + + preact@10.29.8: {} + prelude-ls@1.2.1: {} pretty-format@27.5.1: @@ -11313,6 +11401,8 @@ snapshots: dependencies: side-channel: 1.1.0 + query-selector-shadow-dom@1.0.1: {} + queue-microtask@1.2.3: {} quick-format-unescaped@4.0.4: {} @@ -12572,6 +12662,10 @@ snapshots: web-streams-polyfill@4.0.0-beta.3: {} + web-vitals@6.2.1: {} + + web-vitals@6.2.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@8.0.1: {} diff --git a/render.yaml b/render.yaml index b418147..7d1a87f 100644 --- a/render.yaml +++ b/render.yaml @@ -70,6 +70,12 @@ services: value: "60000" - key: RATE_LIMIT_MAX value: "40" + # Telemetry (docs/telemetry.md): count the hosted demo separately from self-hosted installs. + - key: OIUI_DISTRIBUTION + value: hosted_demo + # PostHog project key for hosted-demo page analytics; needed at build time. + - key: NEXT_PUBLIC_POSTHOG_KEY + sync: false buildFilter: paths: - apps/app/**