Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions server/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,20 @@ import { issueTreeControlRoutes } from "./routes/issue-tree-control.js";
import { routineRoutes } from "./routes/routines.js";
import { environmentRoutes } from "./routes/environments.js";
import { executionWorkspaceRoutes } from "./routes/execution-workspaces.js";
import { secretRoutes } from "./routes/secrets.js";
import { goalRoutes } from "./routes/goals.js";
import { approvalRoutes } from "./routes/approvals.js";
import { secretRoutes } from "./routes/secrets.js";
import { costRoutes } from "./routes/costs.js";
import { activityRoutes } from "./routes/activity.js";
import { agentAnalyticsRoutes } from "./routes/agent-analytics.js";
import { dashboardRoutes } from "./routes/dashboard.js";
import { userProfileRoutes } from "./routes/user-profiles.js";
import { sidebarBadgeRoutes } from "./routes/sidebar-badges.js";
import { sidebarPreferenceRoutes } from "./routes/sidebar-preferences.js";
import { inboxDismissalRoutes } from "./routes/inbox-dismissals.js";
import { instanceSettingsRoutes } from "./routes/instance-settings.js";
import {
instanceDatabaseBackupRoutes,
type InstanceDatabaseBackupService,
} from "./routes/instance-database-backups.js";
import { instanceDatabaseBackupRoutes,
type InstanceDatabaseBackupService,} from "./routes/instance-database-backups.js";
import { llmRoutes } from "./routes/llms.js";
import { authRoutes } from "./routes/auth.js";
import { assetRoutes } from "./routes/assets.js";
Expand All @@ -59,7 +58,6 @@ import { pluginRegistryService } from "./services/plugin-registry.js";
import { createHostClientHandlers } from "@paperclipai/plugin-sdk";
import type { BetterAuthSessionResult } from "./auth/better-auth.js";
import { createCachedViteHtmlRenderer } from "./vite-html-renderer.js";

type UiMode = "none" | "static" | "vite-dev";
const FEEDBACK_EXPORT_FLUSH_INTERVAL_MS = 5_000;
const VITE_DEV_ASSET_PREFIXES = [
Expand Down Expand Up @@ -205,6 +203,7 @@ export async function createApp(
api.use(secretRoutes(db));
api.use(costRoutes(db, { pluginWorkerManager: workerManager }));
api.use(activityRoutes(db));
api.use(agentAnalyticsRoutes(db));
api.use(dashboardRoutes(db));
api.use(userProfileRoutes(db));
api.use(sidebarBadgeRoutes(db));
Expand Down
26 changes: 26 additions & 0 deletions server/src/routes/agent-analytics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { Router } from "express";
import type { Db } from "@paperclipai/db";

import { agentAnalyticsService } from "../services/agent-analytics.js";
import { assertCompanyAccess } from "./authz.js";

export function agentAnalyticsRoutes(db: Db) {
const router = Router();

const svc = agentAnalyticsService(db);

router.get(
"/companies/:companyId/analytics/agents",
async (req, res) => {
const companyId = req.params.companyId as string;

assertCompanyAccess(req, companyId);

const result = await svc.summary(companyId);

res.json(result);
},
);

return router;
}
148 changes: 148 additions & 0 deletions server/src/services/agent-analytics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
import { and, eq, gte, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { approvals, heartbeatRuns, costEvents } from "@paperclipai/db";

const ANALYTICS_DAYS = 14;

function formatUtcDateKey(date: Date): string {
return date.toISOString().slice(0, 10);
}

function getRecentUtcDateKeys(now: Date, days: number): string[] {
const todayUtc = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());

return Array.from({ length: days }, (_, index) => {
const dayOffset = index - (days - 1);

return formatUtcDateKey(
new Date(todayUtc + dayOffset * 24 * 60 * 60 * 1000),
);
});
}

export function agentAnalyticsService(db: Db) {
return {
summary: async (companyId: string) => {
const now = new Date();

const analyticsDays = getRecentUtcDateKeys(now, ANALYTICS_DAYS);

const analyticsStart = new Date(
`${analyticsDays[0]}T00:00:00.000Z`,
);

const runDayExpr = sql<string>`
to_char(${heartbeatRuns.createdAt} at time zone 'UTC', 'YYYY-MM-DD')
`;

const runRows = await db
.select({
date: runDayExpr,
status: heartbeatRuns.status,
count: sql<number>`count(*)::double precision`,
})
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, companyId),
gte(heartbeatRuns.createdAt, analyticsStart),
),
)
.groupBy(runDayExpr, heartbeatRuns.status);

const tasksCompletedOverTime = new Map(
analyticsDays.map((date) => [
date,
{
date,
completed: 0,
failed: 0,
total: 0,
},
]),
);

for (const row of runRows) {
const bucket = tasksCompletedOverTime.get(row.date);

if (!bucket) continue;

const count = Number(row.count);

if (row.status === "succeeded") {
bucket.completed += count;
} else if (
row.status === "failed" ||
row.status === "timed_out"
) {
bucket.failed += count;
}

bucket.total += count;
}

const approvedCount = await db
.select({
count: sql<number>`count(*)`,
})
.from(approvals)
.where(
and(
eq(approvals.companyId, companyId),
eq(approvals.status, "approved"),
),
)
.then((rows) => Number(rows[0]?.count ?? 0));

const totalApprovalCount = await db
.select({
count: sql<number>`count(*)`,
})
.from(approvals)
.where(eq(approvals.companyId, companyId))
.then((rows) => Number(rows[0]?.count ?? 0));

const approvalRate =
totalApprovalCount > 0
? (approvedCount / totalApprovalCount) * 100
: 0;

const failedRuns = Array.from(tasksCompletedOverTime.values())
.reduce((sum, item) => sum + item.failed, 0);

const totalRuns = Array.from(tasksCompletedOverTime.values())
.reduce((sum, item) => sum + item.total, 0);

const errorRate =
totalRuns > 0
? (failedRuns / totalRuns) * 100
: 0;

const [{ totalCost }] = await db
.select({
totalCost: sql<number>`
coalesce(sum(${costEvents.costCents}), 0)::double precision
`,
})
.from(costEvents)
.where(eq(costEvents.companyId, companyId));

const averageCostPerTask =
totalRuns > 0
? Number(totalCost) / totalRuns
: 0;

return {
tasksCompletedOverTime: Array.from(
tasksCompletedOverTime.values(),
),
approvalRate: Number(approvalRate.toFixed(2)),
errorRate: Number(errorRate.toFixed(2)),
averageCostPerTask: Number(
averageCostPerTask.toFixed(2),
),
};
},
};
}

9 changes: 9 additions & 0 deletions ui/src/api/agent-analytics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { api } from "./client";

export const agentAnalyticsApi = {
summary: async (companyId: string) => {
return api.get(
`/companies/${companyId}/analytics/agents`,
);
},
};
Empty file added ui/src/pages/AgentAnalytics.tsx
Empty file.
23 changes: 21 additions & 2 deletions ui/src/pages/Dashboard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { useEffect, useMemo, useRef, useState } from "react";
import { Link } from "@/lib/router";
import { useQuery } from "@tanstack/react-query";
import { dashboardApi } from "../api/dashboard";
import { agentAnalyticsApi } from "../api/agent-analytics";
import { activityApi } from "../api/activity";
import { accessApi } from "../api/access";
import { issuesApi } from "../api/issues";
Expand All @@ -15,7 +16,6 @@ import { queryKeys } from "../lib/queryKeys";
import { MetricCard } from "../components/MetricCard";
import { EmptyState } from "../components/EmptyState";
import { StatusIcon } from "../components/StatusIcon";

import { ActivityRow } from "../components/ActivityRow";
import { Identity } from "../components/Identity";
import { timeAgo } from "../lib/timeAgo";
Expand Down Expand Up @@ -59,6 +59,12 @@ export function Dashboard() {
enabled: !!selectedCompanyId,
});

const { data: analytics } = useQuery({
queryKey: ["agent-analytics", selectedCompanyId],
queryFn: () => agentAnalyticsApi.summary(selectedCompanyId!),
enabled: !!selectedCompanyId,
});

const { data: activity } = useQuery({
queryKey: [...queryKeys.activity(selectedCompanyId!), { limit: DASHBOARD_ACTIVITY_LIMIT }],
queryFn: () => activityApi.list(selectedCompanyId!, { limit: DASHBOARD_ACTIVITY_LIMIT }),
Expand Down Expand Up @@ -289,7 +295,20 @@ export function Dashboard() {
</span>
}
/>
</div>
{analytics && (
<MetricCard
icon={Bot}
value={`${analytics.approvalRate ?? 0}%`}
label="Approval Rate"
description={
<span>
Error Rate: {analytics.errorRate ?? 0}%
</span>
}
/>
)}

</div>

<div className="grid grid-cols-2 lg:grid-cols-4 gap-4">
<ChartCard title="Run Activity" subtitle="Last 14 days">
Expand Down