From 8b0934241f558015e623054e4a77c6c2134725e2 Mon Sep 17 00:00:00 2001 From: peetzweg Date: Wed, 8 Jul 2026 21:31:23 +0200 Subject: [PATCH] Add async org-indexing worker and per-member monthly resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The request path builds orgs at lifetime-totals resolution only. This adds a background worker (scripts/refresh-orgs.ts) that upgrades orgs to monthly resolution — the data behind a new company commit chart — paced well under GitHub's rate limit with a reserved floor for live traffic. - org_member_monthly table (one row per org/member/month) + migration - fetchOrgMemberWindows returns per-window totals; fetchOrgMemberContributions now sums them for the request path - getOrgSummary assembles a cumulative chart series from monthly_commits rows - OrgView renders the company chart once the worker has written monthly rows - oversized orgs are recorded (builtAt null) so the worker can pick them up --- drizzle/0006_sudden_demogoblin.sql | 13 + drizzle/meta/0006_snapshot.json | 554 +++++++++++++++++++++++++++++ drizzle/meta/_journal.json | 7 + scripts/refresh-orgs.ts | 401 +++++++++++++++++++++ src/components/OrgView.tsx | 13 + src/lib/db/schema.ts | 24 ++ src/lib/github.ts | 92 +++-- src/lib/org-cache.ts | 88 ++++- 8 files changed, 1147 insertions(+), 45 deletions(-) create mode 100644 drizzle/0006_sudden_demogoblin.sql create mode 100644 drizzle/meta/0006_snapshot.json create mode 100644 scripts/refresh-orgs.ts diff --git a/drizzle/0006_sudden_demogoblin.sql b/drizzle/0006_sudden_demogoblin.sql new file mode 100644 index 0000000..6fd3208 --- /dev/null +++ b/drizzle/0006_sudden_demogoblin.sql @@ -0,0 +1,13 @@ +CREATE TABLE "org_member_monthly" ( + "org_id" text NOT NULL, + "member_id" text NOT NULL, + "month" date NOT NULL, + "commits" integer DEFAULT 0 NOT NULL, + "pull_requests" integer DEFAULT 0 NOT NULL, + "reviews" integer DEFAULT 0 NOT NULL, + "issues" integer DEFAULT 0 NOT NULL, + CONSTRAINT "org_member_monthly_org_id_member_id_month_pk" PRIMARY KEY("org_id","member_id","month") +); +--> statement-breakpoint +ALTER TABLE "org_member_monthly" ADD CONSTRAINT "org_member_monthly_org_id_entities_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."entities"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "org_member_monthly" ADD CONSTRAINT "org_member_monthly_member_id_entities_id_fk" FOREIGN KEY ("member_id") REFERENCES "public"."entities"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/drizzle/meta/0006_snapshot.json b/drizzle/meta/0006_snapshot.json new file mode 100644 index 0000000..89e37a5 --- /dev/null +++ b/drizzle/meta/0006_snapshot.json @@ -0,0 +1,554 @@ +{ + "id": "45eac976-1b7d-446c-bad1-84fba64f98e5", + "prevId": "36846044-13e2-4e4e-aa2a-3bf3cdc91e3c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.entities": { + "name": "entities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "login": { + "name": "login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_commits": { + "name": "total_commits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_restricted": { + "name": "total_restricted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_issues": { + "name": "total_issues", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "total_pull_requests": { + "name": "total_pull_requests", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "total_reviews": { + "name": "total_reviews", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "total_repos": { + "name": "total_repos", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "followers": { + "name": "followers", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "following": { + "name": "following", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_repos": { + "name": "public_repos", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "company": { + "name": "company", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "twitter_username": { + "name": "twitter_username", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_verified": { + "name": "is_verified", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "github_node_id": { + "name": "github_node_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_count": { + "name": "member_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_fetched": { + "name": "last_fetched", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "built_at": { + "name": "built_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "suspended_reason": { + "name": "suspended_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.lookups": { + "name": "lookups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "searched_at": { + "name": "searched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "lookups_entity_id_entities_id_fk": { + "name": "lookups_entity_id_entities_id_fk", + "tableFrom": "lookups", + "tableTo": "entities", + "columnsFrom": [ + "entity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.monthly_commits": { + "name": "monthly_commits", + "schema": "", + "columns": { + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "month": { + "name": "month", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "commits": { + "name": "commits", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "restricted": { + "name": "restricted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "issues": { + "name": "issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "pull_requests": { + "name": "pull_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "reviews": { + "name": "reviews", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "repos": { + "name": "repos", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "monthly_commits_entity_id_entities_id_fk": { + "name": "monthly_commits_entity_id_entities_id_fk", + "tableFrom": "monthly_commits", + "tableTo": "entities", + "columnsFrom": [ + "entity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "monthly_commits_entity_id_month_pk": { + "name": "monthly_commits_entity_id_month_pk", + "columns": [ + "entity_id", + "month" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_member_monthly": { + "name": "org_member_monthly", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "month": { + "name": "month", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "commits": { + "name": "commits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "pull_requests": { + "name": "pull_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "reviews": { + "name": "reviews", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "issues": { + "name": "issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "org_member_monthly_org_id_entities_id_fk": { + "name": "org_member_monthly_org_id_entities_id_fk", + "tableFrom": "org_member_monthly", + "tableTo": "entities", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "org_member_monthly_member_id_entities_id_fk": { + "name": "org_member_monthly_member_id_entities_id_fk", + "tableFrom": "org_member_monthly", + "tableTo": "entities", + "columnsFrom": [ + "member_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "org_member_monthly_org_id_member_id_month_pk": { + "name": "org_member_monthly_org_id_member_id_month_pk", + "columns": [ + "org_id", + "member_id", + "month" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_members": { + "name": "org_members", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public_member'" + }, + "commits": { + "name": "commits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "pull_requests": { + "name": "pull_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "reviews": { + "name": "reviews", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "issues": { + "name": "issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_fetched": { + "name": "last_fetched", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "org_members_member_idx": { + "name": "org_members_member_idx", + "columns": [ + { + "expression": "member_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "org_members_org_id_entities_id_fk": { + "name": "org_members_org_id_entities_id_fk", + "tableFrom": "org_members", + "tableTo": "entities", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "org_members_member_id_entities_id_fk": { + "name": "org_members_member_id_entities_id_fk", + "tableFrom": "org_members", + "tableTo": "entities", + "columnsFrom": [ + "member_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "org_members_org_id_member_id_pk": { + "name": "org_members_org_id_member_id_pk", + "columns": [ + "org_id", + "member_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 9f8e84d..970cf0b 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1783493299385, "tag": "0005_clammy_bloodaxe", "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1783535721021, + "tag": "0006_sudden_demogoblin", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/refresh-orgs.ts b/scripts/refresh-orgs.ts new file mode 100644 index 0000000..014dac0 --- /dev/null +++ b/scripts/refresh-orgs.ts @@ -0,0 +1,401 @@ +/** + * Org ("company") background worker: seeds new orgs and upgrades existing ones from lifetime + * totals to MONTHLY resolution — the data behind the company commit chart. + * + * The request path (org-cache.ts) deliberately stores lifetime totals only (~2 requests per + * member). Monthly resolution costs ~12× that, so it lives here, paced well under GitHub's + * 5,000 points/hour with a reserved floor for live traffic — same politeness model as + * backfill-contributions.ts. Run with bun (auto-loads .env; beware a shell-exported + * GITHUB_TOKEN overriding it — prefix with `env -u GITHUB_TOKEN` if your shell sets one): + * + * bun scripts/refresh-orgs.ts # monthly-refresh every org, stalest first + * bun scripts/refresh-orgs.ts # one org + * bun scripts/refresh-orgs.ts --add # seed new orgs (lifetime totals only) + * + * Per org: re-fetch the profile, re-enumerate members (new public members join as pending; + * departed members' rows are KEPT with their last totals — see #97), then per member fetch + * monthly org-scoped windows → org_member_monthly rows + refreshed org_members lifetime + * totals. Finally roll up: org entity totals + org-level monthly_commits rows (the chart). + * + * Safe to re-run and interrupt: everything is an idempotent upsert, the org's lastFetched is + * stamped only at the end (an interrupted org stays stalest and is retried first), and members + * already refreshed in this run are skipped on resume. + */ +import { and, asc, eq, sql } from "drizzle-orm"; +import { db } from "#/lib/db"; +import { + entities, + monthlyCommits, + orgMemberMonthly, + orgMembers, +} from "#/lib/db/schema"; +import { + fetchOrgMembers, + fetchOrgMemberWindows, + fetchOrgProfile, + GitHubError, + monthlyWindows, + type OrgMemberTotals, +} from "#/lib/github"; +import { getOrgSummary } from "#/lib/org-cache"; + +const GITHUB_TOKEN = process.env.GITHUB_TOKEN; +if (!process.env.DATABASE_URL) + throw new Error("DATABASE_URL is required (add it to .env)"); +if (!GITHUB_TOKEN) throw new Error("GITHUB_TOKEN is required (add it to .env)"); +if (!db) throw new Error("Database client failed to initialise."); +const token = GITHUB_TOKEN; +const database = db; + +const orgEntityId = (login: string) => `org:${login.trim().toLowerCase()}`; +const userEntityId = (login: string) => `user:${login.trim().toLowerCase()}`; + +// Requests/hour we aim to spend (≈ GraphQL points), well under the 5,000/hr limit so the live +// site keeps working. Override with REFRESH_RATE=. +const TARGET_RATE = Number(process.env.REFRESH_RATE ?? 2500); +// Never let the remaining budget drop below this — headroom for live traffic. +const REMAINING_FLOOR = 500; +const CHUNK_ROWS = 100; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +interface RateLimit { + remaining: number; + resetAt: string; +} + +/** Query GitHub's current rate-limit budget. `rateLimit` queries themselves cost 0 points. */ +async function rateLimit(): Promise { + try { + const res = await fetch("https://api.github.com/graphql", { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "User-Agent": "commit-history-refresh-orgs", + }, + body: JSON.stringify({ + query: "query { rateLimit { remaining resetAt } }", + }), + }); + const json = (await res.json()) as { data?: { rateLimit: RateLimit } }; + return json.data?.rateLimit ?? null; + } catch { + return null; + } +} + +/** If we're near the reserved floor, sleep until GitHub's window resets (plus a small buffer). */ +async function respectFloor(): Promise { + const rl = await rateLimit(); + if (!rl) return; + if (rl.remaining > REMAINING_FLOOR) return; + const waitMs = Math.max(0, new Date(rl.resetAt).getTime() - Date.now()) + 2000; + console.log( + `… budget low (${rl.remaining} left) — pausing ${Math.ceil(waitMs / 1000)}s until reset`, + ); + await sleep(waitMs); +} + +const sumTotals = (windows: OrgMemberTotals[]): OrgMemberTotals => + windows.reduce( + (acc, w) => ({ + commits: acc.commits + w.commits, + issues: acc.issues + w.issues, + pullRequests: acc.pullRequests + w.pullRequests, + reviews: acc.reviews + w.reviews, + }), + { commits: 0, issues: 0, pullRequests: 0, reviews: 0 }, + ); + +/** Monthly-refresh one org. Returns the (approximate) number of GitHub requests spent. */ +async function refreshOrg(orgId: string, login: string): Promise { + const runStart = new Date(); + let requests = 0; + + // Profile first: refreshes the mutable metadata and guarantees nodeId/createdAt. + const profile = await fetchOrgProfile(login, token); + requests += 1; + await database + .update(entities) + .set({ + login: profile.login, + name: profile.name, + avatarUrl: profile.avatarUrl, + htmlUrl: profile.htmlUrl, + createdAt: new Date(profile.createdAt), + bio: profile.description, + location: profile.location, + websiteUrl: profile.websiteUrl, + twitterUsername: profile.twitterUsername, + publicRepos: profile.publicRepos, + isVerified: profile.isVerified, + githubNodeId: profile.nodeId, + memberCount: profile.memberCount, + }) + .where(eq(entities.id, orgId)); + const orgCreated = new Date(profile.createdAt); + + // Re-enumerate: new public members join as pending rows; departed rows are kept (#97). + const members = await fetchOrgMembers(login, token); + requests += Math.max(1, Math.ceil(members.length / 100)); + for (let i = 0; i < members.length; i += CHUNK_ROWS) { + const chunk = members.slice(i, i + CHUNK_ROWS); + await database + .insert(entities) + .values( + chunk.map((m) => ({ + id: userEntityId(m.login), + kind: "user", + login: m.login, + name: m.name, + avatarUrl: m.avatarUrl, + htmlUrl: `https://github.com/${m.login}`, + createdAt: new Date(m.createdAt), + })), + ) + .onConflictDoNothing(); + await database + .insert(orgMembers) + .values( + chunk.map((m) => ({ + orgId, + memberId: userEntityId(m.login), + role: m.role, + source: "public_member", + })), + ) + .onConflictDoNothing(); + } + const currentIds = new Set(members.map((m) => userEntityId(m.login))); + const allRows = await database + .select({ memberId: orgMembers.memberId, lastFetched: orgMembers.lastFetched }) + .from(orgMembers) + .where(eq(orgMembers.orgId, orgId)); + const departed = allRows.filter((r) => !currentIds.has(r.memberId)).length; + if (departed > 0) { + console.log(` keeping ${departed} departed member row(s) frozen (#97)`); + } + const lastFetchedById = new Map( + allRows.map((r) => [r.memberId, r.lastFetched]), + ); + + for (const m of members) { + const memberId = userEntityId(m.login); + // Resume support: skip members already refreshed since this run started. + const prev = lastFetchedById.get(memberId); + if (prev && prev >= runStart) continue; + + const start = new Date( + Math.max(orgCreated.getTime(), new Date(m.createdAt).getTime()), + ); + const windows = monthlyWindows(start, runStart); + let counts: OrgMemberTotals[]; + try { + counts = await fetchOrgMemberWindows(m.login, profile.nodeId, token, windows); + } catch (e) { + // A deleted/renamed member 404s forever — record zeros and move on. + if (!(e instanceof GitHubError && e.status === 404)) throw e; + counts = []; + } + const req = Math.max(1, Math.ceil(windows.length / 6)); + requests += req; + + // Store active months only — absent rows read as zero when the chart assembles. + const monthRows = windows + .map((w, i) => ({ + orgId, + memberId, + month: w.label, + commits: counts[i]?.commits ?? 0, + pullRequests: counts[i]?.pullRequests ?? 0, + reviews: counts[i]?.reviews ?? 0, + issues: counts[i]?.issues ?? 0, + })) + .filter((r) => r.commits || r.pullRequests || r.reviews || r.issues); + for (let i = 0; i < monthRows.length; i += CHUNK_ROWS) { + await database + .insert(orgMemberMonthly) + .values(monthRows.slice(i, i + CHUNK_ROWS)) + .onConflictDoUpdate({ + target: [ + orgMemberMonthly.orgId, + orgMemberMonthly.memberId, + orgMemberMonthly.month, + ], + set: { + commits: sql`excluded.commits`, + pullRequests: sql`excluded.pull_requests`, + reviews: sql`excluded.reviews`, + issues: sql`excluded.issues`, + }, + }); + } + const totals = sumTotals(counts); + await database + .update(orgMembers) + .set({ ...totals, lastFetched: new Date() }) + .where( + and(eq(orgMembers.orgId, orgId), eq(orgMembers.memberId, memberId)), + ); + console.log( + ` ✓ ${m.login.padEnd(24)} ${totals.commits.toLocaleString()} commits · ${monthRows.length} active months`, + ); + // Politeness pacing: spread this member's request cost across the target hourly rate. + await sleep((req / TARGET_RATE) * 3_600_000); + } + + // Roll up: org entity totals from org_members (departed rows included, frozen — #97) and + // org-level month rows from org_member_monthly (the company chart's data). + const [sums] = await database + .select({ + commits: sql`coalesce(sum(${orgMembers.commits}), 0)`, + pullRequests: sql`coalesce(sum(${orgMembers.pullRequests}), 0)`, + reviews: sql`coalesce(sum(${orgMembers.reviews}), 0)`, + issues: sql`coalesce(sum(${orgMembers.issues}), 0)`, + }) + .from(orgMembers) + .where(eq(orgMembers.orgId, orgId)); + const monthly = await database + .select({ + month: orgMemberMonthly.month, + commits: sql`sum(${orgMemberMonthly.commits})`, + pullRequests: sql`sum(${orgMemberMonthly.pullRequests})`, + reviews: sql`sum(${orgMemberMonthly.reviews})`, + issues: sql`sum(${orgMemberMonthly.issues})`, + }) + .from(orgMemberMonthly) + .where(eq(orgMemberMonthly.orgId, orgId)) + .groupBy(orgMemberMonthly.month); + for (let i = 0; i < monthly.length; i += CHUNK_ROWS) { + await database + .insert(monthlyCommits) + .values( + monthly.slice(i, i + CHUNK_ROWS).map((r) => ({ + entityId: orgId, + month: r.month, + commits: Number(r.commits), + pullRequests: Number(r.pullRequests), + reviews: Number(r.reviews), + issues: Number(r.issues), + })), + ) + .onConflictDoUpdate({ + target: [monthlyCommits.entityId, monthlyCommits.month], + set: { + commits: sql`excluded.commits`, + pullRequests: sql`excluded.pull_requests`, + reviews: sql`excluded.reviews`, + issues: sql`excluded.issues`, + }, + }); + } + await database + .update(entities) + .set({ + totalCommits: Number(sums?.commits ?? 0), + totalPullRequests: Number(sums?.pullRequests ?? 0), + totalReviews: Number(sums?.reviews ?? 0), + totalIssues: Number(sums?.issues ?? 0), + builtAt: new Date(), + lastFetched: new Date(), + }) + .where(eq(entities.id, orgId)); + return requests; +} + +/** Seed one new org via the same code path the site uses, looping the resumable build. */ +async function seedOrg(login: string): Promise { + for (let i = 0; i < 200; i++) { + try { + const s = await getOrgSummary(login, token); + console.log( + `✓ ${login.padEnd(24)} ${s.totalCommits.toLocaleString()} commits · ${s.membersTracked} members`, + ); + return true; + } catch (e) { + if (e instanceof GitHubError && e.status === 503) { + await sleep(500); // budget slice done, progress persisted — continue immediately-ish + continue; + } + // Too large for a live build (422): the request path still recorded the profile row, + // so the org is now enrolled — `refreshOrg` (via --all or by name) builds it here, + // paced, with no size cap. This is the intended path for mega-orgs like `google`. + if (e instanceof GitHubError && e.status === 422) { + console.log( + `• ${login.padEnd(24)} enrolled — too large for a live build; run to index: bun scripts/refresh-orgs.ts ${login}`, + ); + return true; + } + console.log(`✗ ${login.padEnd(24)} ${(e as Error).message}`); + return false; + } + } + console.log(`✗ ${login.padEnd(24)} did not converge — rerun to resume`); + return false; +} + +const args = process.argv.slice(2); + +if (args[0] === "--add") { + const logins = args.slice(1); + if (logins.length === 0) { + console.error("Usage: bun scripts/refresh-orgs.ts --add [login…]"); + process.exit(1); + } + console.log(`Seeding ${logins.length} org(s) (lifetime totals only)\n`); + let ok = 0; + for (const login of logins) { + await respectFloor(); + if (await seedOrg(login)) ok++; + } + console.log(`\nDone. ${ok}/${logins.length} seeded.`); +} else if (args[0] && !args[0].startsWith("--")) { + const [row] = await database + .select({ id: entities.id, login: entities.login }) + .from(entities) + .where(eq(entities.id, orgEntityId(args[0]))) + .limit(1); + if (!row) { + console.error( + `✗ "${args[0]}" is not a known org — seed it first: bun scripts/refresh-orgs.ts --add ${args[0]}`, + ); + process.exit(1); + } + console.log(`Refreshing ${row.login} to monthly resolution…`); + const requests = await refreshOrg(row.id, row.login); + console.log(`Done (${requests} requests).`); +} else if (args.length === 0 || args[0] === "--all") { + // Stalest first; an interrupted org keeps its old lastFetched and is retried first. + const rows = await database + .select({ id: entities.id, login: entities.login }) + .from(entities) + .where(eq(entities.kind, "org")) + .orderBy(sql`${entities.lastFetched} asc nulls first`, asc(entities.id)); + console.log( + `${rows.length} org(s) to refresh to monthly resolution, stalest first, ~${TARGET_RATE} req/hr\n`, + ); + let spent = 0; + for (const row of rows) { + await respectFloor(); + console.log(`${row.login}:`); + try { + spent += await refreshOrg(row.id, row.login); + } catch (e) { + console.log(`✗ ${row.login} failed: ${(e as Error).message} — continuing`); + } + } + console.log(`\nDone. ~${spent.toLocaleString()} requests spent.`); +} else { + console.log( + [ + "Usage:", + " bun scripts/refresh-orgs.ts monthly-refresh every org, stalest first", + " bun scripts/refresh-orgs.ts one org", + " bun scripts/refresh-orgs.ts --add seed new orgs (lifetime totals only)", + "", + " REFRESH_RATE= target requests/hour (default 2500)", + ].join("\n"), + ); + process.exit(1); +} diff --git a/src/components/OrgView.tsx b/src/components/OrgView.tsx index 48c02d0..faf77c2 100644 --- a/src/components/OrgView.tsx +++ b/src/components/OrgView.tsx @@ -1,6 +1,7 @@ import { Link } from "@tanstack/react-router"; import { BadgeCheck } from "lucide-react"; import { motion } from "motion/react"; +import { CommitChart } from "#/components/CommitChart"; import type { BuildProgress } from "#/lib/github"; import type { OrgMemberEntry, OrgResult } from "#/lib/org"; import type { OrgSummary } from "#/lib/org-cache"; @@ -241,6 +242,18 @@ function LoadedOrg({ + {/* The company chart appears once the refresh-orgs worker has written monthly rows. */} + {org.points.length > 0 && ( + + + + )} +

Lifetime contributions of {org.membersTracked.toLocaleString()} public member{org.membersTracked === 1 ? "" : "s"} to {org.login}’s diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index ed8e82b..f15e150 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -104,6 +104,30 @@ export const orgMembers = pgTable( ], ); +/** + * Per-month resolution of org_members — one row per (org, member, month), written by the + * refresh-orgs worker (the request path stores lifetime totals only). Summed across members + * per month these produce the org's own monthly_commits rows (the company chart); per-row + * they enable member-level charts and precise incremental refresh later. + */ +export const orgMemberMonthly = pgTable( + "org_member_monthly", + { + orgId: text("org_id") + .notNull() + .references(() => entities.id), + memberId: text("member_id") + .notNull() + .references(() => entities.id), + month: date("month").notNull(), // YYYY-MM-01 + commits: integer("commits").notNull().default(0), + pullRequests: integer("pull_requests").notNull().default(0), + reviews: integer("reviews").notNull().default(0), + issues: integer("issues").notNull().default(0), + }, + (t) => [primaryKey({ columns: [t.orgId, t.memberId, t.month] })], +); + /** Every search — powers "recent lookups" and the all-time leaderboard. */ export const lookups = pgTable("lookups", { id: serial("id").primaryKey(), diff --git a/src/lib/github.ts b/src/lib/github.ts index f950c6a..be6b9b9 100644 --- a/src/lib/github.ts +++ b/src/lib/github.ts @@ -556,12 +556,20 @@ function isServerHiccup(e: unknown): e is GitHubError { return e instanceof GitHubError && RETRYABLE_STATUS.has(e.status); } +const ZERO_TOTALS: OrgMemberTotals = { + commits: 0, + issues: 0, + pullRequests: 0, + reviews: 0, +}; + /** - * Fetch one member's lifetime contributions *to one org*: aliased org-scoped - * `contributionsCollection(organizationID: …)` windows, batched like `fetchMonthlyCommits`, - * summed into a single totals tuple (per-window resolution is the later worker's job). - * Batches run sequentially — the org build already processes members concurrently, and - * nesting concurrency would burst past the global CONCURRENCY cap. + * Fetch one member's org-scoped contributions per window: aliased + * `contributionsCollection(organizationID: …)` fields, batched like `fetchMonthlyCommits`, + * returned aligned 1:1 with `windows`. The request path sums yearly windows (lifetime totals); + * the refresh-orgs worker stores monthly windows as-is. Batches run sequentially — callers + * already parallelize across members, and nesting concurrency would burst past the global + * CONCURRENCY cap. * * GitHub 500s *deterministically* on some (account, window) combos — a single corrupted year * fails the same way on every attempt. A batch that still fails after graphql()'s retries is @@ -569,18 +577,18 @@ function isServerHiccup(e: unknown): e is GitHubError { * zero. Without this, one poisoned window wedges its member — and the whole org build — * forever. Rate limits and hard 4xx still propagate. */ -export async function fetchOrgMemberContributions( +export async function fetchOrgMemberWindows( rawLogin: string, orgNodeId: string, token: string, windows: MonthWindow[], -): Promise { +): Promise { const login = assertLogin(rawLogin); if (!NODE_ID_RE.test(orgNodeId)) { throw new GitHubError(`Invalid organization node id.`, 400); } - async function fetchBatch(batch: MonthWindow[]): Promise { + async function fetchBatch(batch: MonthWindow[]): Promise { const aliases = batch .map( (w, j) => @@ -599,53 +607,61 @@ export async function fetchOrgMemberContributions( > | null; }>(token, `query { user(login: "${login}") { ${aliases} } }`); if (!data.user) throw new GitHubError(`User "${login}" not found.`, 404); - const sums: OrgMemberTotals = { - commits: 0, - issues: 0, - pullRequests: 0, - reviews: 0, - }; - for (let j = 0; j < batch.length; j++) { - const w = data.user[`w${j}`]; - sums.commits += w?.totalCommitContributions ?? 0; - sums.issues += w?.totalIssueContributions ?? 0; - sums.pullRequests += w?.totalPullRequestContributions ?? 0; - sums.reviews += w?.totalPullRequestReviewContributions ?? 0; - } - return sums; + return batch.map((_, j) => { + const w = data.user?.[`w${j}`]; + return { + commits: w?.totalCommitContributions ?? 0, + issues: w?.totalIssueContributions ?? 0, + pullRequests: w?.totalPullRequestContributions ?? 0, + reviews: w?.totalPullRequestReviewContributions ?? 0, + }; + }); } - const totals: OrgMemberTotals = { - commits: 0, - issues: 0, - pullRequests: 0, - reviews: 0, - }; - const add = (t: OrgMemberTotals) => { - totals.commits += t.commits; - totals.issues += t.issues; - totals.pullRequests += t.pullRequests; - totals.reviews += t.reviews; - }; - + const results: OrgMemberTotals[] = []; for (let i = 0; i < windows.length; i += BATCH) { const batch = windows.slice(i, i + BATCH); try { - add(await fetchBatch(batch)); + results.push(...(await fetchBatch(batch))); } catch (err) { if (!isServerHiccup(err)) throw err; // Isolate the poisoned window: the healthy windows keep their real counts. for (const w of batch) { try { - add(await fetchBatch([w])); + results.push(...(await fetchBatch([w]))); } catch (e) { if (!isServerHiccup(e)) throw e; // This window alone still 500s — GitHub can't serve it; count it as zero. + results.push({ ...ZERO_TOTALS }); } } } } - return totals; + return results; +} + +/** One member's summed lifetime contributions to one org — the request path's shape. */ +export async function fetchOrgMemberContributions( + rawLogin: string, + orgNodeId: string, + token: string, + windows: MonthWindow[], +): Promise { + const perWindow = await fetchOrgMemberWindows( + rawLogin, + orgNodeId, + token, + windows, + ); + return perWindow.reduce( + (acc, w) => ({ + commits: acc.commits + w.commits, + issues: acc.issues + w.issues, + pullRequests: acc.pullRequests + w.pullRequests, + reviews: acc.reviews + w.reviews, + }), + { ...ZERO_TOTALS }, + ); } /** diff --git a/src/lib/org-cache.ts b/src/lib/org-cache.ts index 6dd00f0..2b70fc5 100644 --- a/src/lib/org-cache.ts +++ b/src/lib/org-cache.ts @@ -1,11 +1,15 @@ import { and, eq, isNull, sql } from "drizzle-orm"; import { type DB, db } from "#/lib/db"; -import { entities, orgMembers } from "#/lib/db/schema"; +import { entities, monthlyCommits, orgMembers } from "#/lib/db/schema"; import { + buildPoints, + type CommitPoint, fetchOrgMemberContributions, fetchOrgMembers, fetchOrgProfile, GitHubError, + type MonthlyCount, + monthlyWindows, type OrgMember, type OrgProfile, yearlyWindows, @@ -68,6 +72,12 @@ export interface OrgSummary { totalPullRequests: number; totalReviews: number; totalIssues: number; + /** + * Cumulative monthly series for the company chart — built from the org's own + * monthly_commits rows, which only exist once the refresh-orgs worker has run. + * Empty until then (the page simply shows no chart). + */ + points: CommitPoint[]; } export function orgEntityId(login: string) { @@ -98,9 +108,59 @@ export async function getOrgSummary( type EntityRow = typeof entities.$inferSelect; +const EMPTY_MONTH: MonthlyCount = { + commits: 0, + restricted: 0, + issues: 0, + pullRequests: 0, + reviews: 0, + repos: 0, +}; + +/** + * Assemble the org's cumulative chart series from its monthly_commits rows (written by the + * refresh-orgs worker; the worker skips inactive months, so gaps read as zero). Empty — and + * cheap — until the worker has run. Best-effort: a failure here must not sink the page. + */ +async function orgPoints( + database: DB, + id: string, + createdAt: Date | null, + now: Date, +): Promise { + try { + const rows = await database + .select() + .from(monthlyCommits) + .where(eq(monthlyCommits.entityId, id)); + if (rows.length === 0 || !createdAt) return []; + const byMonth = new Map( + rows.map((r) => [ + r.month, + { + commits: r.commits, + restricted: r.restricted, + issues: r.issues, + pullRequests: r.pullRequests, + reviews: r.reviews, + repos: r.repos, + }, + ]), + ); + const windows = monthlyWindows(createdAt, now); + return buildPoints( + windows, + windows.map((w) => byMonth.get(w.label) ?? EMPTY_MONTH), + ); + } catch { + return []; + } +} + function summaryFromRow( row: EntityRow, membersTracked: number, + points: CommitPoint[], now: Date, ): OrgSummary { return { @@ -121,6 +181,7 @@ function summaryFromRow( totalPullRequests: row.totalPullRequests ?? 0, totalReviews: row.totalReviews ?? 0, totalIssues: row.totalIssues ?? 0, + points, }; } @@ -141,18 +202,24 @@ async function getFromDb( if (!row) { // First sighting: the profile fetch validates the login and yields the node id (keys every - // org-scoped contribution query) + createdAt (caps member windows). Refuse oversized orgs - // BEFORE writing anything — a half-enumerated mega-org would burn quota on every poll. + // org-scoped contribution query) + createdAt (caps member windows). An oversized org is + // still recorded (profile row, builtAt null, no member rows) so the refresh-orgs worker + // picks it up — the request path just refuses to build it live. const profile = await fetchOrgProfile(login, token); - assertBuildable(profile); row = await upsertOrgProfile(database, id, profile, now); + assertBuildable(profile); } const complete = row.builtAt != null; const fresh = row.lastFetched && nowMs - row.lastFetched.getTime() < ORG_TTL; if (complete && fresh) { - return summaryFromRow(row, await memberRowCount(database, id), now); + return summaryFromRow( + row, + await memberRowCount(database, id), + await orgPoints(database, id, row.createdAt, now), + now, + ); } if (complete) { @@ -165,7 +232,12 @@ async function getFromDb( } catch { /* keep the stored profile — totals still serve */ } - return summaryFromRow(row, await memberRowCount(database, id), now); + return summaryFromRow( + row, + await memberRowCount(database, id), + await orgPoints(database, id, row.createdAt, now), + now, + ); } // ── Initial build (builtAt null) — resumes here on every poll ────────────── @@ -259,11 +331,13 @@ async function getFromDb( } // Every member stored — roll up and stamp builtAt (same "only a finished build may look - // finished" rule as persistEntity in cache.ts). + // finished" rule as persistEntity in cache.ts). No chart points yet: monthly rows only + // exist once the refresh-orgs worker has visited this org. const totals = await rollUpTotals(database, id, now); return summaryFromRow( { ...row, ...totals, builtAt: now, lastFetched: now }, membersTotal, + [], now, ); }