city: reconcile GitHub's commit count for repo and org cities, in numbers

viewer/src/main.js

@@ -948,7 +948,18 @@ const el = {
   } else if (tot.contributions != null) {
     bits.push(`GitHub shows ${fmtInt(tot.contributions)} contributions since ${tot.window_start}; it also counts PRs, issues, reviews and private activity. Buildings show default-branch commits.`);
   } else if (!PERSONAL) {
-    bits.push(`Buildings show every non-merge commit on ${DATA.kind === 'org' ? "each repo's" : 'the'} default branch since ${tot.window_start}.`);
+    const org = DATA.kind === 'org', ld = tot.landed;
+    if (ld && ld.commits >= (tot.commits || 0) + (ld.merges || 0)) {
+      // GitHub's count = the buildings + merges + commits written before the window start, exactly (bundle.ts)
+      const earlier = ld.commits - (tot.commits || 0) - (ld.merges || 0);
+      const where = org ? `the default branches of these ${DATA.repos.length} repos` : 'the default branch';
+      const left = [...(ld.merges ? [`${fmtInt(ld.merges)} ${ld.merges === 1 ? 'merge' : 'merges'}`] : []),
+        ...(earlier ? [`${fmtInt(earlier)} ${earlier === 1 ? 'commit' : 'commits'} written before ${tot.window_start}`] : [])];
+      bits.push(`GitHub counts ${fmtInt(ld.commits)} commits on ${where} since ${tot.window_start}, by the day each landed. `
+        + (left.length ? `Buildings show the ${fmtInt(tot.commits || 0)} written since then. Not drawn: ${list(left)}.` : 'Buildings show all of them.'));
+    } else {
+      bits.push(`Buildings show every non-merge commit written since ${tot.window_start} on ${org ? "each repo's" : 'the'} default branch.`);
+    }
   }
   if (rest > 0) bits.push(`${rest} ${big.length ? 'more ' : ''}${rest === 1 ? 'repo' : 'repos'} (${fmtInt(restCommits)} commits) not drawn: the very largest are skipped, and only the busiest are shown.`);
   // what is left of GitHub's count: commits it credits to co-authors, commits once pushed to a default branch and

web/lib/city.ts

@@ -216,6 +216,7 @@ export async function finishBuild(ticketToken: string, planToken: string): Promi
   const repos = p.r.map((r, i) => ({ r, rec: records[i] })).filter(x => x.rec).map(({ r, rec }) => ({
     name: r.n, fork: r.f, record: rec!,
     mine: p.mode === 'mine' ? mineOf(rec!, { shas: new Set(r.sh ?? []) }) : mineOf(rec!, { since: p.s }),
+    ...(p.mode === 'all' ? { landed: r.c } : {}),     // discovery's history(since) count: GitHub's, by landing day
   }));
   const missing = p.r.filter((_, i) => !records[i]);
   let payload: Payload;

web/lib/engine/bundle.ts

@@ -1,7 +1,7 @@
 // Turn replayed repo records into the viewer payload (schema 1). The TypeScript twin of
 // commit_city/bundle.py: same ranking, aggregation, budgets, timezone and epoch rules.
 import { LANG_CLASSES } from './langs.js';
-import type { RepoRecord } from './replay.js';
+import { MAX_COMMITS, type RepoRecord } from './replay.js';
 
 export const SCHEMA_VERSION = 1;
 export const GENERATOR = 'commit.city 0.2.0';
@@ -11,7 +11,10 @@ type Mine = [number, number, number, number[], string, string];   // [author min
 /** A commit row: [t, n files, file idx], plus w (index into Payload.who, >= 1) when not by a human. */
 export type MineRow = [number, number, number[]] | [number, number, number[], number];
 
-export interface BundleRepo { name: string; fork: boolean; record: RepoRecord; mine: Mine[] }
+export interface BundleRepo {
+  name: string; fork: boolean; record: RepoRecord; mine: Mine[];
+  landed?: number;                  // GitHub's count of default-branch commits that landed since the window start (repo and org cities)
+}
 
 export interface BundleInput {
   login: string;                    // shown in the HUD: a user, an org, or owner/repo
@@ -47,6 +50,9 @@ export interface Payload {
     commits: number; contributions: number | null; restricted: number; window_start: string; omitted_repos: number; omitted_commits: number;
     too_big?: { repo: string; commits: number }[];
     github?: { commits: number; capped?: true };
+    // GitHub's count for the drawn repos, by the day each commit landed, and the merges in it: the rest of it
+    // beyond the buildings is commits written before the window start
+    landed?: { commits: number; merges: number };
   };
   repos: { n: string; fork: boolean; anon: boolean; fp: number; agg: number; p: string[]; l: number[]; e: number[][]; m: MineRow[] }[];
 }
@@ -189,6 +195,14 @@ export function bundle(inp: BundleInput): Payload {
   const caps = budgetCaps(records.map(r => r.record.files.length), inp.maxFilesPerRepo ?? 2500, inp.maxFilesTotal ?? 60000);
   const who = actors(records);
   const widx = new Map(who.map((k, i) => [k, i]));
+  // GitHub counts a repo's commits by the day they landed, merges included; the buildings are the non-merge commits
+  // written since the window start. Drawn + merges + written earlier = GitHub's count, exactly when every record is
+  // whole (not approximate, not cut at MAX_COMMITS), so the split is given only then, and only if GitHub's count is
+  // not below the other two
+  const drawn = records.reduce((a, r) => a + r.mine.length, 0);
+  const merges = records.reduce((a, r) => a + r.record.commits.filter(c => c[3] === 1 && c[1] * 60 >= inp.start).length, 0);
+  const landed = records.every(r => r.landed != null) && !records.some(r => r.record.partial || r.record.commits.length >= MAX_COMMITS)
+    ? records.reduce((a, r) => a + r.landed!, 0) : null;
   const repos = records.map((r, k) => {
     const [files, remap] = aggregate(r.record.files, caps[k]);
     const m = r.mine.map(([t, , , idx, , w]): MineRow => {
@@ -215,14 +229,15 @@ export function bundle(inp: BundleInput): Payload {
     langs: LANG_CLASSES,
     who,
     totals: {
-      commits: repos.reduce((a, r) => a + r.m.length, 0),
+      commits: drawn,
       contributions: inp.contributions,
       restricted: inp.restricted,
       window_start: isoDay(inp.start),
       omitted_repos: omitted.length + (inp.omittedRepos ?? 0),
       omitted_commits: omitted.reduce((a, r) => a + r.mine.length, 0) + (inp.omittedCommits ?? 0),
       ...(inp.tooBig?.length ? { too_big: inp.tooBig.map(([repo, commits]) => ({ repo, commits })) } : {}),
       ...(inp.github ? { github: inp.github.capped ? { commits: inp.github.commits, capped: true } : { commits: inp.github.commits } } : {}),
+      ...(landed != null && landed >= drawn + merges ? { landed: { commits: landed, merges } } : {}),
     },
     repos,
   };

web/lib/engine/replay.ts

@@ -7,7 +7,7 @@ import { type Commit, type ObjectDB, type TreeEntry, parseTree } from './pack.js
 
 export const BINARY_SIZE = 12;          // a binary file stands in as a 12-line file
 export const RECORD_VERSION = 2;        // bump when records change shape: cached records of another version are misses
-const MAX_COMMITS = 20000;              // per-repo cap on recorded commit touch lists
+export const MAX_COMMITS = 20000;       // per-repo cap on recorded commit touch lists (the newest by author time)
 const MAX_TOUCH = 300;
 
 /** [sha12, author minute, author UTC offset, is merge (0|1), files touched, first 300 file indexes, who] where who

web/test/bundle.test.ts

@@ -91,6 +91,34 @@ test('bundle matches python (aggregation, budgets, epoch, timezone)', async () =
   assert.ok(ts.repos.every(r => r.m.every(m => m.length === 3 || (m.length === 4 && m[3] > 0))), 'human rows stay 3 long');
 });
 
+test("GitHub's count for a repo city: the buildings, the merges, and commits written before the window", async () => {
+  const r = makeRepo(path.join(tmpdir(), 'landed'), 1);
+  const start = T0 + 2 * DAY;
+  const { record } = await replayRepo({ repo: 'me/alpha', head: r.head(), start, transport: localTransport(r.bare()) });
+  // GitHub dates a commit by when it landed; the city by when it was written. Make one commit a merge and one
+  // written before the window (and landed in it), as in golang/go: 3,748 on GitHub, 3,491 drawn, 32 merges, 225 earlier
+  const commits = record.commits.map(c => [...c] as typeof c);
+  commits[0][3] = 1;
+  commits[1][1] = Math.floor(start / 60) - 7 * 1440;
+  const rec = { ...record, commits };
+  const mine = mineOf(rec, { since: start });
+  assert.equal(mine.length, commits.length - 2);
+  const input = (landed: number | undefined, recd = rec) => ({
+    login: 'me/alpha', kind: 'repo' as const, name: 'me/alpha', avatar: '', url: '', now: T0 + 30 * DAY, start, contributions: null, restricted: 0,
+    repos: [{ name: 'me/alpha', fork: false, record: recd, mine, ...(landed === undefined ? {} : { landed }) }],
+  });
+  const p = bundle(input(commits.length));
+  assert.equal(p.totals.commits, commits.length - 2);
+  assert.deepEqual(p.totals.landed, { commits: commits.length, merges: 1 });   // written earlier: the remaining 1
+  // no split where it would not add up: no count from GitHub, a count below the buildings and merges, a record
+  // that is approximate or cut at its commit cap
+  assert.equal(bundle(input(undefined)).totals.landed, undefined);
+  assert.equal(bundle(input(commits.length - 2)).totals.landed, undefined);
+  assert.equal(bundle(input(commits.length, { ...rec, partial: 'window start state unavailable' })).totals.landed, undefined);
+  const cut = { ...rec, commits: [...commits, ...Array.from({ length: 20_000 - commits.length }, () => commits[2])] };
+  assert.equal(bundle(input(20_000, cut)).totals.landed, undefined);
+});
+
 test('aggregate helpers match the python rules', () => {
   assert.deepEqual(mergeEvents([[1, 5, 3, 0], [2, 7]]), [1, 5, 2, 12, 3, 7]);
   assert.deepEqual(budgetCaps([10, 50, 3], 40, 30), [10, 17, 3]);
@@ -148,6 +176,7 @@ test('finish step: records of another version are never drawn; meta and gallery
   assert.deepEqual([payload.totals.omitted_repos, payload.totals.omitted_commits], [2, 905]);
   assert.deepEqual(payload.totals.too_big, [{ repo: 'me/huge', commits: 900 }]);
   assert.deepEqual(payload.totals.github, { commits: 950 });            // GitHub's count rides along; capped only when true
+  assert.deepEqual(payload.totals.landed, { commits: 7, merges: 0 });   // and, for an org, its history count of the drawn repos
   assert.equal((await getJSON<{ v: number }>('meta/old-me.json'))?.v, meta.v);   // the name the build was asked for points here
 });