gallery: read the version the store holds, and write it conditionally

web/lib/city.ts

@@ -7,7 +7,7 @@ import { replayRepo } from './engine/repo.js';
 import { HttpError } from './http.js';
 import { sign, verify } from './secret.js';
 import { type Sky, skyline } from './skyline.js';
-import { getJSON, getStore, putJSON } from './store.js';
+import { getJSON, getStore, putJSON, StoreConflict } from './store.js';
 import { cityKey } from './target.js';
 
 // the record version is part of the key, so records of an older shape are cache misses and get replayed again.
@@ -55,6 +55,7 @@ export interface GalleryEntry {
   agent: number; bot: number;     // shares of the commits, 0..1 to 3 decimals
 }
 export const GALLERY = 'gallery/recent.json';
+const galleryAt = (id: string) => `gallery/recent/${id}.json`;
 const GALLERY_MAX = 60;
 
 export interface Settings { hidden?: boolean; exclude?: string[]; updated?: string }
@@ -208,10 +209,41 @@ export async function finishBuild(ticketToken: string, planToken: string): Promi
   return meta;
 }
 
-async function addToGallery(m: CityMeta) {
-  const list = (await getJSON<GalleryEntry[]>(GALLERY, true)) ?? [];
-  const next = [galleryEntry(m), ...list.filter(e => e.key !== m.key)].slice(0, GALLERY_MAX);
-  await putJSON(GALLERY, next, { overwrite: true, maxAge: 60 });
+// gallery/recent.json is rewritten by every build, and the blob CDN serves an overwritten blob long past its
+// max-age: in production a copy 30 minutes stale showed vercel's earlier numbers, and a build that read it and
+// wrote it back undid the build before it (sindresorhus). So each version is also written where it is never
+// overwritten, and the rewritten blob's content type, which a head request reads from the store itself, names
+// the current one. Writes are conditional on its etag, so two builds finishing together both get in.
+async function galleryNow(): Promise<{ list: GalleryEntry[]; etag: string | null }> {
+  const st = await getStore().stat(GALLERY);
+  if (!st) return { list: [], etag: null };
+  const id = /;\s*version=([\w-]+)/.exec(st.type)?.[1];
+  return { list: (await getJSON<GalleryEntry[]>(id ? galleryAt(id) : GALLERY)) ?? [], etag: st.etag };
+}
+
+export async function readGallery(): Promise<GalleryEntry[]> {
+  return (await galleryNow()).list;
+}
+
+async function updateGallery(change: (list: GalleryEntry[]) => GalleryEntry[] | null) {
+  for (let attempt = 1; ; attempt++) {
+    const { list, etag } = await galleryNow();
+    const next = change(list);
+    if (!next) return;
+    const id = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    await putJSON(galleryAt(id), next);
+    try {
+      await putJSON(GALLERY, next, { maxAge: 60, type: `application/json; version=${id}`, ...(etag ? { ifMatch: etag } : { overwrite: false }) });
+      return;
+    } catch (e) {
+      if (!(e instanceof StoreConflict) || attempt >= 8) throw e;    // another build wrote first: apply this change to theirs
+      await new Promise(r => setTimeout(r, Math.random() * 100 * attempt));
+    }
+  }
+}
+
+export async function addToGallery(m: CityMeta) {
+  await updateGallery(list => [galleryEntry(m), ...list.filter(e => e.key !== m.key)].slice(0, GALLERY_MAX));
 }
 
 const share = (n: number, total: number) => (total ? Math.round((1000 * n) / total) / 1000 : 0);
@@ -224,10 +256,11 @@ export function galleryEntry(m: CityMeta): GalleryEntry {
 }
 
 export async function removeFromGallery(owner: string) {
-  const list = (await getJSON<GalleryEntry[]>(GALLERY, true)) ?? [];
   const o = owner.toLowerCase();
-  const next = list.filter(e => e.key !== o && !e.key.startsWith(o + '/'));
-  if (next.length !== list.length) await putJSON(GALLERY, next, { overwrite: true, maxAge: 60 });
+  await updateGallery(list => {
+    const next = list.filter(e => e.key !== o && !e.key.startsWith(o + '/'));
+    return next.length !== list.length ? next : null;
+  });
 }
 
 export function summarize(payload: Payload, base: Pick<CityMeta, 'v' | 'key' | 'target' | 'kind' | 'url'>): CityMeta {

web/lib/routes/gallery.ts

@@ -1,20 +1,28 @@
 // Cities for the landing page: a hand-picked few, and the most recently built.
-import { GALLERY, type GalleryEntry, readMeta } from '../city.js';
+import { type GalleryEntry, readGallery, readMeta } from '../city.js';
 import { handle, json } from '../http.js';
-import { getJSON } from '../store.js';
 
-const FEATURED = ['torvalds', 'sindresorhus', 'vercel', 'facebook/react', 'denoland/deno', 'gaearon', 'karpathy', 'rust-lang/rust', 'withastro', 'antfu'];
+const FEATURED = ['torvalds', 'sindresorhus', 'vercel', 'react/react', 'denoland/deno', 'gaearon', 'karpathy', 'rust-lang/rust', 'withastro', 'antfu'];
+
+// a city built under a repo's old name (facebook/react, now react/react) is a second city of the same repo: the
+// list shows the one under the name GitHub uses now
+const current = (e: GalleryEntry) => e.key === e.target.toLowerCase() + (e.key.endsWith('@all') ? '@all' : '');
 
 export const GET = handle(async () => {
-  const [recent, metas] = await Promise.all([
-    getJSON<GalleryEntry[]>(GALLERY).catch(() => null),
-    Promise.all(FEATURED.map(k => readMeta(k).catch(() => null))),
-  ]);
-  const featured: GalleryEntry[] = metas.filter(m => m !== null).map(m => ({
-    key: m.key, target: m.target, kind: m.kind, avatar: m.avatar, commits: m.commits, repos: m.repos, lines: m.lines, v: m.v,
-    // commit shares as galleryEntry() in city.ts writes them; metas built before the agent lens have no `who`
-    agent: m.who && m.commits ? Math.round((1000 * m.who.agent) / m.commits) / 1000 : 0,
-    bot: m.who && m.commits ? Math.round((1000 * m.who.bot) / m.commits) / 1000 : 0,
-  }));
-  return json({ featured, recent: recent ?? [] }, 200, { 'Cache-Control': 'public, s-maxage=60, stale-while-revalidate=600' });
+  const recent = (await readGallery().catch(() => [] as GalleryEntry[])).filter(current);
+  // a featured city built lately takes its entry from the list, so both lists show the same build; its meta otherwise
+  const latest = new Map(recent.map(e => [e.key, e]));
+  const metas = await Promise.all(FEATURED.map(k => (latest.has(k) ? null : readMeta(k).catch(() => null))));
+  const featured: GalleryEntry[] = FEATURED.flatMap((k, i) => {
+    const e = latest.get(k), m = metas[i];
+    if (e) return [e];
+    if (!m || m.key !== m.target.toLowerCase()) return [];
+    return [{
+      key: m.key, target: m.target, kind: m.kind, avatar: m.avatar, commits: m.commits, repos: m.repos, lines: m.lines, v: m.v,
+      // commit shares as galleryEntry() in city.ts writes them; metas built before the agent lens have no `who`
+      agent: m.who && m.commits ? Math.round((1000 * m.who.agent) / m.commits) / 1000 : 0,
+      bot: m.who && m.commits ? Math.round((1000 * m.who.bot) / m.commits) / 1000 : 0,
+    }];
+  });
+  return json({ featured, recent }, 200, { 'Cache-Control': 'public, s-maxage=60, stale-while-revalidate=600' });
 });

web/lib/store.ts

@@ -1,16 +1,23 @@
 // Storage: Vercel Blob in production (a public store: everything in it is public GitHub data), a local
 // directory in development and tests.
-import { BlobNotFoundError, del as blobDel, get as blobGet, head as blobHead, list as blobList, put as blobPut } from '@vercel/blob';
+import { BlobNotFoundError, BlobPreconditionFailedError, del as blobDel, get as blobGet, head as blobHead, list as blobList, put as blobPut } from '@vercel/blob';
+import { createHash } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 import { gunzipSync, gzipSync } from 'node:zlib';
 
-export interface PutOptions { contentType: string; maxAge?: number; overwrite?: boolean }
+export interface PutOptions { contentType: string; maxAge?: number; overwrite?: boolean; ifMatch?: string }
 export interface Listed { pathname: string; uploadedAt: Date; size: number }
+export interface Stat { type: string; etag: string }
+
+/** A conditional write (ifMatch) lost: the blob changed after its etag was read. */
+export class StoreConflict extends Error {}
 
 export interface Store {
   get(pathname: string, fresh?: boolean): Promise<Buffer | null>;
   exists(pathname: string): Promise<boolean>;
+  /** Content type and etag as the store itself holds them, never a CDN copy; null if there is no such blob. */
+  stat(pathname: string): Promise<Stat | null>;
   put(pathname: string, data: Buffer, o: PutOptions): Promise<string>;
   url(pathname: string): string;
   list(prefix: string, limit?: number): Promise<Listed[]>;
@@ -45,12 +52,20 @@ class BlobStore implements Store {
   async exists(pathname: string) {
     try { await blobHead(pathname); return true; } catch (e) { if (e instanceof BlobNotFoundError) return false; throw e; }
   }
+  async stat(pathname: string) {
+    try { const h = await blobHead(pathname); return { type: h.contentType, etag: h.etag }; } catch (e) { if (e instanceof BlobNotFoundError) return null; throw e; }
+  }
   async put(pathname: string, data: Buffer, o: PutOptions) {
-    const r = await blobPut(pathname, data, {
-      access: 'public', addRandomSuffix: false, allowOverwrite: o.overwrite ?? false,
-      contentType: o.contentType, cacheControlMaxAge: o.maxAge ?? 31536000,
-    });
-    return r.url;
+    try {
+      const r = await blobPut(pathname, data, {
+        access: 'public', addRandomSuffix: false, allowOverwrite: o.overwrite ?? false,
+        contentType: o.contentType, cacheControlMaxAge: o.maxAge ?? 31536000, ...(o.ifMatch ? { ifMatch: o.ifMatch } : {}),
+      });
+      return r.url;
+    } catch (e) {
+      if (e instanceof BlobPreconditionFailedError) throw new StoreConflict(`${pathname} changed`);
+      throw e;
+    }
   }
   url(pathname: string) { return `${this.base}/${pathname}`; }
   async list(prefix: string, limit = 1000) {
@@ -75,12 +90,22 @@ class DirStore implements Store {
   }
   async get(p: string) { try { return fs.readFileSync(this.file(p)); } catch { return null; } }
   async exists(p: string) { return fs.existsSync(this.file(p)); }
+  async stat(p: string) { return this.statSync(p); }
+  private statSync(p: string): Stat | null {
+    const f = this.file(p);
+    if (!fs.existsSync(f)) return null;
+    let type = 'application/octet-stream';
+    try { type = fs.readFileSync(f + '.type', 'utf8'); } catch { /* written before types were kept */ }
+    return { type, etag: createHash('sha1').update(fs.readFileSync(f)).digest('hex') };
+  }
   async put(p: string, data: Buffer, o: PutOptions) {
     const f = this.file(p);
-    if (!o.overwrite && fs.existsSync(f)) throw new Error(`blob exists: ${p}`);
+    if (o.ifMatch !== undefined) { if (this.statSync(p)?.etag !== o.ifMatch) throw new StoreConflict(`${p} changed`); }
+    else if (!o.overwrite && fs.existsSync(f)) throw new Error(`blob exists: ${p}`);
     fs.mkdirSync(path.dirname(f), { recursive: true });
     fs.writeFileSync(f + '.tmp', data);
     fs.renameSync(f + '.tmp', f);
+    fs.writeFileSync(f + '.type', o.contentType);
     return this.url(p);
   }
   url(p: string) { return `${this.urlBase}/${p}`; }
@@ -91,7 +116,7 @@ class DirStore implements Store {
       for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
         const f = path.join(dir, e.name);
         if (e.isDirectory()) walk(f);
-        else if (!e.name.endsWith('.tmp')) {
+        else if (!e.name.endsWith('.tmp') && !e.name.endsWith('.type')) {
           const rel = path.relative(this.root, f).split(path.sep).join('/');
           if (rel.startsWith(prefix)) { const st = fs.statSync(f); out.push({ pathname: rel, uploadedAt: st.mtime, size: st.size }); }
         }
@@ -120,10 +145,10 @@ export async function getJSON<T>(pathname: string, fresh = false): Promise<T | n
   return JSON.parse((pathname.endsWith('.gz') ? gunzipSync(b) : b).toString('utf8')) as T;
 }
 
-export async function putJSON(pathname: string, value: unknown, o: { maxAge?: number; overwrite?: boolean } = {}): Promise<string> {
+export async function putJSON(pathname: string, value: unknown, o: { maxAge?: number; overwrite?: boolean; type?: string; ifMatch?: string } = {}): Promise<string> {
   const text = Buffer.from(JSON.stringify(value));
   const gz = pathname.endsWith('.gz');
   return getStore().put(pathname, gz ? gzipSync(text, { level: 6 }) : text, {
-    contentType: gz ? 'application/gzip' : 'application/json', maxAge: o.maxAge, overwrite: o.overwrite,
+    contentType: o.type ?? (gz ? 'application/gzip' : 'application/json'), maxAge: o.maxAge, overwrite: o.overwrite, ifMatch: o.ifMatch,
   });
 }

web/test/bundle.test.ts

@@ -1,14 +1,15 @@
 // The TS bundle must produce the same payload as commit_city/bundle.py from the same records.
 import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 import { test } from 'node:test';
-import { finishBuild, GALLERY, type GalleryEntry, REPLAY_GEN, repoPath, usableGen } from '../lib/city';
+import { finishBuild, readGallery, REPLAY_GEN, repoPath, usableGen } from '../lib/city';
 import { aggregate, budgetCaps, bundle, mergeEvents, mineOf, type Payload } from '../lib/engine/bundle';
 import { replayRepo } from '../lib/engine/repo';
 import type { RepoRecord } from '../lib/engine/replay';
 import { sign } from '../lib/secret';
-import { getJSON, putJSON, setStore, type Store } from '../lib/store';
+import { getJSON, putJSON, setStore, type Store, StoreConflict } from '../lib/store';
 import { localTransport, Repo, REPO_ROOT, sh, tmpdir } from './helpers';
 
 const DAY = 86400;
@@ -101,11 +102,15 @@ test('aggregate helpers match the python rules', () => {
 
 class MemStore implements Store {
   blobs = new Map<string, Buffer>();
+  types = new Map<string, string>();
   async get(p: string) { return this.blobs.get(p) ?? null; }
   async exists(p: string) { return this.blobs.has(p); }
-  async put(p: string, data: Buffer, o: { overwrite?: boolean }) {
-    if (!o.overwrite && this.blobs.has(p)) throw new Error(`blob exists: ${p}`);
+  async stat(p: string) { const b = this.blobs.get(p); return b ? { type: this.types.get(p)!, etag: createHash('sha1').update(b).digest('hex') } : null; }
+  async put(p: string, data: Buffer, o: { contentType: string; overwrite?: boolean; ifMatch?: string }) {
+    if (o.ifMatch !== undefined) { if ((await this.stat(p))?.etag !== o.ifMatch) throw new StoreConflict(`${p} changed`); }
+    else if (!o.overwrite && this.blobs.has(p)) throw new Error(`blob exists: ${p}`);
     this.blobs.set(p, data);
+    this.types.set(p, o.contentType);
     return this.url(p);
   }
   url(p: string) { return `mem:/${p}`; }
@@ -135,7 +140,7 @@ test('finish step: records of another version are never drawn; meta and gallery
   assert.equal(meta.repos, 1);
   assert.equal(meta.commits, 7);                                    // k = 2..8, everyone's
   assert.deepEqual(meta.who, { human: 1, bot: 1, agent: 5, agents: [['claude', 2], ['copilot', 2], ['cursor', 1]] });
-  const gallery = (await getJSON<GalleryEntry[]>(GALLERY))!;
+  const gallery = await readGallery();
   assert.deepEqual([gallery[0].agent, gallery[0].bot], [0.714, 0.143]);
   const payload = (await getJSON<Payload>(`cities/me/${meta.v}.json.gz`))!;
   assert.deepEqual(payload.who, ['human', 'bot', 'claude', 'copilot', 'cursor']);

web/test/gallery.test.ts

@@ -0,0 +1,76 @@
+import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
+import { test } from 'node:test';
+import { addToGallery, type CityMeta, GALLERY, readGallery, removeFromGallery } from '../lib/city';
+import { putJSON, setStore, type Store, StoreConflict } from '../lib/store';
+
+// The blob store as production showed it: a read of an overwritten blob can return the first copy ever written
+// (a stale CDN), while head requests (stat) and conditional writes see the store itself.
+class StaleCdnStore implements Store {
+  blobs = new Map<string, { data: Buffer; type: string }>();
+  first = new Map<string, Buffer>();
+  async get(p: string) { return this.first.get(p) ?? null; }
+  async exists(p: string) { return this.blobs.has(p); }
+  async stat(p: string) { const b = this.blobs.get(p); return b ? { type: b.type, etag: createHash('sha1').update(b.data).digest('hex') } : null; }
+  async put(p: string, data: Buffer, o: { contentType: string; overwrite?: boolean; ifMatch?: string }) {
+    await new Promise(r => setTimeout(r, Math.random() * 5));     // writers interleave, as concurrent builds do
+    if (o.ifMatch !== undefined) { if ((await this.stat(p))?.etag !== o.ifMatch) throw new StoreConflict(`${p} changed`); }
+    else if (!o.overwrite && this.blobs.has(p)) throw new Error(`blob exists: ${p}`);
+    this.blobs.set(p, { data, type: o.contentType });
+    if (!this.first.has(p)) this.first.set(p, data);
+    return `mem:/${p}`;
+  }
+  url(p: string) { return `mem:/${p}`; }
+  async list(prefix: string) { return [...this.blobs.keys()].filter(k => k.startsWith(prefix)).map(pathname => ({ pathname, uploadedAt: new Date(0), size: 0 })); }
+  async del(p: string) { this.blobs.delete(p); }
+}
+
+const meta = (key: string, commits: number) => ({
+  key, target: key, kind: key.includes('/') ? 'repo' : 'user', avatar: '', commits, repos: 1, lines: 10, v: Date.now(),
+  who: { human: commits, bot: 0, agent: 0, agents: [] },
+}) as unknown as CityMeta;
+
+test('builds finishing together all reach the gallery, through a stale CDN copy', async () => {
+  setStore(new StaleCdnStore());
+  await addToGallery(meta('first', 1));
+  await Promise.all(['a', 'b', 'c', 'd', 'e'].map((k, i) => addToGallery(meta(k, 10 + i))));
+  const keys = (await readGallery()).map(e => e.key);
+  assert.equal(keys.length, 6);
+  assert.deepEqual(new Set(keys), new Set(['first', 'a', 'b', 'c', 'd', 'e']));
+  assert.equal(keys.at(-1), 'first');
+});
+
+test('a rebuilt city moves to the front once, with its new numbers', async () => {
+  setStore(new StaleCdnStore());
+  await addToGallery(meta('vercel', 14661));
+  await addToGallery(meta('torvalds', 863));
+  await addToGallery(meta('vercel', 21136));
+  const g = await readGallery();
+  assert.deepEqual(g.map(e => [e.key, e.commits]), [['vercel', 21136], ['torvalds', 863]]);
+});
+
+test('hiding an owner takes their cities out of the gallery', async () => {
+  setStore(new StaleCdnStore());
+  for (const k of ['me', 'me/app', 'meadow', 'you']) await addToGallery(meta(k, 5));
+  await removeFromGallery('Me');
+  assert.deepEqual((await readGallery()).map(e => e.key), ['you', 'meadow']);
+});
+
+test('a gallery written before versions is read, and carried into the first version', async () => {
+  setStore(new StaleCdnStore());
+  await putJSON(GALLERY, [{ ...meta('old', 3), agent: 0, bot: 0 }], { overwrite: true });
+  assert.deepEqual((await readGallery()).map(e => e.key), ['old']);
+  await addToGallery(meta('new', 4));
+  assert.deepEqual((await readGallery()).map(e => e.key), ['new', 'old']);
+});
+
+test('the landing lists show one city per repo, under the name GitHub uses now', async () => {
+  setStore(new StaleCdnStore());
+  const { GET } = await import('../lib/routes/gallery');
+  await addToGallery({ ...meta('facebook/react', 685), target: 'react/react' });      // built before the repo moved
+  await addToGallery(meta('react/react', 689));
+  await putJSON('meta/react/react.json', meta('react/react', 689));
+  const g = await (await GET(new Request('https://commit.city/api/gallery'))).json() as { featured: { key: string }[]; recent: { key: string; commits: number }[] };
+  assert.deepEqual(g.recent.map(e => [e.key, e.commits]), [['react/react', 689]]);
+  assert.deepEqual(g.featured.map(e => e.key), ['react/react']);
+});