diff --git a/web/lib/city.ts b/web/lib/city.ts index 7597062..179637c 100644 --- a/web/lib/city.ts +++ b/web/lib/city.ts @@ -7,7 +7,7 @@ import { replayRepo } from './engine/repo.js'; import { HttpError } from './http.js'; import { sign, verify } from './secret.js'; import { type Sky, skyline } from './skyline.js'; -import { getJSON, getStore, putJSON } from './store.js'; +import { getJSON, getStore, putJSON, StoreConflict } from './store.js'; import { cityKey } from './target.js'; // the record version is part of the key, so records of an older shape are cache misses and get replayed again. @@ -55,6 +55,7 @@ export interface GalleryEntry { agent: number; bot: number; // shares of the commits, 0..1 to 3 decimals } export const GALLERY = 'gallery/recent.json'; +const galleryAt = (id: string) => `gallery/recent/${id}.json`; const GALLERY_MAX = 60; export interface Settings { hidden?: boolean; exclude?: string[]; updated?: string } @@ -208,10 +209,41 @@ export async function finishBuild(ticketToken: string, planToken: string): Promi return meta; } -async function addToGallery(m: CityMeta) { - const list = (await getJSON(GALLERY, true)) ?? []; - const next = [galleryEntry(m), ...list.filter(e => e.key !== m.key)].slice(0, GALLERY_MAX); - await putJSON(GALLERY, next, { overwrite: true, maxAge: 60 }); +// gallery/recent.json is rewritten by every build, and the blob CDN serves an overwritten blob long past its +// max-age: in production a copy 30 minutes stale showed vercel's earlier numbers, and a build that read it and +// wrote it back undid the build before it (sindresorhus). So each version is also written where it is never +// overwritten, and the rewritten blob's content type, which a head request reads from the store itself, names +// the current one. Writes are conditional on its etag, so two builds finishing together both get in. +async function galleryNow(): Promise<{ list: GalleryEntry[]; etag: string | null }> { + const st = await getStore().stat(GALLERY); + if (!st) return { list: [], etag: null }; + const id = /;\s*version=([\w-]+)/.exec(st.type)?.[1]; + return { list: (await getJSON(id ? galleryAt(id) : GALLERY)) ?? [], etag: st.etag }; +} + +export async function readGallery(): Promise { + return (await galleryNow()).list; +} + +async function updateGallery(change: (list: GalleryEntry[]) => GalleryEntry[] | null) { + for (let attempt = 1; ; attempt++) { + const { list, etag } = await galleryNow(); + const next = change(list); + if (!next) return; + const id = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + await putJSON(galleryAt(id), next); + try { + await putJSON(GALLERY, next, { maxAge: 60, type: `application/json; version=${id}`, ...(etag ? { ifMatch: etag } : { overwrite: false }) }); + return; + } catch (e) { + if (!(e instanceof StoreConflict) || attempt >= 8) throw e; // another build wrote first: apply this change to theirs + await new Promise(r => setTimeout(r, Math.random() * 100 * attempt)); + } + } +} + +export async function addToGallery(m: CityMeta) { + await updateGallery(list => [galleryEntry(m), ...list.filter(e => e.key !== m.key)].slice(0, GALLERY_MAX)); } const share = (n: number, total: number) => (total ? Math.round((1000 * n) / total) / 1000 : 0); @@ -224,10 +256,11 @@ export function galleryEntry(m: CityMeta): GalleryEntry { } export async function removeFromGallery(owner: string) { - const list = (await getJSON(GALLERY, true)) ?? []; const o = owner.toLowerCase(); - const next = list.filter(e => e.key !== o && !e.key.startsWith(o + '/')); - if (next.length !== list.length) await putJSON(GALLERY, next, { overwrite: true, maxAge: 60 }); + await updateGallery(list => { + const next = list.filter(e => e.key !== o && !e.key.startsWith(o + '/')); + return next.length !== list.length ? next : null; + }); } export function summarize(payload: Payload, base: Pick): CityMeta { diff --git a/web/lib/routes/gallery.ts b/web/lib/routes/gallery.ts index 2574440..dd09937 100644 --- a/web/lib/routes/gallery.ts +++ b/web/lib/routes/gallery.ts @@ -1,20 +1,28 @@ // Cities for the landing page: a hand-picked few, and the most recently built. -import { GALLERY, type GalleryEntry, readMeta } from '../city.js'; +import { type GalleryEntry, readGallery, readMeta } from '../city.js'; import { handle, json } from '../http.js'; -import { getJSON } from '../store.js'; -const FEATURED = ['torvalds', 'sindresorhus', 'vercel', 'facebook/react', 'denoland/deno', 'gaearon', 'karpathy', 'rust-lang/rust', 'withastro', 'antfu']; +const FEATURED = ['torvalds', 'sindresorhus', 'vercel', 'react/react', 'denoland/deno', 'gaearon', 'karpathy', 'rust-lang/rust', 'withastro', 'antfu']; + +// a city built under a repo's old name (facebook/react, now react/react) is a second city of the same repo: the +// list shows the one under the name GitHub uses now +const current = (e: GalleryEntry) => e.key === e.target.toLowerCase() + (e.key.endsWith('@all') ? '@all' : ''); export const GET = handle(async () => { - const [recent, metas] = await Promise.all([ - getJSON(GALLERY).catch(() => null), - Promise.all(FEATURED.map(k => readMeta(k).catch(() => null))), - ]); - const featured: GalleryEntry[] = metas.filter(m => m !== null).map(m => ({ - key: m.key, target: m.target, kind: m.kind, avatar: m.avatar, commits: m.commits, repos: m.repos, lines: m.lines, v: m.v, - // commit shares as galleryEntry() in city.ts writes them; metas built before the agent lens have no `who` - agent: m.who && m.commits ? Math.round((1000 * m.who.agent) / m.commits) / 1000 : 0, - bot: m.who && m.commits ? Math.round((1000 * m.who.bot) / m.commits) / 1000 : 0, - })); - return json({ featured, recent: recent ?? [] }, 200, { 'Cache-Control': 'public, s-maxage=60, stale-while-revalidate=600' }); + const recent = (await readGallery().catch(() => [] as GalleryEntry[])).filter(current); + // a featured city built lately takes its entry from the list, so both lists show the same build; its meta otherwise + const latest = new Map(recent.map(e => [e.key, e])); + const metas = await Promise.all(FEATURED.map(k => (latest.has(k) ? null : readMeta(k).catch(() => null)))); + const featured: GalleryEntry[] = FEATURED.flatMap((k, i) => { + const e = latest.get(k), m = metas[i]; + if (e) return [e]; + if (!m || m.key !== m.target.toLowerCase()) return []; + return [{ + key: m.key, target: m.target, kind: m.kind, avatar: m.avatar, commits: m.commits, repos: m.repos, lines: m.lines, v: m.v, + // commit shares as galleryEntry() in city.ts writes them; metas built before the agent lens have no `who` + agent: m.who && m.commits ? Math.round((1000 * m.who.agent) / m.commits) / 1000 : 0, + bot: m.who && m.commits ? Math.round((1000 * m.who.bot) / m.commits) / 1000 : 0, + }]; + }); + return json({ featured, recent }, 200, { 'Cache-Control': 'public, s-maxage=60, stale-while-revalidate=600' }); }); diff --git a/web/lib/store.ts b/web/lib/store.ts index 8f64937..4140cf5 100644 --- a/web/lib/store.ts +++ b/web/lib/store.ts @@ -1,16 +1,23 @@ // Storage: Vercel Blob in production (a public store: everything in it is public GitHub data), a local // directory in development and tests. -import { BlobNotFoundError, del as blobDel, get as blobGet, head as blobHead, list as blobList, put as blobPut } from '@vercel/blob'; +import { BlobNotFoundError, BlobPreconditionFailedError, del as blobDel, get as blobGet, head as blobHead, list as blobList, put as blobPut } from '@vercel/blob'; +import { createHash } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { gunzipSync, gzipSync } from 'node:zlib'; -export interface PutOptions { contentType: string; maxAge?: number; overwrite?: boolean } +export interface PutOptions { contentType: string; maxAge?: number; overwrite?: boolean; ifMatch?: string } export interface Listed { pathname: string; uploadedAt: Date; size: number } +export interface Stat { type: string; etag: string } + +/** A conditional write (ifMatch) lost: the blob changed after its etag was read. */ +export class StoreConflict extends Error {} export interface Store { get(pathname: string, fresh?: boolean): Promise; exists(pathname: string): Promise; + /** Content type and etag as the store itself holds them, never a CDN copy; null if there is no such blob. */ + stat(pathname: string): Promise; put(pathname: string, data: Buffer, o: PutOptions): Promise; url(pathname: string): string; list(prefix: string, limit?: number): Promise; @@ -45,12 +52,20 @@ class BlobStore implements Store { async exists(pathname: string) { try { await blobHead(pathname); return true; } catch (e) { if (e instanceof BlobNotFoundError) return false; throw e; } } + async stat(pathname: string) { + try { const h = await blobHead(pathname); return { type: h.contentType, etag: h.etag }; } catch (e) { if (e instanceof BlobNotFoundError) return null; throw e; } + } async put(pathname: string, data: Buffer, o: PutOptions) { - const r = await blobPut(pathname, data, { - access: 'public', addRandomSuffix: false, allowOverwrite: o.overwrite ?? false, - contentType: o.contentType, cacheControlMaxAge: o.maxAge ?? 31536000, - }); - return r.url; + try { + const r = await blobPut(pathname, data, { + access: 'public', addRandomSuffix: false, allowOverwrite: o.overwrite ?? false, + contentType: o.contentType, cacheControlMaxAge: o.maxAge ?? 31536000, ...(o.ifMatch ? { ifMatch: o.ifMatch } : {}), + }); + return r.url; + } catch (e) { + if (e instanceof BlobPreconditionFailedError) throw new StoreConflict(`${pathname} changed`); + throw e; + } } url(pathname: string) { return `${this.base}/${pathname}`; } async list(prefix: string, limit = 1000) { @@ -75,12 +90,22 @@ class DirStore implements Store { } async get(p: string) { try { return fs.readFileSync(this.file(p)); } catch { return null; } } async exists(p: string) { return fs.existsSync(this.file(p)); } + async stat(p: string) { return this.statSync(p); } + private statSync(p: string): Stat | null { + const f = this.file(p); + if (!fs.existsSync(f)) return null; + let type = 'application/octet-stream'; + try { type = fs.readFileSync(f + '.type', 'utf8'); } catch { /* written before types were kept */ } + return { type, etag: createHash('sha1').update(fs.readFileSync(f)).digest('hex') }; + } async put(p: string, data: Buffer, o: PutOptions) { const f = this.file(p); - if (!o.overwrite && fs.existsSync(f)) throw new Error(`blob exists: ${p}`); + if (o.ifMatch !== undefined) { if (this.statSync(p)?.etag !== o.ifMatch) throw new StoreConflict(`${p} changed`); } + else if (!o.overwrite && fs.existsSync(f)) throw new Error(`blob exists: ${p}`); fs.mkdirSync(path.dirname(f), { recursive: true }); fs.writeFileSync(f + '.tmp', data); fs.renameSync(f + '.tmp', f); + fs.writeFileSync(f + '.type', o.contentType); return this.url(p); } url(p: string) { return `${this.urlBase}/${p}`; } @@ -91,7 +116,7 @@ class DirStore implements Store { for (const e of fs.readdirSync(dir, { withFileTypes: true })) { const f = path.join(dir, e.name); if (e.isDirectory()) walk(f); - else if (!e.name.endsWith('.tmp')) { + else if (!e.name.endsWith('.tmp') && !e.name.endsWith('.type')) { const rel = path.relative(this.root, f).split(path.sep).join('/'); if (rel.startsWith(prefix)) { const st = fs.statSync(f); out.push({ pathname: rel, uploadedAt: st.mtime, size: st.size }); } } @@ -120,10 +145,10 @@ export async function getJSON(pathname: string, fresh = false): Promise { +export async function putJSON(pathname: string, value: unknown, o: { maxAge?: number; overwrite?: boolean; type?: string; ifMatch?: string } = {}): Promise { const text = Buffer.from(JSON.stringify(value)); const gz = pathname.endsWith('.gz'); return getStore().put(pathname, gz ? gzipSync(text, { level: 6 }) : text, { - contentType: gz ? 'application/gzip' : 'application/json', maxAge: o.maxAge, overwrite: o.overwrite, + contentType: o.type ?? (gz ? 'application/gzip' : 'application/json'), maxAge: o.maxAge, overwrite: o.overwrite, ifMatch: o.ifMatch, }); } diff --git a/web/test/bundle.test.ts b/web/test/bundle.test.ts index 7b8ec39..d29de91 100644 --- a/web/test/bundle.test.ts +++ b/web/test/bundle.test.ts @@ -1,14 +1,15 @@ // The TS bundle must produce the same payload as commit_city/bundle.py from the same records. import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'node:test'; -import { finishBuild, GALLERY, type GalleryEntry, REPLAY_GEN, repoPath, usableGen } from '../lib/city'; +import { finishBuild, readGallery, REPLAY_GEN, repoPath, usableGen } from '../lib/city'; import { aggregate, budgetCaps, bundle, mergeEvents, mineOf, type Payload } from '../lib/engine/bundle'; import { replayRepo } from '../lib/engine/repo'; import type { RepoRecord } from '../lib/engine/replay'; import { sign } from '../lib/secret'; -import { getJSON, putJSON, setStore, type Store } from '../lib/store'; +import { getJSON, putJSON, setStore, type Store, StoreConflict } from '../lib/store'; import { localTransport, Repo, REPO_ROOT, sh, tmpdir } from './helpers'; const DAY = 86400; @@ -101,11 +102,15 @@ test('aggregate helpers match the python rules', () => { class MemStore implements Store { blobs = new Map(); + types = new Map(); async get(p: string) { return this.blobs.get(p) ?? null; } async exists(p: string) { return this.blobs.has(p); } - async put(p: string, data: Buffer, o: { overwrite?: boolean }) { - if (!o.overwrite && this.blobs.has(p)) throw new Error(`blob exists: ${p}`); + async stat(p: string) { const b = this.blobs.get(p); return b ? { type: this.types.get(p)!, etag: createHash('sha1').update(b).digest('hex') } : null; } + async put(p: string, data: Buffer, o: { contentType: string; overwrite?: boolean; ifMatch?: string }) { + if (o.ifMatch !== undefined) { if ((await this.stat(p))?.etag !== o.ifMatch) throw new StoreConflict(`${p} changed`); } + else if (!o.overwrite && this.blobs.has(p)) throw new Error(`blob exists: ${p}`); this.blobs.set(p, data); + this.types.set(p, o.contentType); return this.url(p); } url(p: string) { return `mem:/${p}`; } @@ -135,7 +140,7 @@ test('finish step: records of another version are never drawn; meta and gallery assert.equal(meta.repos, 1); assert.equal(meta.commits, 7); // k = 2..8, everyone's assert.deepEqual(meta.who, { human: 1, bot: 1, agent: 5, agents: [['claude', 2], ['copilot', 2], ['cursor', 1]] }); - const gallery = (await getJSON(GALLERY))!; + const gallery = await readGallery(); assert.deepEqual([gallery[0].agent, gallery[0].bot], [0.714, 0.143]); const payload = (await getJSON(`cities/me/${meta.v}.json.gz`))!; assert.deepEqual(payload.who, ['human', 'bot', 'claude', 'copilot', 'cursor']); diff --git a/web/test/gallery.test.ts b/web/test/gallery.test.ts new file mode 100644 index 0000000..419f023 --- /dev/null +++ b/web/test/gallery.test.ts @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { test } from 'node:test'; +import { addToGallery, type CityMeta, GALLERY, readGallery, removeFromGallery } from '../lib/city'; +import { putJSON, setStore, type Store, StoreConflict } from '../lib/store'; + +// The blob store as production showed it: a read of an overwritten blob can return the first copy ever written +// (a stale CDN), while head requests (stat) and conditional writes see the store itself. +class StaleCdnStore implements Store { + blobs = new Map(); + first = new Map(); + async get(p: string) { return this.first.get(p) ?? null; } + async exists(p: string) { return this.blobs.has(p); } + async stat(p: string) { const b = this.blobs.get(p); return b ? { type: b.type, etag: createHash('sha1').update(b.data).digest('hex') } : null; } + async put(p: string, data: Buffer, o: { contentType: string; overwrite?: boolean; ifMatch?: string }) { + await new Promise(r => setTimeout(r, Math.random() * 5)); // writers interleave, as concurrent builds do + if (o.ifMatch !== undefined) { if ((await this.stat(p))?.etag !== o.ifMatch) throw new StoreConflict(`${p} changed`); } + else if (!o.overwrite && this.blobs.has(p)) throw new Error(`blob exists: ${p}`); + this.blobs.set(p, { data, type: o.contentType }); + if (!this.first.has(p)) this.first.set(p, data); + return `mem:/${p}`; + } + url(p: string) { return `mem:/${p}`; } + async list(prefix: string) { return [...this.blobs.keys()].filter(k => k.startsWith(prefix)).map(pathname => ({ pathname, uploadedAt: new Date(0), size: 0 })); } + async del(p: string) { this.blobs.delete(p); } +} + +const meta = (key: string, commits: number) => ({ + key, target: key, kind: key.includes('/') ? 'repo' : 'user', avatar: '', commits, repos: 1, lines: 10, v: Date.now(), + who: { human: commits, bot: 0, agent: 0, agents: [] }, +}) as unknown as CityMeta; + +test('builds finishing together all reach the gallery, through a stale CDN copy', async () => { + setStore(new StaleCdnStore()); + await addToGallery(meta('first', 1)); + await Promise.all(['a', 'b', 'c', 'd', 'e'].map((k, i) => addToGallery(meta(k, 10 + i)))); + const keys = (await readGallery()).map(e => e.key); + assert.equal(keys.length, 6); + assert.deepEqual(new Set(keys), new Set(['first', 'a', 'b', 'c', 'd', 'e'])); + assert.equal(keys.at(-1), 'first'); +}); + +test('a rebuilt city moves to the front once, with its new numbers', async () => { + setStore(new StaleCdnStore()); + await addToGallery(meta('vercel', 14661)); + await addToGallery(meta('torvalds', 863)); + await addToGallery(meta('vercel', 21136)); + const g = await readGallery(); + assert.deepEqual(g.map(e => [e.key, e.commits]), [['vercel', 21136], ['torvalds', 863]]); +}); + +test('hiding an owner takes their cities out of the gallery', async () => { + setStore(new StaleCdnStore()); + for (const k of ['me', 'me/app', 'meadow', 'you']) await addToGallery(meta(k, 5)); + await removeFromGallery('Me'); + assert.deepEqual((await readGallery()).map(e => e.key), ['you', 'meadow']); +}); + +test('a gallery written before versions is read, and carried into the first version', async () => { + setStore(new StaleCdnStore()); + await putJSON(GALLERY, [{ ...meta('old', 3), agent: 0, bot: 0 }], { overwrite: true }); + assert.deepEqual((await readGallery()).map(e => e.key), ['old']); + await addToGallery(meta('new', 4)); + assert.deepEqual((await readGallery()).map(e => e.key), ['new', 'old']); +}); + +test('the landing lists show one city per repo, under the name GitHub uses now', async () => { + setStore(new StaleCdnStore()); + const { GET } = await import('../lib/routes/gallery'); + await addToGallery({ ...meta('facebook/react', 685), target: 'react/react' }); // built before the repo moved + await addToGallery(meta('react/react', 689)); + await putJSON('meta/react/react.json', meta('react/react', 689)); + const g = await (await GET(new Request('https://commit.city/api/gallery'))).json() as { featured: { key: string }[]; recent: { key: string; commits: number }[] }; + assert.deepEqual(g.recent.map(e => [e.key, e.commits]), [['react/react', 689]]); + assert.deepEqual(g.featured.map(e => e.key), ['react/react']); +});