fix(design-studio): verify scan source before publishing

Author: Bill Leoutsakos

scripts/design-conformance/source-revision.ts

@@ -0,0 +1,20 @@
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import path from 'node:path'
+
+/** Identity of HEAD, tracked edits, and nonignored new files in a local checkout. */
+export function workingTreeRevision(repo: string): string {
+  const git = (args: string[], maxBuffer?: number) =>
+    execFileSync('git', args, { cwd: repo, ...(maxBuffer ? { maxBuffer } : {}) })
+  const head = git(['rev-parse', 'HEAD']).toString().trim()
+  const diff = git(['diff', '--binary', 'HEAD'], 64 * 1024 * 1024)
+  const untracked = git(['ls-files', '--others', '--exclude-standard', '-z'])
+    .toString()
+    .split('\0')
+    .filter(Boolean)
+    .sort()
+  const digest = createHash('sha256').update(head).update(diff)
+  for (const file of untracked) digest.update(file).update(readFileSync(path.join(repo, file)))
+  return digest.digest('hex')
+}

scripts/design-scan/identity.ts

@@ -24,7 +24,7 @@ export function scannerIdentity() {
     .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
 
   return {
-    version: 'design-scan/2',
+    version: 'design-scan/3',
     sourceHash: hash(canonical(files)),
     policy: registry.policy,
     contractsHash,

scripts/design-scan/report.ts

@@ -51,7 +51,7 @@ export function writeResults(
       path.join(temp, 'scan.json'),
       `${JSON.stringify(
         {
-          version: 2,
+          version: 3,
           identity: {
             ...(identity as object),
             commit: inventory.commit,

scripts/design-scan/scan.ts

@@ -10,6 +10,7 @@ import { classifyLayout, type LayoutAllowance } from '#control-analysis/layout-a
 import type { ReviewReport } from '#control-analysis/review'
 import { mergeSourceFindings } from '#control-analysis/review'
 import { inspectionFailure } from '#design-conformance/model'
+import { workingTreeRevision } from '#design-conformance/source-revision'
 import { GitSource } from '#design-conformance/worktree-source'
 import { scannerIdentity } from './identity'
 import { inspectInventory } from './inventory'
@@ -61,6 +62,7 @@ export async function main(argv = process.argv.slice(2)): Promise<number> {
     const output = validateOutput(values.output, values.repo)
     const identity = scannerIdentity()
     const source = new GitSource(values.repo, values.ref ?? 'HEAD', !!values['working-tree'])
+    const sourceRevision = source.mode === 'working-tree' ? workingTreeRevision(source.repo) : null
     let controls: ControlInventory | undefined
     let colourAssignments: ColourAssignmentReport | undefined
     let layoutAllowances: LayoutAllowance[] = []
@@ -99,10 +101,12 @@ export async function main(argv = process.argv.slice(2)): Promise<number> {
     })
     if (!controls) throw new Error('Control analysis did not complete')
     source.assertUnchanged()
+    if (sourceRevision && workingTreeRevision(source.repo) !== sourceRevision)
+      throw new Error('Working-tree source changed during the scan; rerun for a consistent report')
     writeResults(
       output,
       inventory,
-      identity,
+      { ...identity, sourceRevision },
       {
         elapsedMs: performance.now() - start,
         maxRss: process.resourceUsage().maxRSS,

scripts/design-studio/README.md

@@ -18,6 +18,8 @@ The published catalog groups exports and scanner signals into visual treatments.
 
 Update fixture imports and switch cases when exports are renamed or removed. Product Extras require a maintained source mapping for a real component preview; automatically generated style samples are indicative. A generic sample shows one detected class or value at a time, so mutually exclusive source branches are never combined into one specimen.
 
+An entry is `ready` when its preview can render. The preview provenance distinguishes live product or EMCN source from an indicative sample; `ready` does not imply a source-backed fixture.
+
 ## Live previews
 
 Studio previews render the current checkout. Their catalog entries and fixture coverage come from the last published refresh. Source mismatches are shown beside the preview and above the catalog. Unresolved styling/control analysis and scanner limits remain separate from fixture coverage.

scripts/design-studio/refresh.mjs

@@ -17,6 +17,7 @@ import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { groupComponents, groupExtras } from '../../tools/design-studio/_lib/catalog.ts'
 import { productScope } from '../design-conformance/control-scope.ts'
+import { workingTreeRevision } from '../design-conformance/source-revision.ts'
 
 const toolRoot = path.dirname(fileURLToPath(import.meta.url))
 const repo = path.resolve(process.env.SIM_STUDIO_REPO ?? path.join(toolRoot, '../..'))
@@ -495,30 +496,13 @@ function git(args) {
   return result.stdout.trim()
 }
 
-function sourceRevision() {
-  const changed = spawnSync('git', ['diff', '--binary', 'HEAD'], {
-    cwd: repo,
-    maxBuffer: 64 * 1024 * 1024,
-  })
-  if (changed.status !== 0) throw new Error(String(changed.stderr))
-  const untracked = git(['ls-files', '--others', '--exclude-standard', '-z'])
-    .split('\0')
-    .filter(Boolean)
-    .sort()
-  const digest = createHash('sha256')
-    .update(git(['rev-parse', 'HEAD']))
-    .update(changed.stdout)
-  for (const file of untracked) digest.update(file).update(readFileSync(path.join(repo, file)))
-  return digest.digest('hex')
-}
-
 async function main() {
   if (!existsSync(repo) || (!process.versions.bun && !process.env.SIM_STUDIO_SCAN_DIR))
     throw new Error('Run refresh with Bun from the product checkout')
   const relativeOutput = path.relative(repo, outputRoot)
   if (!relativeOutput.startsWith('..') && !path.isAbsolute(relativeOutput))
     throw new Error('Studio output must be outside the product checkout')
-  const initialSourceRevision = sourceRevision()
+  const initialSourceRevision = workingTreeRevision(repo)
   mkdirSync(outputRoot, { recursive: true })
   const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${git(['rev-parse', '--short=10', 'HEAD'])}`
   const runDir = path.join(outputRoot, `run-${runId}`)
@@ -548,9 +532,15 @@ async function main() {
     if (scan.code > 1) throw new Error(`Scanner failed with exit ${scan.code}`)
   }
   const scanReport = JSON.parse(readFileSync(path.join(scanDir, 'scan.json'), 'utf8'))
-  if (scanReport.version !== 2 || !scanReport.inventory?.metadata || !scanReport.controls)
+  if (scanReport.version !== 3 || !scanReport.inventory?.metadata || !scanReport.controls)
     throw new Error('Invalid scanner report')
   const { identity, inventory, controls } = scanReport
+  if (inventory.mode !== 'working-tree' || !identity || identity.sourceRevision === null)
+    throw new Error('Studio requires a working-tree scan of the current checkout')
+  if (identity.sourceRevision !== initialSourceRevision)
+    throw new Error(
+      'Scan source revision does not match the current checkout; rerun studio:refresh'
+    )
   const components = componentInventory(inventory.metadata)
   attachTracedUses(components.entries, controls)
   const extras = extraInventory(inventory.findings)
@@ -584,7 +574,7 @@ async function main() {
       limitations: inventory.limitations,
     },
   }
-  if (sourceRevision() !== manifest.sourceRevision)
+  if (workingTreeRevision(repo) !== manifest.sourceRevision)
     manifest.coverageFailures.push({
       file: '<product-working-tree>',
       reason: 'Source changed during refresh; refresh again.',

scripts/design-studio/refresh.test.mjs

@@ -1,10 +1,11 @@
 import assert from 'node:assert/strict'
 import { spawnSync } from 'node:child_process'
-import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { test } from 'vitest'
+import { workingTreeRevision } from '../design-conformance/source-revision.ts'
 
 const script = path.join(path.dirname(fileURLToPath(import.meta.url)), 'refresh.mjs')
 const metadataScript = `
@@ -28,6 +29,8 @@ function updateMetadata(repo, scan) {
   const report = JSON.parse(readFileSync(file, 'utf8'))
   const { version, sourceHash, exports, diagnostics } = JSON.parse(result.stdout)
   report.inventory.metadata = { version, sourceHash, exports, diagnostics }
+  report.inventory.mode = 'working-tree'
+  report.identity.sourceRevision = workingTreeRevision(repo)
   writeFileSync(file, JSON.stringify(report))
 }
 
@@ -111,7 +114,7 @@ function guardedRefresh() {
     scan,
     'scan.json',
     JSON.stringify({
-      version: 2,
+      version: 3,
       identity: { commit: 'test', treeHash: 'test', scanner: {} },
       inventory: {
         findings: [],
@@ -154,31 +157,28 @@ function guardedRefresh() {
   return { repo, refresh }
 }
 
-test('Studio consumes the unmodified report written by the scanner CLI', () => {
-  const { repo } = guardedRefresh()
-  write(
-    repo,
-    'apps/sim/app/product/page.tsx',
-    "import { Example } from '@sim/emcn'\nexport default function Page() { return <div style={{ color: '#123456' }}><Example /></div> }\n"
-  )
-  const root = path.dirname(repo)
-  const scan = path.join(root, 'real-scan')
-  const scanner = path.resolve('scripts/design-scan/scan.ts')
+function runScanner(repo, source = '--working-tree') {
+  const scan = path.join(path.dirname(repo), `real-scan-${source.slice(2)}`)
   const scanned = spawnSync(
     process.env.DESIGN_TEST_BUN ?? 'bun',
-    ['--no-env-file', scanner, '--repo', repo, '--working-tree', '--output', scan],
+    [
+      '--no-env-file',
+      path.resolve('scripts/design-scan/scan.ts'),
+      '--repo',
+      repo,
+      source,
+      ...(source === '--ref' ? ['HEAD'] : []),
+      '--output',
+      scan,
+    ],
     { encoding: 'utf8', timeout: 120000 }
   )
   assert.ok([0, 1].includes(scanned.status), scanned.stderr)
-  const reportFile = path.join(scan, 'scan.json')
-  const reportBytes = readFileSync(reportFile)
-  const report = JSON.parse(reportBytes.toString())
-  assert.equal(report.version, 2)
-  assert.ok(report.inventory.metadata.exports.Example)
-  assert.ok(report.inventory.findings.some((finding) => finding.value === '#123456'))
+  return scan
+}
 
-  const output = path.join(root, 'real-studio')
-  const refreshed = spawnSync('node', [script], {
+function publishScan(repo, scan, output) {
+  return spawnSync('node', [script], {
     env: {
       ...process.env,
       SIM_STUDIO_REPO: repo,
@@ -188,6 +188,26 @@ test('Studio consumes the unmodified report written by the scanner CLI', () => {
     encoding: 'utf8',
     timeout: 120000,
   })
+}
+
+test('Studio consumes the unmodified report written by the scanner CLI', () => {
+  const { repo } = guardedRefresh()
+  write(
+    repo,
+    'apps/sim/app/product/page.tsx',
+    "import { Example } from '@sim/emcn'\nexport default function Page() { return <div style={{ color: '#123456' }}><Example /></div> }\n"
+  )
+  const root = path.dirname(repo)
+  const scan = runScanner(repo)
+  const reportFile = path.join(scan, 'scan.json')
+  const reportBytes = readFileSync(reportFile)
+  const report = JSON.parse(reportBytes.toString())
+  assert.equal(report.version, 3)
+  assert.ok(report.inventory.metadata.exports.Example)
+  assert.ok(report.inventory.findings.some((finding) => finding.value === '#123456'))
+
+  const output = path.join(root, 'real-studio')
+  const refreshed = publishScan(repo, scan, output)
   assert.ok([0, 1].includes(refreshed.status), refreshed.stderr)
   assert.deepEqual(readFileSync(reportFile), reportBytes)
   const pointer = JSON.parse(readFileSync(path.join(output, 'latest.json'), 'utf8'))
@@ -207,6 +227,53 @@ test('Studio consumes the unmodified report written by the scanner CLI', () => {
   )
 }, 120000)
 
+for (const [name, change] of [
+  [
+    'tracked edit',
+    (repo) =>
+      write(
+        repo,
+        'packages/emcn/src/components/example/example.tsx',
+        "export const Example = ({size, variant}: {size?: 'sm' | 'lg'; variant?: 'plain' | 'filled'}) => <button data-changed />\n"
+      ),
+  ],
+  [
+    'new product file',
+    (repo) =>
+      write(
+        repo,
+        'apps/sim/app/product/new.tsx',
+        'export const NewProductSurface = () => <button />\n'
+      ),
+  ],
+]) {
+  test(`a ${name} makes an injected scan stale without replacing the latest publication`, () => {
+    const { repo } = guardedRefresh()
+    const scan = runScanner(repo)
+    const output = path.join(path.dirname(repo), 'real-studio')
+    const first = publishScan(repo, scan, output)
+    assert.ok([0, 1].includes(first.status), first.stderr)
+    const pointer = path.join(output, 'latest.json')
+    const published = readFileSync(pointer)
+
+    change(repo)
+    const stale = publishScan(repo, scan, output)
+    assert.equal(stale.status, 2, stale.stderr)
+    assert.match(stale.stderr, /scan.*source revision|source revision.*scan/i)
+    assert.deepEqual(readFileSync(pointer), published)
+  }, 120000)
+}
+
+test('an immutable scan cannot publish a current Studio catalog', () => {
+  const { repo } = guardedRefresh()
+  const scan = runScanner(repo, '--ref')
+  const output = path.join(path.dirname(repo), 'real-studio')
+  const result = publishScan(repo, scan, output)
+  assert.equal(result.status, 2, result.stderr)
+  assert.match(result.stderr, /working.tree scan/i)
+  assert.equal(existsSync(path.join(output, 'latest.json')), false)
+}, 120000)
+
 test('refresh tracks source changes in imported stylesheets', () => {
   const { repo, refresh } = guardedRefresh()
   const first = refresh()
@@ -393,7 +460,7 @@ test('refresh catalogs every detected treatment and new EMCN export', () => {
     scan,
     'scan.json',
     JSON.stringify({
-      version: 2,
+      version: 3,
       identity: { commit: 'test-head', treeHash: 'test-tree', scanner: {} },
       inventory: {
         findings: [

tools/design-studio/_components/generated-gallery.tsx

@@ -512,7 +512,9 @@ function TreatmentDetail({
               ? 'Needs fixture'
               : failedCount
                 ? `${failedCount} fixture gaps`
-                : 'Live fixture available'}
+                : entry.kind === 'extra' && entry.previewKind !== 'source-component'
+                  ? 'Indicative preview'
+                  : 'Live source preview'}
         </span>
       </div>
 

tools/design-studio/_lib/manifest.ts

@@ -1,10 +1,10 @@
 import 'server-only'
 
 import { execFileSync } from 'node:child_process'
-import { createHash } from 'node:crypto'
 import { existsSync, readFileSync } from 'node:fs'
 import { homedir } from 'node:os'
 import path from 'node:path'
+import { workingTreeRevision } from '../../../scripts/design-conformance/source-revision'
 
 export interface StudioLocation {
   file: string
@@ -122,21 +122,7 @@ export function getStudioPageManifest(mode: 'components' | 'extras'): {
       cwd: process.env.SIM_STUDIO_REPO ?? process.cwd(),
       encoding: 'utf8',
     }).trim()
-    const head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).trim()
-    const diff = execFileSync('git', ['diff', '--binary', 'HEAD'], {
-      cwd: repo,
-      maxBuffer: 64 * 1024 * 1024,
-    })
-    const untracked = execFileSync('git', ['ls-files', '--others', '--exclude-standard', '-z'], {
-      cwd: repo,
-      encoding: 'utf8',
-    })
-      .split('\0')
-      .filter(Boolean)
-      .sort()
-    const digest = createHash('sha256').update(head).update(diff)
-    for (const name of untracked) digest.update(name).update(readFileSync(path.join(repo, name)))
-    stale = digest.digest('hex') !== manifest.sourceRevision
+    stale = workingTreeRevision(repo) !== manifest.sourceRevision
   } catch {
     stale = true
   }