From a52b2895d9e0d05e848f348dc569988aaa0e52e4 Mon Sep 17 00:00:00 2001 From: Jonathan Burdge Date: Fri, 9 Oct 2026 18:21:37 +0300 Subject: [PATCH] Download source artifacts from standalone dashboard data --- app/app.js | 16 +++++++++-- docs/development.md | 8 ++++++ tests/test_public_browser.mjs | 53 +++++++++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/app/app.js b/app/app.js index 60ba05c..593091a 100644 --- a/app/app.js +++ b/app/app.js @@ -239,7 +239,7 @@ function start(){ catalogueGroups=new Map(); html+=groups.map(f=>{const allRows=all.filter(r=>r.eval===f.name),missing=warnings.filter(w=>w.eval===f.name&&['Missing scoring field','Missing scoring setting','Few-shot mismatch excluded'].includes(w.type)),inconsistent=warnings.filter(w=>w.eval===f.name&&w.type==='Inconsistent scoring settings'),missingTasks=new Set(missing.flatMap(w=>w.tasks)).size;catalogueGroups.set(f.name,{f,missing});const langs=[...new Set(f.tasks.flatMap(t=>languages(t.rows[0]).map(languageLabel)))].sort();return '
'+esc(f.name)+' ('+f.tasks.length+')'+esc(f.category)+''+scoringOptions(f,allRows)+(missing.length?''+missingTasks+' tasks missing configured field/settings':'')+(inconsistent.length?'Inconsistent scoring settings':'')+''+esc(langs.length>4?langs.slice(0,3).join(', ')+' + '+(langs.length-3)+' more':langs.join(', '))+''+esc(normalizationLabel(f))+(f.warning?'Config warning':'')+''+selectionInfo(f)+normalizationInfo(f)+aggregationInfo(f)+inconsistent.map(w=>'

'+esc(w.model+': '+w.detail)+'

').join('')+'
'+(groups.length===1?catalogueTasks(f,missing):'')+'
'+'
';}).join(''); if(!groups.length)html+='

No evals match the filters.

'; - html+='
Scoring assumptions and source files
    '+(scheme.notes||[]).map(n=>'
  1. '+esc(n)+'
    2. ').join('')+'

Source row audit · Language assignments · Analysis JSON

'+esc(DATA.source)+' · SHA-256 '+esc(DATA.sha256)+'

'; + html+='
Scoring assumptions and source files
    '+(scheme.notes||[]).map(n=>'
  1. '+esc(n)+'
    2. ').join('')+'

Download the original build snapshot. These files describe the published inputs, before any browser uploads or edits.

· ·

'+(DATA.sources.length?'
'+table(['Source CSV','SHA-256'],DATA.sources.map(s=>''+td(s.file)+''+esc(s.sha256)+''))+'
':'

No CSV files were embedded in this build.

')+'
'; const configControls='
Global eval catalogue

The catalogue defines matching, categories, scoring fields, normalization, and language assignments for every known eval. It does not require models to run all those evals.

Normalization: each eval lists its baseline, formula, and sources below. Chance correction and component aggregation affect calculated scores; individual raw scores stay unchanged. acc_norm is length-normalized answer scoring, not chance correction.

Catalogue export includes every eval and language in one portable YAML file.

Active catalogue: '+esc(catalogue.name)+' · '+catalogue.languages.reduce((n,g)=>n+g.tasks.length,0)+' explicit task language assignments.

Weighting profile and optional eval set

Weights apply independently of the eval set. Any available compares shared recognized measurements and warns on differences. A named set requires all known catalogue tasks for each listed eval, except excluded languages. It can override metric, metric_filter, and shots for a whole eval. Missing requirements warn; extra measurements are excluded. An incomplete named-set score uses the shared subset with redistributed weights.

Active eval set: '+esc(suite.name)+(suite.exclude?.length?' · Excluded evals: '+suite.exclude.map(esc).join(', '):'')+(suite.exclude_languages?.length?' · Excluded languages across the set: '+suite.exclude_languages.map(esc).join(', '):'')+(suite.evals?.some(e=>e.exclude_languages?.length)?' · Per-eval language exclusions: '+suite.evals.filter(e=>e.exclude_languages?.length).map(e=>esc(e.name)+': '+e.exclude_languages.map(esc).join(', ')).join('; '):'')+'.

All imports are temporary and stay in your browser. Exports save the three inputs separately. Weight exports include your edits and active score calculation.

'; html=html.replace('
',configControls+'
'); @@ -289,7 +289,16 @@ function start(){ const next=resolveConfig(catalogue,suite,config);profile=config;scheme=next;activeProfile=preset; weights={...next.weights};englishWeights={...next.english_weights};state.aggregate=next.aggregate;refreshConfig(); } - function exportFile(content,name){const url=URL.createObjectURL(new Blob([content],{type:'application/yaml'})),a=document.createElement('a');a.href=url;a.download=name;a.click();setTimeout(()=>URL.revokeObjectURL(url),1000);} + function exportFile(content,name,type='application/yaml'){const url=URL.createObjectURL(new Blob([content],{type})),a=document.createElement('a');a.href=url;a.download=name;a.click();setTimeout(()=>URL.revokeObjectURL(url),1000);} + function exportBuildFile(name){ + if(name==='analysis.json'){exportFile(JSON.stringify(DATA,null,2),'analysis.json','application/json');return;} + const rows=name==='row-audit.csv'?DATA.rows:name==='language-metadata.csv'?DATA.metadata:null; + if(!rows?.length)return; + const fields=[...new Set(rows.flatMap(row=>Object.keys(row)))]; + const cell=value=>{const text=value===true?'True':value===false?'False':String(value??'');return /[",\r\n]/.test(text)?'"'+text.replaceAll('"','""')+'"':text;}; + const csv=[fields,...rows.map(row=>fields.map(field=>row[field]))].map(row=>row.map(cell).join(',')).join('\r\n')+'\r\n'; + exportFile(csv,name,'text/csv;charset=utf-8'); + } function exportConfig(){exportFile(serializeCatalogue(catalogue),'catalogue.yaml');} function exportWeights(){exportFile(serializeWeightProfile({...profile,weights:{...weights},english_weights:{...englishWeights},aggregate:state.aggregate}),'weights.yaml');} $('view').onchange=async e=>{try{ @@ -326,7 +335,8 @@ function start(){ } $('view').onclick=e=>{const d=e.target.dataset; const details=e.target.closest('summary')?.parentElement;if(details&&!details.open)populateCatalogue(details); - if(d.openConfig){state.view='config';render();} + if(d.buildDownload)exportBuildFile(d.buildDownload); + else if(d.openConfig){state.view='config';render();} else if(d.scoreCategory){state.scoreCategory=d.scoreCategory;render();$('view').querySelector('#scoreCategory').scrollIntoView({block:'center'});} else if(d.evals||d.category||d.language){clearFilters();if(d.evals)$('eval').value=d.evals;if(d.category)$('category').value=d.category;if(d.language)$('language').value=d.language;if(d.role)$('direction').value=d.role;state.view='comparisons';state.group='variant';render();} else if(d.languageSort){ diff --git a/docs/development.md b/docs/development.md index feb6471..7afe1d3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -28,6 +28,14 @@ Open each generated `index.html` in a browser. The shared build embeds the globa Check the views affected by your change, including their warnings and failed-input behavior. Generated output is self-contained; do not commit it. Private exports belong in `data/`, never in the public `results/` directory. +Under **Eval configuration → Scoring assumptions and source files**, the audit +CSV, language-assignment CSV, and analysis JSON download from the original data +embedded in the HTML. They work when only `index.html` is shared or published. +These downloads describe the original build, before browser uploads or edits; +the YAML export controls save the active configurations. Source CSV filenames +and their individual SHA-256 hashes identify the build inputs. Empty builds +provide analysis JSON and disable the CSV downloads. + ## Run tests Before opening or updating a PR, run the same public checks as CI from your activated Python environment: diff --git a/tests/test_public_browser.mjs b/tests/test_public_browser.mjs index 278f641..72b9234 100644 --- a/tests/test_public_browser.mjs +++ b/tests/test_public_browser.mjs @@ -50,13 +50,44 @@ try{ const navigate=async url=>{await send('Page.navigate',{url});for(let i=0;i<50;i++){await new Promise(r=>setTimeout(r,50));if(await evaluate("document.querySelector('#view')?.textContent.length>0"))return;}throw Error('Dashboard did not render');}; await send('Runtime.discardConsoleEntries'); await send('Runtime.enable');await send('Network.enable');await send('Page.enable'); - await navigate(pathToFileURL(path.join(empty,'index.html')).href); + const standaloneEmpty=path.join(temporary,'standalone-empty');fs.mkdirSync(standaloneEmpty); + fs.copyFileSync(path.join(empty,'index.html'),path.join(standaloneEmpty,'index.html')); + await navigate(pathToFileURL(path.join(standaloneEmpty,'index.html')).href); assert.equal(await evaluate("document.querySelector('#modelA').options.length"),0); assert.equal(await evaluate("document.querySelector('#cards').hidden"),true); assert.match(await evaluate("document.querySelector('#view').textContent"),/Compare your evaluation results/); assert.equal(await evaluate("document.querySelector('#suitePreset').options.length"),shippedSets.files.length); assert.equal(await evaluate("DATA.suites[Number(document.querySelector('#suitePreset').value)].file"),shippedSets.defaultFile); for(const view of ['categories','languages','comparisons','config','warnings','score'])await click('[data-view='+view+']'); + // Audit downloads must work with only index.html, including empty builds. + const checkBuildDownloads=async()=>{ + const previousView=await evaluate("document.querySelector('.tabs button.active').dataset.view"); + await click('[data-view=config]'); + assert.equal(await evaluate("document.querySelectorAll('a[href=\"row-audit.csv\"],a[href=\"language-metadata.csv\"],a[href=\"analysis.json\"]').length"),0); + const expected=await evaluate('DATA'); + assert.equal(await evaluate("document.querySelector('[data-build-download=\"row-audit.csv\"]').disabled"),!expected.rows.length); + assert.equal(await evaluate("document.querySelector('[data-build-download=\"language-metadata.csv\"]').disabled"),!expected.metadata.length); + assert.deepEqual(await evaluate("[...document.querySelectorAll('#buildSources tbody tr')].map(r=>[...r.cells].map(c=>c.textContent))"),expected.sources.map(s=>[s.file,s.sha256])); + await evaluate(`window.originalCreate=URL.createObjectURL;window.originalClick=HTMLAnchorElement.prototype.click;URL.createObjectURL=b=>{window.exportBlob=b;return 'blob:test'};HTMLAnchorElement.prototype.click=function(){window.downloadName=this.download;window.downloadHref=this.href};`); + const download=async(name,type)=>{ + await click('[data-build-download="'+name+'"]'); + assert.equal(await evaluate('downloadName'),name); + assert.equal(await evaluate('downloadHref'),'blob:test'); + assert.equal(await evaluate('exportBlob.type'),type); + return evaluate('exportBlob.text()'); + }; + assert.deepEqual(JSON.parse(await download('analysis.json','application/json')),expected); + const csvValue=v=>v===true?'True':v===false?'False':v==null?'':String(v); + for(const [name,rows] of [['row-audit.csv',expected.rows],['language-metadata.csv',expected.metadata]]){ + if(!rows.length)continue; + const text=await download(name,'text/csv;charset=utf-8'); + const parsed=require('../app/eval_config.js').parseCSV(text); + assert.deepEqual(parsed,rows.map(row=>Object.fromEntries(Object.entries(row).map(([k,v])=>[k,csvValue(v)])))); + } + await evaluate('URL.createObjectURL=originalCreate;HTMLAnchorElement.prototype.click=originalClick'); + await click('[data-view='+previousView+']'); + }; + await checkBuildDownloads(); const upload=async(selector,content,name)=>evaluate(`(async()=>{const dt=new DataTransfer();dt.items.add(new File([${JSON.stringify(content)}],${JSON.stringify(name)}));const e=document.querySelector(${JSON.stringify(selector)});e.files=dt.files;if(e.id==='modelFile')await e.onchange({target:e});else await document.querySelector('#view').onchange({target:e});})()`); await click('[data-view=config]');await upload('#suiteFile',serializeSuite({version:1,name:'Any example',mode:'available'}),'any.yaml');await upload('#configFile',serializeCatalogue(fixture),'catalogue.yaml'); await change('#weightPreset','1'); @@ -91,6 +122,8 @@ try{ await change('#modelB','Example B'); await click('[data-view=config]'); const original=await evaluate("document.querySelector('#cards').textContent"); + // Downloads describe the original build even after temporary CSV/config imports. + await checkBuildDownloads(); // A catalogue without loaded data is not a coverage requirement. const extended=structuredClone(fixture);extended.evals.push({...extended.evals[0],name:'Unused eval',match:{name:'unused'}}); await upload('#configFile',serializeCatalogue(extended),'extended.yaml'); @@ -137,10 +170,22 @@ try{ assert.equal(await evaluate("document.querySelector('#modelA').options.length"),0); // A shared-results build embeds both models and defaults to the real comparison. const results=path.join(temporary,'results');fs.mkdirSync(results); - fs.copyFileSync(path.join(root,'examples/scores.csv'),path.join(results,'example.csv')); + // Include commas, quotes and newlines in audit-only metadata to exercise CSV downloads. + execFileSync('python3',['-c',` +import csv, sys +with open(sys.argv[1], newline='') as source: + rows = list(csv.DictReader(source)) +for row in rows: + row['source_note'] = 'A "quoted", multiline\\nnote' +with open(sys.argv[2], 'w', newline='') as target: + writer = csv.DictWriter(target, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) +`,path.join(root,'examples/scores.csv'),path.join(results,'example.csv')],{cwd:root,stdio:'pipe'}); const shared=path.join(temporary,'shared'); execFileSync('python3',['-m','app.build','--results-dir',results,'--catalogue',path.join(root,'configs/examples/catalogue.yaml'),'--weights',path.join(root,'configs/examples/weights.yaml'),'--eval-set',path.join(root,'configs/examples/eval-set.yaml'),'--output',shared],{cwd:root,stdio:'pipe'}); await navigate(pathToFileURL(path.join(shared,'index.html')).href); + await checkBuildDownloads(); assert.equal(await evaluate("document.querySelector('#modelB').value"),'Example B'); for(const view of ['score','categories','languages','comparisons','config','warnings'])await click('[data-view='+view+']'); assert.equal(await evaluate("document.querySelector('#error').textContent"),''); @@ -156,10 +201,11 @@ try{ assert.equal(new Set(syntheticScores).size,syntheticNames.length); // Explicit defaults can select any embedded pair without hiding older models. const preferred=path.join(temporary,'preferred'); - fs.writeFileSync(path.join(results,'alternative.csv'),fs.readFileSync(path.join(root,'examples/scores.csv'),'utf8').replaceAll('Example A','Alternative A').replaceAll('Example B','Alternative B')); + fs.writeFileSync(path.join(results,'alternative.csv'),fs.readFileSync(path.join(results,'example.csv'),'utf8').replaceAll('Example A','Alternative A').replaceAll('Example B','Alternative B')); fs.writeFileSync(path.join(results,'default.yaml'),'a: Example B\nb: Alternative A\n'); execFileSync('python3',['-m','app.build','--results-dir',results,'--catalogue',path.join(root,'configs/examples/catalogue.yaml'),'--weights',path.join(root,'configs/examples/weights.yaml'),'--eval-set',path.join(root,'configs/examples/eval-set.yaml'),'--output',preferred],{cwd:root,stdio:'pipe'}); await navigate(pathToFileURL(path.join(preferred,'index.html')).href); + await checkBuildDownloads(); assert.equal(await evaluate("document.querySelector('#modelA').value"),'Example B'); assert.equal(await evaluate("document.querySelector('#modelB').value"),'Alternative A'); assert.equal(await evaluate("document.querySelector('#cards .score-card:last-child strong').textContent"),'2.50'); @@ -187,6 +233,7 @@ reconcile(Path(sys.argv[3]), source) assert.match(await evaluate('document.body.textContent'),/Current/); const previewLinks=await evaluate("({main:document.querySelector('a[href=\"../index.html\"]').href,pr:document.querySelector('a[href=\"pr-11/index.html\"]').href})"); await navigate(previewLinks.pr); + await checkBuildDownloads(); assert.match(await evaluate('document.title'),/^\[DEV PREVIEW\] PR #11 · /); assert.equal(await evaluate("document.querySelectorAll('#quickdash-dev-preview').length"),1); assert.match(await evaluate("document.querySelector('#quickdash-dev-preview').textContent"),/DEV PREVIEW.*Unmerged changes.*PR #11.*Main dashboard/);