From a30af1c8e25f57bf46bbb144feb1b454fcb03afc Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:37:28 +0200 Subject: [PATCH 01/10] fix(check): fail closed on --semantic and re-reduce verdicts at check time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check --semantic no longer trusts the precomputed ok persisted in VERIFY.json — the verdict is re-reduced from verdicts[] on every check, so a stale or hand-tampered summary can never green-light the gate (the one false-pass found in the 2026-07-07 skill-test round). It is also fail-closed now: a missing, unreadable, or verdict-less (legacy) VERIFY.json fails the check with an actionable error instead of warning-and-passing, so a green --semantic exit always means the support gate actually engaged. Pass --allow-unverified to restore the old advisory behavior. Co-Authored-By: Claude Fable 5 --- scripts/construct.mjs | 554 +++++++++++++------------ skills/construct/scripts/construct.mjs | 554 +++++++++++++------------ src/check.ts | 53 ++- src/cli.ts | 9 +- src/types.ts | 4 + tests/check.test.ts | 37 +- tests/cli.test.ts | 6 + tests/e2e.test.ts | 23 +- tests/semantic-review.test.ts | 65 ++- 9 files changed, 757 insertions(+), 548 deletions(-) diff --git a/scripts/construct.mjs b/scripts/construct.mjs index 542e52b..08da3f9 100755 --- a/scripts/construct.mjs +++ b/scripts/construct.mjs @@ -2989,8 +2989,224 @@ function renderSRD(brief, evidence, opts) { } // src/check.ts -import { existsSync as existsSync5, readFileSync as readFileSync4, readdirSync as readdirSync3, statSync as statSync2 } from "fs"; -import { join as join10, relative as relative2, sep as sep2 } from "path"; +import { existsSync as existsSync6, readFileSync as readFileSync5, readdirSync as readdirSync3, statSync as statSync2 } from "fs"; +import { join as join11, relative as relative2, sep as sep2 } from "path"; + +// src/review.ts +import { existsSync as existsSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync5 } from "fs"; +import { join as join10 } from "path"; +var REVIEW_MAX = 40; +var VALID_VERDICTS = ["supported", "partial", "refuted", "unsupported"]; +function loadEvidence(path) { + if (!existsSync5(path)) return []; + try { + const data = JSON.parse(readFileSync4(path, "utf8")); + return Array.isArray(data) ? data.filter( + (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" + ) : []; + } catch { + return []; + } +} +function srdClaims(srd) { + const out = []; + for (const f of srd.functional) { + const ac = f.acceptance.map((a) => `${a.given} / ${a.when} / ${a.then}`).join("; "); + out.push({ id: f.id, kind: "FR", text: `${f.title}: ${f.description}${ac ? " \u2014 " + ac : ""}`, ev: f.rationaleEvidence }); + } + for (const n of srd.nonFunctional) { + out.push({ id: n.id, kind: "NFR", text: `${n.category}: ${n.statement}${n.metric ? ` (${n.metric})` : ""}`, ev: n.rationaleEvidence }); + } + for (const a of srd.architecture.adrs) { + out.push({ id: `ADR-${a.id}`, kind: "ADR", text: `${a.title}: ${a.decision}`, ev: a.evidence }); + } + srd.competitive.competitors.forEach((c, i) => out.push({ id: `COMP-${i + 1}`, kind: "competitor", text: `${c.name}: ${c.note}`, ev: c.evidence })); + srd.competitive.oss.forEach((o, i) => out.push({ id: `OSS-${i + 1}`, kind: "oss", text: `${o.name}: ${o.note}`, ev: o.evidence })); + return out; +} +function claimDigest(snippet, claim, cap = 600) { + if (snippet.length <= cap) return snippet; + const kws = keywords(claim).map((k) => k.toLowerCase()); + if (!kws.length) return snippet.slice(0, cap); + const step = 150; + let best = 0; + let bestCov = -1; + for (let start = 0; start === 0 || start + cap / 2 < snippet.length; start += step) { + const w = snippet.slice(start, start + cap).toLowerCase(); + let cov = 0; + for (const kw of kws) if (w.includes(kw)) cov++; + if (cov >= bestCov) { + bestCov = cov; + best = start; + } + } + return (best > 0 ? "\u2026 " : "") + snippet.slice(best, best + cap).trim(); +} +function runReview(runDir, opts = {}) { + const manifest = srdManifestPath(runDir); + if (!existsSync5(manifest)) throw new Error(`No SRD.json in ${runDir} \u2014 render the SRD first (construct render).`); + let srd; + try { + srd = JSON.parse(readFileSync4(manifest, "utf8")); + } catch (e) { + throw new Error(`SRD.json is unreadable: ${e.message}`); + } + const evidence = loadEvidence(join10(runDir, "evidence", "evidence.json")); + const byId = new Map(evidence.map((e) => [e.id, e])); + const pairs = []; + for (const c of srdClaims(srd)) { + for (const id of [...new Set(c.ev)]) { + const e = byId.get(id); + if (!e) continue; + pairs.push({ + claimId: c.id, + kind: c.kind, + claim: c.text.trim().slice(0, 400), + evidenceId: id, + source: e.source, + digest: claimDigest(e.snippet || e.title || e.ref, c.text), + score: e.score + }); + } + } + const max = Math.max(1, Math.floor(opts.maxReview ?? REVIEW_MAX)); + const kept = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)).slice(0, max) : pairs; + const worklist = { run: runDir, pairs: kept.map(({ score, ...rest }) => rest) }; + const todo = { + run: runDir, + pairs: worklist.pairs.map((p) => ({ ...p, verdict: null, note: "" })) + }; + writeFileSync5(join10(runDir, "VERIFY.todo.json"), JSON.stringify(todo, null, 2)); + writeFileSync5(join10(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, kept.length)); + return worklist; +} +function renderWorklistMd(wl, total, kept) { + const out = []; + out.push(`# Claim-support review worklist`); + out.push(""); + out.push( + `For each pair, open the cited evidence and judge whether it **supports** the claim. In \`VERIFY.todo.json\`, set each \`verdict\` to one of supported \xB7 partial \xB7 refuted \xB7 unsupported, add a short \`note\`, save it (e.g. as \`verdicts.json\`), then run \`construct review --apply verdicts.json --out \`.` + ); + if (kept < total) out.push(` +_Showing ${kept} of ${total} pair(s) \u2014 capped at the highest-score evidence._`); + out.push(""); + for (const p of wl.pairs) { + out.push(`## ${p.claimId} \xB7 ${p.evidenceId} (${p.source})`); + out.push(`**Claim (${p.kind}):** ${p.claim}`); + out.push(`**Cited evidence:** ${p.digest}`); + out.push(`**Verdict:** _____ \xB7 **Note:** _____`); + out.push(""); + } + return out.join("\n"); +} +function applyVerdicts(runDir, verdictsPath) { + if (!existsSync5(verdictsPath)) throw new Error(`verdicts file not found: ${verdictsPath}`); + let raw; + try { + raw = JSON.parse(readFileSync4(verdictsPath, "utf8")); + } catch (e) { + throw new Error(`verdicts file is not valid JSON (${verdictsPath}): ${e.message}`); + } + const list = Array.isArray(raw) ? raw : raw && typeof raw === "object" && Array.isArray(raw.pairs) ? raw.pairs : null; + if (list === null) { + throw new Error(`verdicts file must be a JSON array of verdicts or an object with a "pairs" array (${verdictsPath}).`); + } + const verdicts = []; + const seen = /* @__PURE__ */ new Set(); + const key = (claimId, evidenceId) => `${claimId}::${evidenceId}`; + for (const v of list) { + if (!v || typeof v.claimId !== "string" || typeof v.evidenceId !== "string") continue; + const verdict = VALID_VERDICTS.includes(v.verdict) ? v.verdict : void 0; + verdicts.push({ + claimId: v.claimId, + kind: v.kind, + claim: typeof v.claim === "string" ? v.claim : "", + evidenceId: v.evidenceId, + source: v.source, + digest: typeof v.digest === "string" ? v.digest : "", + verdict, + note: typeof v.note === "string" ? v.note : "" + }); + seen.add(key(v.claimId, v.evidenceId)); + } + const todoPath = join10(runDir, "VERIFY.todo.json"); + if (existsSync5(todoPath)) { + try { + const todo = JSON.parse(readFileSync4(todoPath, "utf8")); + for (const p of todo.pairs ?? []) { + if (!p || typeof p.claimId !== "string" || typeof p.evidenceId !== "string") continue; + if (seen.has(key(p.claimId, p.evidenceId))) continue; + verdicts.push({ + claimId: p.claimId, + kind: p.kind, + claim: p.claim ?? "", + evidenceId: p.evidenceId, + source: p.source, + digest: p.digest ?? "", + verdict: void 0, + note: "" + }); + seen.add(key(p.claimId, p.evidenceId)); + } + } catch { + } + } + const result = reduceVerdicts(verdicts); + writeFileSync5(join10(runDir, "VERIFY.json"), JSON.stringify({ ...result, verdicts }, null, 2)); + return result; +} +function reduceVerdicts(verdicts) { + const counts = { supported: 0, partial: 0, refuted: 0, unsupported: 0 }; + for (const v of verdicts) if (v.verdict && counts[v.verdict] !== void 0) counts[v.verdict]++; + const byClaim = /* @__PURE__ */ new Map(); + for (const v of verdicts) { + const group = byClaim.get(v.claimId) ?? []; + group.push(v); + byClaim.set(v.claimId, group); + } + const failures = []; + const unadjudicated = []; + for (const [claimId, group] of byClaim) { + const adjudicated = group.filter((g) => !!g.verdict); + if (adjudicated.length < group.length) unadjudicated.push(claimId); + const refuted = adjudicated.find((g) => g.verdict === "refuted"); + const hasSupport = adjudicated.some((g) => g.verdict === "supported" || g.verdict === "partial"); + if (refuted) { + failures.push({ claimId, evidenceId: refuted.evidenceId, verdict: "refuted", note: refuted.note }); + } else if (adjudicated.length === group.length && adjudicated.length > 0 && !hasSupport) { + const u = adjudicated.find((g) => g.verdict === "unsupported") ?? adjudicated[0]; + failures.push({ claimId, evidenceId: u.evidenceId, verdict: u.verdict, note: u.note }); + } + } + return { + ok: failures.length === 0, + pairs: verdicts.length, + adjudicated: verdicts.filter((v) => !!v.verdict).length, + supported: counts.supported, + partial: counts.partial, + refuted: counts.refuted, + unsupported: counts.unsupported, + failures, + unadjudicated + }; +} +function formatReviewReport(r) { + const lines = []; + lines.push(`construct review: ${r.adjudicated}/${r.pairs} pair(s) adjudicated`); + lines.push(` supported: ${r.supported} \xB7 partial: ${r.partial} \xB7 refuted: ${r.refuted} \xB7 unsupported: ${r.unsupported}`); + for (const f of r.failures.slice(0, 12)) { + lines.push(` \u2717 ${f.claimId} (${f.evidenceId}): ${f.verdict}${f.note ? " \u2014 " + f.note : ""}`); + } + if (r.unadjudicated.length) { + lines.push(` \u26A0 ${r.unadjudicated.length} claim(s) not fully adjudicated: ${r.unadjudicated.join(", ")}`); + } + lines.push( + !r.ok ? ` \u2717 some claims are refuted or unsupported` : r.unadjudicated.length ? ` \u2713 no refuted or unsupported claims (${r.unadjudicated.length} still unadjudicated \u2014 see above)` : ` \u2713 every grounded claim is backed by its cited evidence` + ); + return lines.join("\n"); +} + +// src/check.ts var DESIGN_REQUIRED_FILES = [ "design/PRINCIPLES.md", "design/DESIGN-TOKENS.md", @@ -3021,7 +3237,7 @@ function mdFiles(runDir) { continue; } for (const name of entries) { - const abs = join10(dir, name); + const abs = join11(dir, name); let st; try { st = statSync2(abs); @@ -3039,13 +3255,13 @@ function mdFiles(runDir) { } return out.sort(); } -function loadEvidence(runDir) { - const path = join10(runDir, "evidence", "evidence.json"); - if (!existsSync5(path)) { +function loadEvidence2(runDir) { + const path = join11(runDir, "evidence", "evidence.json"); + if (!existsSync6(path)) { return { evidence: [], note: `No evidence/evidence.json \u2014 grounding coverage is 0 (run \`construct research\` to ground the SRD).` }; } try { - const data = JSON.parse(readFileSync4(path, "utf8")); + const data = JSON.parse(readFileSync5(path, "utf8")); const evidence = Array.isArray(data) ? data.filter( (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" ) : []; @@ -3093,28 +3309,46 @@ function computeCoverage(srd, evidence) { } var TEMPLATED_THEN_RE = /is persisted and visible to the user$/; var TEMPLATED_METRIC_RE = /^A measurable target for "/; -function applySemantic(runDir, result) { - const p = join10(runDir, "VERIFY.json"); - if (!existsSync5(p)) { - result.structural.warnings.push("--semantic: no VERIFY.json \u2014 run `construct review` then `review --apply ` first; semantic gate skipped."); +function applySemantic(runDir, result, allowUnverified) { + const p = join11(runDir, "VERIFY.json"); + const skip = (reason, hint) => { + if (allowUnverified) { + result.structural.warnings.push(`--semantic: ${reason} \u2014 ${hint}; semantic gate skipped (--allow-unverified).`); + } else { + result.semanticError = `${reason} \u2014 ${hint}, or pass --allow-unverified to degrade this to a warning.`; + result.ok = false; + } + }; + if (!existsSync6(p)) { + skip("no VERIFY.json", "run `construct review` then `review --apply ` first"); return; } + let sem; try { - const sem = JSON.parse(readFileSync4(p, "utf8")); - result.semantic = sem; - if (!sem.ok) result.ok = false; - if (sem.unadjudicated?.length) { - result.structural.warnings.push(`${sem.unadjudicated.length} claim(s) not fully adjudicated by review.`); - } + sem = JSON.parse(readFileSync5(p, "utf8")); } catch (e) { - result.structural.warnings.push(`--semantic: VERIFY.json is unreadable (${e.message}).`); + skip(`VERIFY.json is unreadable (${e.message})`, "re-run `review --apply ` to regenerate it"); + return; + } + if (!Array.isArray(sem.verdicts)) { + skip("VERIFY.json carries no verdicts[] (legacy or hand-edited)", "re-run `review --apply ` to regenerate it"); + return; + } + const reduced = reduceVerdicts(sem.verdicts); + if (reduced.ok !== sem.ok) { + result.structural.warnings.push("VERIFY.json's persisted summary disagreed with its verdicts \u2014 recomputed at check time."); + } + result.semantic = { ...reduced, verdicts: sem.verdicts }; + if (!reduced.ok) result.ok = false; + if (reduced.unadjudicated.length) { + result.structural.warnings.push(`${reduced.unadjudicated.length} claim(s) not fully adjudicated by review.`); } } function checkDesign(runDir, srd, errors, warnings) { const ds = srd.design; if (!ds) return; for (const f of DESIGN_REQUIRED_FILES) { - if (!existsSync5(join10(runDir, f))) errors.push(`Missing required design file: ${f} (re-render at --level complex).`); + if (!existsSync6(join11(runDir, f))) errors.push(`Missing required design file: ${f} (re-render at --level complex).`); } const frIds = new Set(srd.functional.map((f) => f.id)); if (ds.components.length === 0) errors.push("Design system has no components \u2014 a complex SRD's design must name its UI components."); @@ -3136,8 +3370,8 @@ function checkDesign(runDir, srd, errors, warnings) { for (const r of ds.accessibility.requirements) { if (!r.acceptance.length) errors.push(`Accessibility requirement ${r.id} has no acceptance criteria.`); } - const tokenDoc = join10(runDir, "design", "DESIGN-TOKENS.md"); - if (existsSync5(tokenDoc) && readFileSync4(tokenDoc, "utf8").includes(DESIGN_TOKENS_SEEDED_BANNER)) { + const tokenDoc = join11(runDir, "design", "DESIGN-TOKENS.md"); + if (existsSync6(tokenDoc) && readFileSync5(tokenDoc, "utf8").includes(DESIGN_TOKENS_SEEDED_BANNER)) { warnings.push("Design tokens are still seeded defaults \u2014 replace them with the product's real brand values (see references/design-system-authoring.md)."); } } @@ -3145,11 +3379,11 @@ function checkModules(runDir, srd, errors, warnings) { const mods = srd.modules; if (!mods?.length) return; const moduleIds = new Set(mods.map((m) => m.id)); - if (!existsSync5(join10(runDir, "prd", "README.md"))) { + if (!existsSync6(join11(runDir, "prd", "README.md"))) { errors.push(`Missing required module-PRD index: prd/README.md (re-render).`); } for (const m of mods) { - if (!existsSync5(join10(runDir, "prd", m.id, "PRD.md"))) { + if (!existsSync6(join11(runDir, "prd", m.id, "PRD.md"))) { errors.push(`Missing required module PRD: prd/${m.id}/PRD.md (re-render).`); } for (const dep of m.dependsOn) { @@ -3180,22 +3414,22 @@ function checkRun(runDir, opts = {}) { resolved: [] }; for (const f of REQUIRED_FILES) { - if (!existsSync5(join10(runDir, f))) errors.push(`Missing required file: ${f} (run \`construct render --out ${runDir}\`).`); + if (!existsSync6(join11(runDir, f))) errors.push(`Missing required file: ${f} (run \`construct render --out ${runDir}\`).`); } const manifest = srdManifestPath(runDir); - if (!existsSync5(manifest)) { + if (!existsSync6(manifest)) { errors.push(`No SRD.json in ${runDir} \u2014 render the SRD first.`); return { ok: false, structural: { ok: false, errors, warnings }, coverage: emptyCoverage }; } let srd; try { - srd = JSON.parse(readFileSync4(manifest, "utf8")); + srd = JSON.parse(readFileSync5(manifest, "utf8")); } catch (e) { errors.push(`SRD.json is unreadable: ${e.message}`); return { ok: false, structural: { ok: false, errors, warnings }, coverage: emptyCoverage }; } for (const rel of mdFiles(runDir)) { - const text = readFileSync4(join10(runDir, rel), "utf8"); + const text = readFileSync5(join11(runDir, rel), "utf8"); if (DECISION_RE.test(text)) errors.push(`Unresolved decision (\u{1F9E0}) in ${rel} \u2014 resolve it before the SRD is complete.`); else if (PLACEHOLDER_RE.test(text)) warnings.push(`Possible leftover placeholder (TODO/TBD/FIXME) in ${rel} \u2014 confirm it is intentional.`); } @@ -3247,7 +3481,7 @@ function checkRun(runDir, opts = {}) { if (templatedMetrics) { warnings.push(`${templatedMetrics} NFR metric(s) are still generic placeholders \u2014 set measurable targets (see references/acceptance-criteria.md).`); } - const { evidence, note } = loadEvidence(runDir); + const { evidence, note } = loadEvidence2(runDir); if (note) warnings.push(note); const coverage = computeCoverage(srd, evidence); if (coverage.dangling.length) { @@ -3263,7 +3497,7 @@ function checkRun(runDir, opts = {}) { } const ok = structuralOk && (grounding?.ok ?? true); const result = { ok, structural: { ok: structuralOk, errors, warnings }, coverage, grounding }; - if (opts.semantic) applySemantic(runDir, result); + if (opts.semantic) applySemantic(runDir, result, opts.allowUnverified ?? false); return result; } function pct(part, total) { @@ -3294,6 +3528,11 @@ function formatCheckReport(r, runDir) { g.ok ? ` \u2713 PASS \u2014 ${g.actualPct}% of groundable claims are grounded (threshold ${g.threshold}%)` : ` \u2717 FAIL \u2014 ${g.actualPct}% of groundable claims are grounded, below the ${g.threshold}% threshold` ); } + if (r.semanticError) { + lines.push(``); + lines.push(`Semantic claim-support gate (--semantic):`); + lines.push(` \u2717 FAIL \u2014 ${r.semanticError}`); + } if (r.semantic) { const s = r.semantic; lines.push(``); @@ -3308,13 +3547,13 @@ function formatCheckReport(r, runDir) { } // src/analyze.ts -import { existsSync as existsSync6, readFileSync as readFileSync5 } from "fs"; -import { join as join11 } from "path"; -function loadEvidence2(runDir) { - const path = join11(runDir, "evidence", "evidence.json"); - if (!existsSync6(path)) return []; +import { existsSync as existsSync7, readFileSync as readFileSync6 } from "fs"; +import { join as join12 } from "path"; +function loadEvidence3(runDir) { + const path = join12(runDir, "evidence", "evidence.json"); + if (!existsSync7(path)) return []; try { - const data = JSON.parse(readFileSync5(path, "utf8")); + const data = JSON.parse(readFileSync6(path, "utf8")); return Array.isArray(data) ? data.filter( (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" ) : []; @@ -3323,10 +3562,10 @@ function loadEvidence2(runDir) { } } function loadMetaNotes(runDir) { - const path = join11(runDir, "evidence", "meta.json"); - if (!existsSync6(path)) return []; + const path = join12(runDir, "evidence", "meta.json"); + if (!existsSync7(path)) return []; try { - const meta = JSON.parse(readFileSync5(path, "utf8")); + const meta = JSON.parse(readFileSync6(path, "utf8")); return Array.isArray(meta.notes) ? meta.notes.filter((n) => typeof n === "string") : []; } catch { return []; @@ -3337,7 +3576,7 @@ function featureText(f) { } function analyzeRun(runDir) { const brief = loadBrief(runDir); - const evidence = loadEvidence2(runDir); + const evidence = loadEvidence3(runDir); const notes = loadMetaNotes(runDir); const drill = (cmd, q) => `construct ${cmd} --out ${runDir} --q "${q.replace(/"/g, "'")}"`; const bySource = {}; @@ -3397,8 +3636,8 @@ function formatGapReport(r, runDir) { } // src/verify.ts -import { existsSync as existsSync7, readFileSync as readFileSync6 } from "fs"; -import { isAbsolute, join as join12, resolve as resolve2 } from "path"; +import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs"; +import { isAbsolute, join as join13, resolve as resolve2 } from "path"; var TEST_FILE_RE = /\.(test|spec)\.[^./]+$|_(test|spec)\.[^./]+$|(^|\/)test_[^/]+\.[^./]+$/i; var TEST_SUFFIX_RE = /(^|\/)[^/]*[A-Z]\w*Tests?\.(java|kt|kts|cs|scala|groovy)$/; var TEST_DIR_RE = /(^|\/)(tests?|__tests__|spec|specs|e2e)\//i; @@ -3436,7 +3675,7 @@ function verifyRun(runDir, opts = {}) { const warnings = []; const frTestCoverage = []; const planPath = buildPlanPath(runDir); - if (!existsSync7(planPath)) { + if (!existsSync8(planPath)) { errors.push(`No BUILD-PLAN.json in ${runDir} \u2014 render the SRD first (construct render).`); return { ok: false, errors, warnings, frTestCoverage }; } @@ -3450,13 +3689,13 @@ function verifyRun(runDir, opts = {}) { return { ok: false, errors, warnings, frTestCoverage }; } const manifest = srdManifestPath(runDir); - if (!existsSync7(manifest)) { + if (!existsSync8(manifest)) { errors.push(`No SRD.json in ${runDir} \u2014 the plan cannot be verified against a missing SRD.`); return { ok: false, errors, warnings, frTestCoverage }; } let srd; try { - srd = JSON.parse(readFileSync6(manifest, "utf8")); + srd = JSON.parse(readFileSync7(manifest, "utf8")); } catch (e) { errors.push(`SRD.json is unreadable: ${e.message}`); return { ok: false, errors, warnings, frTestCoverage }; @@ -3500,13 +3739,13 @@ function verifyRun(runDir, opts = {}) { const ok2 = errors.length === 0; return { ok: ok2, errors, warnings, frTestCoverage }; } - if (!existsSync7(appDir)) { + if (!existsSync8(appDir)) { errors.push(`App directory does not exist: ${appDir}.`); return { ok: false, errors, warnings, frTestCoverage }; } for (const t of doneTasks) { for (const rel of [...t.artifacts, ...t.tests]) { - if (!existsSync7(join12(appDir, rel))) errors.push(`${t.id} is done but its declared file is missing: ${rel}.`); + if (!existsSync8(join13(appDir, rel))) errors.push(`${t.id} is done but its declared file is missing: ${rel}.`); } if (t.frIds.length && t.tests.length === 0) { warnings.push(`${t.id} is done but declares no tests \u2014 record the test files that exercise ${t.frIds.join(", ")}.`); @@ -3593,220 +3832,6 @@ function formatVerifyReport(r, runDir) { return lines.join("\n"); } -// src/review.ts -import { existsSync as existsSync8, readFileSync as readFileSync7, writeFileSync as writeFileSync5 } from "fs"; -import { join as join13 } from "path"; -var REVIEW_MAX = 40; -var VALID_VERDICTS = ["supported", "partial", "refuted", "unsupported"]; -function loadEvidence3(path) { - if (!existsSync8(path)) return []; - try { - const data = JSON.parse(readFileSync7(path, "utf8")); - return Array.isArray(data) ? data.filter( - (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" - ) : []; - } catch { - return []; - } -} -function srdClaims(srd) { - const out = []; - for (const f of srd.functional) { - const ac = f.acceptance.map((a) => `${a.given} / ${a.when} / ${a.then}`).join("; "); - out.push({ id: f.id, kind: "FR", text: `${f.title}: ${f.description}${ac ? " \u2014 " + ac : ""}`, ev: f.rationaleEvidence }); - } - for (const n of srd.nonFunctional) { - out.push({ id: n.id, kind: "NFR", text: `${n.category}: ${n.statement}${n.metric ? ` (${n.metric})` : ""}`, ev: n.rationaleEvidence }); - } - for (const a of srd.architecture.adrs) { - out.push({ id: `ADR-${a.id}`, kind: "ADR", text: `${a.title}: ${a.decision}`, ev: a.evidence }); - } - srd.competitive.competitors.forEach((c, i) => out.push({ id: `COMP-${i + 1}`, kind: "competitor", text: `${c.name}: ${c.note}`, ev: c.evidence })); - srd.competitive.oss.forEach((o, i) => out.push({ id: `OSS-${i + 1}`, kind: "oss", text: `${o.name}: ${o.note}`, ev: o.evidence })); - return out; -} -function claimDigest(snippet, claim, cap = 600) { - if (snippet.length <= cap) return snippet; - const kws = keywords(claim).map((k) => k.toLowerCase()); - if (!kws.length) return snippet.slice(0, cap); - const step = 150; - let best = 0; - let bestCov = -1; - for (let start = 0; start === 0 || start + cap / 2 < snippet.length; start += step) { - const w = snippet.slice(start, start + cap).toLowerCase(); - let cov = 0; - for (const kw of kws) if (w.includes(kw)) cov++; - if (cov >= bestCov) { - bestCov = cov; - best = start; - } - } - return (best > 0 ? "\u2026 " : "") + snippet.slice(best, best + cap).trim(); -} -function runReview(runDir, opts = {}) { - const manifest = srdManifestPath(runDir); - if (!existsSync8(manifest)) throw new Error(`No SRD.json in ${runDir} \u2014 render the SRD first (construct render).`); - let srd; - try { - srd = JSON.parse(readFileSync7(manifest, "utf8")); - } catch (e) { - throw new Error(`SRD.json is unreadable: ${e.message}`); - } - const evidence = loadEvidence3(join13(runDir, "evidence", "evidence.json")); - const byId = new Map(evidence.map((e) => [e.id, e])); - const pairs = []; - for (const c of srdClaims(srd)) { - for (const id of [...new Set(c.ev)]) { - const e = byId.get(id); - if (!e) continue; - pairs.push({ - claimId: c.id, - kind: c.kind, - claim: c.text.trim().slice(0, 400), - evidenceId: id, - source: e.source, - digest: claimDigest(e.snippet || e.title || e.ref, c.text), - score: e.score - }); - } - } - const max = Math.max(1, Math.floor(opts.maxReview ?? REVIEW_MAX)); - const kept = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)).slice(0, max) : pairs; - const worklist = { run: runDir, pairs: kept.map(({ score, ...rest }) => rest) }; - const todo = { - run: runDir, - pairs: worklist.pairs.map((p) => ({ ...p, verdict: null, note: "" })) - }; - writeFileSync5(join13(runDir, "VERIFY.todo.json"), JSON.stringify(todo, null, 2)); - writeFileSync5(join13(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, kept.length)); - return worklist; -} -function renderWorklistMd(wl, total, kept) { - const out = []; - out.push(`# Claim-support review worklist`); - out.push(""); - out.push( - `For each pair, open the cited evidence and judge whether it **supports** the claim. In \`VERIFY.todo.json\`, set each \`verdict\` to one of supported \xB7 partial \xB7 refuted \xB7 unsupported, add a short \`note\`, save it (e.g. as \`verdicts.json\`), then run \`construct review --apply verdicts.json --out \`.` - ); - if (kept < total) out.push(` -_Showing ${kept} of ${total} pair(s) \u2014 capped at the highest-score evidence._`); - out.push(""); - for (const p of wl.pairs) { - out.push(`## ${p.claimId} \xB7 ${p.evidenceId} (${p.source})`); - out.push(`**Claim (${p.kind}):** ${p.claim}`); - out.push(`**Cited evidence:** ${p.digest}`); - out.push(`**Verdict:** _____ \xB7 **Note:** _____`); - out.push(""); - } - return out.join("\n"); -} -function applyVerdicts(runDir, verdictsPath) { - if (!existsSync8(verdictsPath)) throw new Error(`verdicts file not found: ${verdictsPath}`); - let raw; - try { - raw = JSON.parse(readFileSync7(verdictsPath, "utf8")); - } catch (e) { - throw new Error(`verdicts file is not valid JSON (${verdictsPath}): ${e.message}`); - } - const list = Array.isArray(raw) ? raw : raw && typeof raw === "object" && Array.isArray(raw.pairs) ? raw.pairs : null; - if (list === null) { - throw new Error(`verdicts file must be a JSON array of verdicts or an object with a "pairs" array (${verdictsPath}).`); - } - const verdicts = []; - const seen = /* @__PURE__ */ new Set(); - const key = (claimId, evidenceId) => `${claimId}::${evidenceId}`; - for (const v of list) { - if (!v || typeof v.claimId !== "string" || typeof v.evidenceId !== "string") continue; - const verdict = VALID_VERDICTS.includes(v.verdict) ? v.verdict : void 0; - verdicts.push({ - claimId: v.claimId, - kind: v.kind, - claim: typeof v.claim === "string" ? v.claim : "", - evidenceId: v.evidenceId, - source: v.source, - digest: typeof v.digest === "string" ? v.digest : "", - verdict, - note: typeof v.note === "string" ? v.note : "" - }); - seen.add(key(v.claimId, v.evidenceId)); - } - const todoPath = join13(runDir, "VERIFY.todo.json"); - if (existsSync8(todoPath)) { - try { - const todo = JSON.parse(readFileSync7(todoPath, "utf8")); - for (const p of todo.pairs ?? []) { - if (!p || typeof p.claimId !== "string" || typeof p.evidenceId !== "string") continue; - if (seen.has(key(p.claimId, p.evidenceId))) continue; - verdicts.push({ - claimId: p.claimId, - kind: p.kind, - claim: p.claim ?? "", - evidenceId: p.evidenceId, - source: p.source, - digest: p.digest ?? "", - verdict: void 0, - note: "" - }); - seen.add(key(p.claimId, p.evidenceId)); - } - } catch { - } - } - const result = reduceVerdicts(verdicts); - writeFileSync5(join13(runDir, "VERIFY.json"), JSON.stringify({ ...result, verdicts }, null, 2)); - return result; -} -function reduceVerdicts(verdicts) { - const counts = { supported: 0, partial: 0, refuted: 0, unsupported: 0 }; - for (const v of verdicts) if (v.verdict && counts[v.verdict] !== void 0) counts[v.verdict]++; - const byClaim = /* @__PURE__ */ new Map(); - for (const v of verdicts) { - const group = byClaim.get(v.claimId) ?? []; - group.push(v); - byClaim.set(v.claimId, group); - } - const failures = []; - const unadjudicated = []; - for (const [claimId, group] of byClaim) { - const adjudicated = group.filter((g) => !!g.verdict); - if (adjudicated.length < group.length) unadjudicated.push(claimId); - const refuted = adjudicated.find((g) => g.verdict === "refuted"); - const hasSupport = adjudicated.some((g) => g.verdict === "supported" || g.verdict === "partial"); - if (refuted) { - failures.push({ claimId, evidenceId: refuted.evidenceId, verdict: "refuted", note: refuted.note }); - } else if (adjudicated.length === group.length && adjudicated.length > 0 && !hasSupport) { - const u = adjudicated.find((g) => g.verdict === "unsupported") ?? adjudicated[0]; - failures.push({ claimId, evidenceId: u.evidenceId, verdict: u.verdict, note: u.note }); - } - } - return { - ok: failures.length === 0, - pairs: verdicts.length, - adjudicated: verdicts.filter((v) => !!v.verdict).length, - supported: counts.supported, - partial: counts.partial, - refuted: counts.refuted, - unsupported: counts.unsupported, - failures, - unadjudicated - }; -} -function formatReviewReport(r) { - const lines = []; - lines.push(`construct review: ${r.adjudicated}/${r.pairs} pair(s) adjudicated`); - lines.push(` supported: ${r.supported} \xB7 partial: ${r.partial} \xB7 refuted: ${r.refuted} \xB7 unsupported: ${r.unsupported}`); - for (const f of r.failures.slice(0, 12)) { - lines.push(` \u2717 ${f.claimId} (${f.evidenceId}): ${f.verdict}${f.note ? " \u2014 " + f.note : ""}`); - } - if (r.unadjudicated.length) { - lines.push(` \u26A0 ${r.unadjudicated.length} claim(s) not fully adjudicated: ${r.unadjudicated.join(", ")}`); - } - lines.push( - !r.ok ? ` \u2717 some claims are refuted or unsupported` : r.unadjudicated.length ? ` \u2713 no refuted or unsupported claims (${r.unadjudicated.length} still unadjudicated \u2014 see above)` : ` \u2713 every grounded claim is backed by its cited evidence` - ); - return lines.join("\n"); -} - // src/cli.ts var HELP = `construct v${VERSION} Turn a product idea into a grounded, buildable SRD suite. Interview \u2192 research @@ -3819,7 +3844,7 @@ Usage: construct analyze --out [--json] construct web|oss|tech|so --out [--q ""] [--url ] [--seeds ] construct render --out [--level light|complex] [--merge] [--no-design] [--prd] - construct check --out [--min-grounding <0-100>] [--semantic] [--json] + construct check --out [--min-grounding <0-100>] [--semantic [--allow-unverified]] [--json] construct review --out [--apply ] [--max-review N] [--json] construct verify --out [--app ] [--run-tests] [--strict] [--json] construct status --out [--json] @@ -3859,6 +3884,9 @@ Options: --level light | complex (default: light) --min-grounding For 'check': fail unless \u2265 n% of claims are grounded (opt-in) --semantic For 'check': fold in the 'review' claim-support verdicts + (fail-closed: no/unreadable VERIFY.json fails the check) + --allow-unverified For 'check --semantic': degrade a missing/unreadable + VERIFY.json to a warning instead of failing --apply For 'review': consume an adjudicated verdicts file + gate --app For 'verify': the built app directory (default: conventions.appDir) --run-tests For 'verify': also execute testCommand + per-task verify commands @@ -3900,7 +3928,7 @@ var VALUE_FLAGS = /* @__PURE__ */ new Set([ "apply", "max-review" ]); -var BOOL_FLAGS = /* @__PURE__ */ new Set(["semantic", "merge", "json", "refresh", "run-tests", "strict", "no-design", "prd"]); +var BOOL_FLAGS = /* @__PURE__ */ new Set(["semantic", "merge", "json", "refresh", "run-tests", "strict", "no-design", "prd", "allow-unverified"]); function fail(message) { process.stderr.write(`construct: ${message} `); @@ -4149,7 +4177,7 @@ ${v.errors.map((e) => " - " + e).join("\n")}`); fail("invalid --min-grounding (expected a number between 0 and 100)"); } } - const res = checkRun(out, { minGrounding, semantic: p.bools.has("semantic") }); + const res = checkRun(out, { minGrounding, semantic: p.bools.has("semantic"), allowUnverified: p.bools.has("allow-unverified") }); if (p.bools.has("json")) { process.stdout.write(JSON.stringify(res, null, 2) + "\n"); } else { diff --git a/skills/construct/scripts/construct.mjs b/skills/construct/scripts/construct.mjs index 542e52b..08da3f9 100755 --- a/skills/construct/scripts/construct.mjs +++ b/skills/construct/scripts/construct.mjs @@ -2989,8 +2989,224 @@ function renderSRD(brief, evidence, opts) { } // src/check.ts -import { existsSync as existsSync5, readFileSync as readFileSync4, readdirSync as readdirSync3, statSync as statSync2 } from "fs"; -import { join as join10, relative as relative2, sep as sep2 } from "path"; +import { existsSync as existsSync6, readFileSync as readFileSync5, readdirSync as readdirSync3, statSync as statSync2 } from "fs"; +import { join as join11, relative as relative2, sep as sep2 } from "path"; + +// src/review.ts +import { existsSync as existsSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync5 } from "fs"; +import { join as join10 } from "path"; +var REVIEW_MAX = 40; +var VALID_VERDICTS = ["supported", "partial", "refuted", "unsupported"]; +function loadEvidence(path) { + if (!existsSync5(path)) return []; + try { + const data = JSON.parse(readFileSync4(path, "utf8")); + return Array.isArray(data) ? data.filter( + (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" + ) : []; + } catch { + return []; + } +} +function srdClaims(srd) { + const out = []; + for (const f of srd.functional) { + const ac = f.acceptance.map((a) => `${a.given} / ${a.when} / ${a.then}`).join("; "); + out.push({ id: f.id, kind: "FR", text: `${f.title}: ${f.description}${ac ? " \u2014 " + ac : ""}`, ev: f.rationaleEvidence }); + } + for (const n of srd.nonFunctional) { + out.push({ id: n.id, kind: "NFR", text: `${n.category}: ${n.statement}${n.metric ? ` (${n.metric})` : ""}`, ev: n.rationaleEvidence }); + } + for (const a of srd.architecture.adrs) { + out.push({ id: `ADR-${a.id}`, kind: "ADR", text: `${a.title}: ${a.decision}`, ev: a.evidence }); + } + srd.competitive.competitors.forEach((c, i) => out.push({ id: `COMP-${i + 1}`, kind: "competitor", text: `${c.name}: ${c.note}`, ev: c.evidence })); + srd.competitive.oss.forEach((o, i) => out.push({ id: `OSS-${i + 1}`, kind: "oss", text: `${o.name}: ${o.note}`, ev: o.evidence })); + return out; +} +function claimDigest(snippet, claim, cap = 600) { + if (snippet.length <= cap) return snippet; + const kws = keywords(claim).map((k) => k.toLowerCase()); + if (!kws.length) return snippet.slice(0, cap); + const step = 150; + let best = 0; + let bestCov = -1; + for (let start = 0; start === 0 || start + cap / 2 < snippet.length; start += step) { + const w = snippet.slice(start, start + cap).toLowerCase(); + let cov = 0; + for (const kw of kws) if (w.includes(kw)) cov++; + if (cov >= bestCov) { + bestCov = cov; + best = start; + } + } + return (best > 0 ? "\u2026 " : "") + snippet.slice(best, best + cap).trim(); +} +function runReview(runDir, opts = {}) { + const manifest = srdManifestPath(runDir); + if (!existsSync5(manifest)) throw new Error(`No SRD.json in ${runDir} \u2014 render the SRD first (construct render).`); + let srd; + try { + srd = JSON.parse(readFileSync4(manifest, "utf8")); + } catch (e) { + throw new Error(`SRD.json is unreadable: ${e.message}`); + } + const evidence = loadEvidence(join10(runDir, "evidence", "evidence.json")); + const byId = new Map(evidence.map((e) => [e.id, e])); + const pairs = []; + for (const c of srdClaims(srd)) { + for (const id of [...new Set(c.ev)]) { + const e = byId.get(id); + if (!e) continue; + pairs.push({ + claimId: c.id, + kind: c.kind, + claim: c.text.trim().slice(0, 400), + evidenceId: id, + source: e.source, + digest: claimDigest(e.snippet || e.title || e.ref, c.text), + score: e.score + }); + } + } + const max = Math.max(1, Math.floor(opts.maxReview ?? REVIEW_MAX)); + const kept = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)).slice(0, max) : pairs; + const worklist = { run: runDir, pairs: kept.map(({ score, ...rest }) => rest) }; + const todo = { + run: runDir, + pairs: worklist.pairs.map((p) => ({ ...p, verdict: null, note: "" })) + }; + writeFileSync5(join10(runDir, "VERIFY.todo.json"), JSON.stringify(todo, null, 2)); + writeFileSync5(join10(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, kept.length)); + return worklist; +} +function renderWorklistMd(wl, total, kept) { + const out = []; + out.push(`# Claim-support review worklist`); + out.push(""); + out.push( + `For each pair, open the cited evidence and judge whether it **supports** the claim. In \`VERIFY.todo.json\`, set each \`verdict\` to one of supported \xB7 partial \xB7 refuted \xB7 unsupported, add a short \`note\`, save it (e.g. as \`verdicts.json\`), then run \`construct review --apply verdicts.json --out \`.` + ); + if (kept < total) out.push(` +_Showing ${kept} of ${total} pair(s) \u2014 capped at the highest-score evidence._`); + out.push(""); + for (const p of wl.pairs) { + out.push(`## ${p.claimId} \xB7 ${p.evidenceId} (${p.source})`); + out.push(`**Claim (${p.kind}):** ${p.claim}`); + out.push(`**Cited evidence:** ${p.digest}`); + out.push(`**Verdict:** _____ \xB7 **Note:** _____`); + out.push(""); + } + return out.join("\n"); +} +function applyVerdicts(runDir, verdictsPath) { + if (!existsSync5(verdictsPath)) throw new Error(`verdicts file not found: ${verdictsPath}`); + let raw; + try { + raw = JSON.parse(readFileSync4(verdictsPath, "utf8")); + } catch (e) { + throw new Error(`verdicts file is not valid JSON (${verdictsPath}): ${e.message}`); + } + const list = Array.isArray(raw) ? raw : raw && typeof raw === "object" && Array.isArray(raw.pairs) ? raw.pairs : null; + if (list === null) { + throw new Error(`verdicts file must be a JSON array of verdicts or an object with a "pairs" array (${verdictsPath}).`); + } + const verdicts = []; + const seen = /* @__PURE__ */ new Set(); + const key = (claimId, evidenceId) => `${claimId}::${evidenceId}`; + for (const v of list) { + if (!v || typeof v.claimId !== "string" || typeof v.evidenceId !== "string") continue; + const verdict = VALID_VERDICTS.includes(v.verdict) ? v.verdict : void 0; + verdicts.push({ + claimId: v.claimId, + kind: v.kind, + claim: typeof v.claim === "string" ? v.claim : "", + evidenceId: v.evidenceId, + source: v.source, + digest: typeof v.digest === "string" ? v.digest : "", + verdict, + note: typeof v.note === "string" ? v.note : "" + }); + seen.add(key(v.claimId, v.evidenceId)); + } + const todoPath = join10(runDir, "VERIFY.todo.json"); + if (existsSync5(todoPath)) { + try { + const todo = JSON.parse(readFileSync4(todoPath, "utf8")); + for (const p of todo.pairs ?? []) { + if (!p || typeof p.claimId !== "string" || typeof p.evidenceId !== "string") continue; + if (seen.has(key(p.claimId, p.evidenceId))) continue; + verdicts.push({ + claimId: p.claimId, + kind: p.kind, + claim: p.claim ?? "", + evidenceId: p.evidenceId, + source: p.source, + digest: p.digest ?? "", + verdict: void 0, + note: "" + }); + seen.add(key(p.claimId, p.evidenceId)); + } + } catch { + } + } + const result = reduceVerdicts(verdicts); + writeFileSync5(join10(runDir, "VERIFY.json"), JSON.stringify({ ...result, verdicts }, null, 2)); + return result; +} +function reduceVerdicts(verdicts) { + const counts = { supported: 0, partial: 0, refuted: 0, unsupported: 0 }; + for (const v of verdicts) if (v.verdict && counts[v.verdict] !== void 0) counts[v.verdict]++; + const byClaim = /* @__PURE__ */ new Map(); + for (const v of verdicts) { + const group = byClaim.get(v.claimId) ?? []; + group.push(v); + byClaim.set(v.claimId, group); + } + const failures = []; + const unadjudicated = []; + for (const [claimId, group] of byClaim) { + const adjudicated = group.filter((g) => !!g.verdict); + if (adjudicated.length < group.length) unadjudicated.push(claimId); + const refuted = adjudicated.find((g) => g.verdict === "refuted"); + const hasSupport = adjudicated.some((g) => g.verdict === "supported" || g.verdict === "partial"); + if (refuted) { + failures.push({ claimId, evidenceId: refuted.evidenceId, verdict: "refuted", note: refuted.note }); + } else if (adjudicated.length === group.length && adjudicated.length > 0 && !hasSupport) { + const u = adjudicated.find((g) => g.verdict === "unsupported") ?? adjudicated[0]; + failures.push({ claimId, evidenceId: u.evidenceId, verdict: u.verdict, note: u.note }); + } + } + return { + ok: failures.length === 0, + pairs: verdicts.length, + adjudicated: verdicts.filter((v) => !!v.verdict).length, + supported: counts.supported, + partial: counts.partial, + refuted: counts.refuted, + unsupported: counts.unsupported, + failures, + unadjudicated + }; +} +function formatReviewReport(r) { + const lines = []; + lines.push(`construct review: ${r.adjudicated}/${r.pairs} pair(s) adjudicated`); + lines.push(` supported: ${r.supported} \xB7 partial: ${r.partial} \xB7 refuted: ${r.refuted} \xB7 unsupported: ${r.unsupported}`); + for (const f of r.failures.slice(0, 12)) { + lines.push(` \u2717 ${f.claimId} (${f.evidenceId}): ${f.verdict}${f.note ? " \u2014 " + f.note : ""}`); + } + if (r.unadjudicated.length) { + lines.push(` \u26A0 ${r.unadjudicated.length} claim(s) not fully adjudicated: ${r.unadjudicated.join(", ")}`); + } + lines.push( + !r.ok ? ` \u2717 some claims are refuted or unsupported` : r.unadjudicated.length ? ` \u2713 no refuted or unsupported claims (${r.unadjudicated.length} still unadjudicated \u2014 see above)` : ` \u2713 every grounded claim is backed by its cited evidence` + ); + return lines.join("\n"); +} + +// src/check.ts var DESIGN_REQUIRED_FILES = [ "design/PRINCIPLES.md", "design/DESIGN-TOKENS.md", @@ -3021,7 +3237,7 @@ function mdFiles(runDir) { continue; } for (const name of entries) { - const abs = join10(dir, name); + const abs = join11(dir, name); let st; try { st = statSync2(abs); @@ -3039,13 +3255,13 @@ function mdFiles(runDir) { } return out.sort(); } -function loadEvidence(runDir) { - const path = join10(runDir, "evidence", "evidence.json"); - if (!existsSync5(path)) { +function loadEvidence2(runDir) { + const path = join11(runDir, "evidence", "evidence.json"); + if (!existsSync6(path)) { return { evidence: [], note: `No evidence/evidence.json \u2014 grounding coverage is 0 (run \`construct research\` to ground the SRD).` }; } try { - const data = JSON.parse(readFileSync4(path, "utf8")); + const data = JSON.parse(readFileSync5(path, "utf8")); const evidence = Array.isArray(data) ? data.filter( (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" ) : []; @@ -3093,28 +3309,46 @@ function computeCoverage(srd, evidence) { } var TEMPLATED_THEN_RE = /is persisted and visible to the user$/; var TEMPLATED_METRIC_RE = /^A measurable target for "/; -function applySemantic(runDir, result) { - const p = join10(runDir, "VERIFY.json"); - if (!existsSync5(p)) { - result.structural.warnings.push("--semantic: no VERIFY.json \u2014 run `construct review` then `review --apply ` first; semantic gate skipped."); +function applySemantic(runDir, result, allowUnverified) { + const p = join11(runDir, "VERIFY.json"); + const skip = (reason, hint) => { + if (allowUnverified) { + result.structural.warnings.push(`--semantic: ${reason} \u2014 ${hint}; semantic gate skipped (--allow-unverified).`); + } else { + result.semanticError = `${reason} \u2014 ${hint}, or pass --allow-unverified to degrade this to a warning.`; + result.ok = false; + } + }; + if (!existsSync6(p)) { + skip("no VERIFY.json", "run `construct review` then `review --apply ` first"); return; } + let sem; try { - const sem = JSON.parse(readFileSync4(p, "utf8")); - result.semantic = sem; - if (!sem.ok) result.ok = false; - if (sem.unadjudicated?.length) { - result.structural.warnings.push(`${sem.unadjudicated.length} claim(s) not fully adjudicated by review.`); - } + sem = JSON.parse(readFileSync5(p, "utf8")); } catch (e) { - result.structural.warnings.push(`--semantic: VERIFY.json is unreadable (${e.message}).`); + skip(`VERIFY.json is unreadable (${e.message})`, "re-run `review --apply ` to regenerate it"); + return; + } + if (!Array.isArray(sem.verdicts)) { + skip("VERIFY.json carries no verdicts[] (legacy or hand-edited)", "re-run `review --apply ` to regenerate it"); + return; + } + const reduced = reduceVerdicts(sem.verdicts); + if (reduced.ok !== sem.ok) { + result.structural.warnings.push("VERIFY.json's persisted summary disagreed with its verdicts \u2014 recomputed at check time."); + } + result.semantic = { ...reduced, verdicts: sem.verdicts }; + if (!reduced.ok) result.ok = false; + if (reduced.unadjudicated.length) { + result.structural.warnings.push(`${reduced.unadjudicated.length} claim(s) not fully adjudicated by review.`); } } function checkDesign(runDir, srd, errors, warnings) { const ds = srd.design; if (!ds) return; for (const f of DESIGN_REQUIRED_FILES) { - if (!existsSync5(join10(runDir, f))) errors.push(`Missing required design file: ${f} (re-render at --level complex).`); + if (!existsSync6(join11(runDir, f))) errors.push(`Missing required design file: ${f} (re-render at --level complex).`); } const frIds = new Set(srd.functional.map((f) => f.id)); if (ds.components.length === 0) errors.push("Design system has no components \u2014 a complex SRD's design must name its UI components."); @@ -3136,8 +3370,8 @@ function checkDesign(runDir, srd, errors, warnings) { for (const r of ds.accessibility.requirements) { if (!r.acceptance.length) errors.push(`Accessibility requirement ${r.id} has no acceptance criteria.`); } - const tokenDoc = join10(runDir, "design", "DESIGN-TOKENS.md"); - if (existsSync5(tokenDoc) && readFileSync4(tokenDoc, "utf8").includes(DESIGN_TOKENS_SEEDED_BANNER)) { + const tokenDoc = join11(runDir, "design", "DESIGN-TOKENS.md"); + if (existsSync6(tokenDoc) && readFileSync5(tokenDoc, "utf8").includes(DESIGN_TOKENS_SEEDED_BANNER)) { warnings.push("Design tokens are still seeded defaults \u2014 replace them with the product's real brand values (see references/design-system-authoring.md)."); } } @@ -3145,11 +3379,11 @@ function checkModules(runDir, srd, errors, warnings) { const mods = srd.modules; if (!mods?.length) return; const moduleIds = new Set(mods.map((m) => m.id)); - if (!existsSync5(join10(runDir, "prd", "README.md"))) { + if (!existsSync6(join11(runDir, "prd", "README.md"))) { errors.push(`Missing required module-PRD index: prd/README.md (re-render).`); } for (const m of mods) { - if (!existsSync5(join10(runDir, "prd", m.id, "PRD.md"))) { + if (!existsSync6(join11(runDir, "prd", m.id, "PRD.md"))) { errors.push(`Missing required module PRD: prd/${m.id}/PRD.md (re-render).`); } for (const dep of m.dependsOn) { @@ -3180,22 +3414,22 @@ function checkRun(runDir, opts = {}) { resolved: [] }; for (const f of REQUIRED_FILES) { - if (!existsSync5(join10(runDir, f))) errors.push(`Missing required file: ${f} (run \`construct render --out ${runDir}\`).`); + if (!existsSync6(join11(runDir, f))) errors.push(`Missing required file: ${f} (run \`construct render --out ${runDir}\`).`); } const manifest = srdManifestPath(runDir); - if (!existsSync5(manifest)) { + if (!existsSync6(manifest)) { errors.push(`No SRD.json in ${runDir} \u2014 render the SRD first.`); return { ok: false, structural: { ok: false, errors, warnings }, coverage: emptyCoverage }; } let srd; try { - srd = JSON.parse(readFileSync4(manifest, "utf8")); + srd = JSON.parse(readFileSync5(manifest, "utf8")); } catch (e) { errors.push(`SRD.json is unreadable: ${e.message}`); return { ok: false, structural: { ok: false, errors, warnings }, coverage: emptyCoverage }; } for (const rel of mdFiles(runDir)) { - const text = readFileSync4(join10(runDir, rel), "utf8"); + const text = readFileSync5(join11(runDir, rel), "utf8"); if (DECISION_RE.test(text)) errors.push(`Unresolved decision (\u{1F9E0}) in ${rel} \u2014 resolve it before the SRD is complete.`); else if (PLACEHOLDER_RE.test(text)) warnings.push(`Possible leftover placeholder (TODO/TBD/FIXME) in ${rel} \u2014 confirm it is intentional.`); } @@ -3247,7 +3481,7 @@ function checkRun(runDir, opts = {}) { if (templatedMetrics) { warnings.push(`${templatedMetrics} NFR metric(s) are still generic placeholders \u2014 set measurable targets (see references/acceptance-criteria.md).`); } - const { evidence, note } = loadEvidence(runDir); + const { evidence, note } = loadEvidence2(runDir); if (note) warnings.push(note); const coverage = computeCoverage(srd, evidence); if (coverage.dangling.length) { @@ -3263,7 +3497,7 @@ function checkRun(runDir, opts = {}) { } const ok = structuralOk && (grounding?.ok ?? true); const result = { ok, structural: { ok: structuralOk, errors, warnings }, coverage, grounding }; - if (opts.semantic) applySemantic(runDir, result); + if (opts.semantic) applySemantic(runDir, result, opts.allowUnverified ?? false); return result; } function pct(part, total) { @@ -3294,6 +3528,11 @@ function formatCheckReport(r, runDir) { g.ok ? ` \u2713 PASS \u2014 ${g.actualPct}% of groundable claims are grounded (threshold ${g.threshold}%)` : ` \u2717 FAIL \u2014 ${g.actualPct}% of groundable claims are grounded, below the ${g.threshold}% threshold` ); } + if (r.semanticError) { + lines.push(``); + lines.push(`Semantic claim-support gate (--semantic):`); + lines.push(` \u2717 FAIL \u2014 ${r.semanticError}`); + } if (r.semantic) { const s = r.semantic; lines.push(``); @@ -3308,13 +3547,13 @@ function formatCheckReport(r, runDir) { } // src/analyze.ts -import { existsSync as existsSync6, readFileSync as readFileSync5 } from "fs"; -import { join as join11 } from "path"; -function loadEvidence2(runDir) { - const path = join11(runDir, "evidence", "evidence.json"); - if (!existsSync6(path)) return []; +import { existsSync as existsSync7, readFileSync as readFileSync6 } from "fs"; +import { join as join12 } from "path"; +function loadEvidence3(runDir) { + const path = join12(runDir, "evidence", "evidence.json"); + if (!existsSync7(path)) return []; try { - const data = JSON.parse(readFileSync5(path, "utf8")); + const data = JSON.parse(readFileSync6(path, "utf8")); return Array.isArray(data) ? data.filter( (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" ) : []; @@ -3323,10 +3562,10 @@ function loadEvidence2(runDir) { } } function loadMetaNotes(runDir) { - const path = join11(runDir, "evidence", "meta.json"); - if (!existsSync6(path)) return []; + const path = join12(runDir, "evidence", "meta.json"); + if (!existsSync7(path)) return []; try { - const meta = JSON.parse(readFileSync5(path, "utf8")); + const meta = JSON.parse(readFileSync6(path, "utf8")); return Array.isArray(meta.notes) ? meta.notes.filter((n) => typeof n === "string") : []; } catch { return []; @@ -3337,7 +3576,7 @@ function featureText(f) { } function analyzeRun(runDir) { const brief = loadBrief(runDir); - const evidence = loadEvidence2(runDir); + const evidence = loadEvidence3(runDir); const notes = loadMetaNotes(runDir); const drill = (cmd, q) => `construct ${cmd} --out ${runDir} --q "${q.replace(/"/g, "'")}"`; const bySource = {}; @@ -3397,8 +3636,8 @@ function formatGapReport(r, runDir) { } // src/verify.ts -import { existsSync as existsSync7, readFileSync as readFileSync6 } from "fs"; -import { isAbsolute, join as join12, resolve as resolve2 } from "path"; +import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs"; +import { isAbsolute, join as join13, resolve as resolve2 } from "path"; var TEST_FILE_RE = /\.(test|spec)\.[^./]+$|_(test|spec)\.[^./]+$|(^|\/)test_[^/]+\.[^./]+$/i; var TEST_SUFFIX_RE = /(^|\/)[^/]*[A-Z]\w*Tests?\.(java|kt|kts|cs|scala|groovy)$/; var TEST_DIR_RE = /(^|\/)(tests?|__tests__|spec|specs|e2e)\//i; @@ -3436,7 +3675,7 @@ function verifyRun(runDir, opts = {}) { const warnings = []; const frTestCoverage = []; const planPath = buildPlanPath(runDir); - if (!existsSync7(planPath)) { + if (!existsSync8(planPath)) { errors.push(`No BUILD-PLAN.json in ${runDir} \u2014 render the SRD first (construct render).`); return { ok: false, errors, warnings, frTestCoverage }; } @@ -3450,13 +3689,13 @@ function verifyRun(runDir, opts = {}) { return { ok: false, errors, warnings, frTestCoverage }; } const manifest = srdManifestPath(runDir); - if (!existsSync7(manifest)) { + if (!existsSync8(manifest)) { errors.push(`No SRD.json in ${runDir} \u2014 the plan cannot be verified against a missing SRD.`); return { ok: false, errors, warnings, frTestCoverage }; } let srd; try { - srd = JSON.parse(readFileSync6(manifest, "utf8")); + srd = JSON.parse(readFileSync7(manifest, "utf8")); } catch (e) { errors.push(`SRD.json is unreadable: ${e.message}`); return { ok: false, errors, warnings, frTestCoverage }; @@ -3500,13 +3739,13 @@ function verifyRun(runDir, opts = {}) { const ok2 = errors.length === 0; return { ok: ok2, errors, warnings, frTestCoverage }; } - if (!existsSync7(appDir)) { + if (!existsSync8(appDir)) { errors.push(`App directory does not exist: ${appDir}.`); return { ok: false, errors, warnings, frTestCoverage }; } for (const t of doneTasks) { for (const rel of [...t.artifacts, ...t.tests]) { - if (!existsSync7(join12(appDir, rel))) errors.push(`${t.id} is done but its declared file is missing: ${rel}.`); + if (!existsSync8(join13(appDir, rel))) errors.push(`${t.id} is done but its declared file is missing: ${rel}.`); } if (t.frIds.length && t.tests.length === 0) { warnings.push(`${t.id} is done but declares no tests \u2014 record the test files that exercise ${t.frIds.join(", ")}.`); @@ -3593,220 +3832,6 @@ function formatVerifyReport(r, runDir) { return lines.join("\n"); } -// src/review.ts -import { existsSync as existsSync8, readFileSync as readFileSync7, writeFileSync as writeFileSync5 } from "fs"; -import { join as join13 } from "path"; -var REVIEW_MAX = 40; -var VALID_VERDICTS = ["supported", "partial", "refuted", "unsupported"]; -function loadEvidence3(path) { - if (!existsSync8(path)) return []; - try { - const data = JSON.parse(readFileSync7(path, "utf8")); - return Array.isArray(data) ? data.filter( - (e) => !!e && typeof e === "object" && typeof e.id === "string" && typeof e.source === "string" - ) : []; - } catch { - return []; - } -} -function srdClaims(srd) { - const out = []; - for (const f of srd.functional) { - const ac = f.acceptance.map((a) => `${a.given} / ${a.when} / ${a.then}`).join("; "); - out.push({ id: f.id, kind: "FR", text: `${f.title}: ${f.description}${ac ? " \u2014 " + ac : ""}`, ev: f.rationaleEvidence }); - } - for (const n of srd.nonFunctional) { - out.push({ id: n.id, kind: "NFR", text: `${n.category}: ${n.statement}${n.metric ? ` (${n.metric})` : ""}`, ev: n.rationaleEvidence }); - } - for (const a of srd.architecture.adrs) { - out.push({ id: `ADR-${a.id}`, kind: "ADR", text: `${a.title}: ${a.decision}`, ev: a.evidence }); - } - srd.competitive.competitors.forEach((c, i) => out.push({ id: `COMP-${i + 1}`, kind: "competitor", text: `${c.name}: ${c.note}`, ev: c.evidence })); - srd.competitive.oss.forEach((o, i) => out.push({ id: `OSS-${i + 1}`, kind: "oss", text: `${o.name}: ${o.note}`, ev: o.evidence })); - return out; -} -function claimDigest(snippet, claim, cap = 600) { - if (snippet.length <= cap) return snippet; - const kws = keywords(claim).map((k) => k.toLowerCase()); - if (!kws.length) return snippet.slice(0, cap); - const step = 150; - let best = 0; - let bestCov = -1; - for (let start = 0; start === 0 || start + cap / 2 < snippet.length; start += step) { - const w = snippet.slice(start, start + cap).toLowerCase(); - let cov = 0; - for (const kw of kws) if (w.includes(kw)) cov++; - if (cov >= bestCov) { - bestCov = cov; - best = start; - } - } - return (best > 0 ? "\u2026 " : "") + snippet.slice(best, best + cap).trim(); -} -function runReview(runDir, opts = {}) { - const manifest = srdManifestPath(runDir); - if (!existsSync8(manifest)) throw new Error(`No SRD.json in ${runDir} \u2014 render the SRD first (construct render).`); - let srd; - try { - srd = JSON.parse(readFileSync7(manifest, "utf8")); - } catch (e) { - throw new Error(`SRD.json is unreadable: ${e.message}`); - } - const evidence = loadEvidence3(join13(runDir, "evidence", "evidence.json")); - const byId = new Map(evidence.map((e) => [e.id, e])); - const pairs = []; - for (const c of srdClaims(srd)) { - for (const id of [...new Set(c.ev)]) { - const e = byId.get(id); - if (!e) continue; - pairs.push({ - claimId: c.id, - kind: c.kind, - claim: c.text.trim().slice(0, 400), - evidenceId: id, - source: e.source, - digest: claimDigest(e.snippet || e.title || e.ref, c.text), - score: e.score - }); - } - } - const max = Math.max(1, Math.floor(opts.maxReview ?? REVIEW_MAX)); - const kept = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)).slice(0, max) : pairs; - const worklist = { run: runDir, pairs: kept.map(({ score, ...rest }) => rest) }; - const todo = { - run: runDir, - pairs: worklist.pairs.map((p) => ({ ...p, verdict: null, note: "" })) - }; - writeFileSync5(join13(runDir, "VERIFY.todo.json"), JSON.stringify(todo, null, 2)); - writeFileSync5(join13(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, kept.length)); - return worklist; -} -function renderWorklistMd(wl, total, kept) { - const out = []; - out.push(`# Claim-support review worklist`); - out.push(""); - out.push( - `For each pair, open the cited evidence and judge whether it **supports** the claim. In \`VERIFY.todo.json\`, set each \`verdict\` to one of supported \xB7 partial \xB7 refuted \xB7 unsupported, add a short \`note\`, save it (e.g. as \`verdicts.json\`), then run \`construct review --apply verdicts.json --out \`.` - ); - if (kept < total) out.push(` -_Showing ${kept} of ${total} pair(s) \u2014 capped at the highest-score evidence._`); - out.push(""); - for (const p of wl.pairs) { - out.push(`## ${p.claimId} \xB7 ${p.evidenceId} (${p.source})`); - out.push(`**Claim (${p.kind}):** ${p.claim}`); - out.push(`**Cited evidence:** ${p.digest}`); - out.push(`**Verdict:** _____ \xB7 **Note:** _____`); - out.push(""); - } - return out.join("\n"); -} -function applyVerdicts(runDir, verdictsPath) { - if (!existsSync8(verdictsPath)) throw new Error(`verdicts file not found: ${verdictsPath}`); - let raw; - try { - raw = JSON.parse(readFileSync7(verdictsPath, "utf8")); - } catch (e) { - throw new Error(`verdicts file is not valid JSON (${verdictsPath}): ${e.message}`); - } - const list = Array.isArray(raw) ? raw : raw && typeof raw === "object" && Array.isArray(raw.pairs) ? raw.pairs : null; - if (list === null) { - throw new Error(`verdicts file must be a JSON array of verdicts or an object with a "pairs" array (${verdictsPath}).`); - } - const verdicts = []; - const seen = /* @__PURE__ */ new Set(); - const key = (claimId, evidenceId) => `${claimId}::${evidenceId}`; - for (const v of list) { - if (!v || typeof v.claimId !== "string" || typeof v.evidenceId !== "string") continue; - const verdict = VALID_VERDICTS.includes(v.verdict) ? v.verdict : void 0; - verdicts.push({ - claimId: v.claimId, - kind: v.kind, - claim: typeof v.claim === "string" ? v.claim : "", - evidenceId: v.evidenceId, - source: v.source, - digest: typeof v.digest === "string" ? v.digest : "", - verdict, - note: typeof v.note === "string" ? v.note : "" - }); - seen.add(key(v.claimId, v.evidenceId)); - } - const todoPath = join13(runDir, "VERIFY.todo.json"); - if (existsSync8(todoPath)) { - try { - const todo = JSON.parse(readFileSync7(todoPath, "utf8")); - for (const p of todo.pairs ?? []) { - if (!p || typeof p.claimId !== "string" || typeof p.evidenceId !== "string") continue; - if (seen.has(key(p.claimId, p.evidenceId))) continue; - verdicts.push({ - claimId: p.claimId, - kind: p.kind, - claim: p.claim ?? "", - evidenceId: p.evidenceId, - source: p.source, - digest: p.digest ?? "", - verdict: void 0, - note: "" - }); - seen.add(key(p.claimId, p.evidenceId)); - } - } catch { - } - } - const result = reduceVerdicts(verdicts); - writeFileSync5(join13(runDir, "VERIFY.json"), JSON.stringify({ ...result, verdicts }, null, 2)); - return result; -} -function reduceVerdicts(verdicts) { - const counts = { supported: 0, partial: 0, refuted: 0, unsupported: 0 }; - for (const v of verdicts) if (v.verdict && counts[v.verdict] !== void 0) counts[v.verdict]++; - const byClaim = /* @__PURE__ */ new Map(); - for (const v of verdicts) { - const group = byClaim.get(v.claimId) ?? []; - group.push(v); - byClaim.set(v.claimId, group); - } - const failures = []; - const unadjudicated = []; - for (const [claimId, group] of byClaim) { - const adjudicated = group.filter((g) => !!g.verdict); - if (adjudicated.length < group.length) unadjudicated.push(claimId); - const refuted = adjudicated.find((g) => g.verdict === "refuted"); - const hasSupport = adjudicated.some((g) => g.verdict === "supported" || g.verdict === "partial"); - if (refuted) { - failures.push({ claimId, evidenceId: refuted.evidenceId, verdict: "refuted", note: refuted.note }); - } else if (adjudicated.length === group.length && adjudicated.length > 0 && !hasSupport) { - const u = adjudicated.find((g) => g.verdict === "unsupported") ?? adjudicated[0]; - failures.push({ claimId, evidenceId: u.evidenceId, verdict: u.verdict, note: u.note }); - } - } - return { - ok: failures.length === 0, - pairs: verdicts.length, - adjudicated: verdicts.filter((v) => !!v.verdict).length, - supported: counts.supported, - partial: counts.partial, - refuted: counts.refuted, - unsupported: counts.unsupported, - failures, - unadjudicated - }; -} -function formatReviewReport(r) { - const lines = []; - lines.push(`construct review: ${r.adjudicated}/${r.pairs} pair(s) adjudicated`); - lines.push(` supported: ${r.supported} \xB7 partial: ${r.partial} \xB7 refuted: ${r.refuted} \xB7 unsupported: ${r.unsupported}`); - for (const f of r.failures.slice(0, 12)) { - lines.push(` \u2717 ${f.claimId} (${f.evidenceId}): ${f.verdict}${f.note ? " \u2014 " + f.note : ""}`); - } - if (r.unadjudicated.length) { - lines.push(` \u26A0 ${r.unadjudicated.length} claim(s) not fully adjudicated: ${r.unadjudicated.join(", ")}`); - } - lines.push( - !r.ok ? ` \u2717 some claims are refuted or unsupported` : r.unadjudicated.length ? ` \u2713 no refuted or unsupported claims (${r.unadjudicated.length} still unadjudicated \u2014 see above)` : ` \u2713 every grounded claim is backed by its cited evidence` - ); - return lines.join("\n"); -} - // src/cli.ts var HELP = `construct v${VERSION} Turn a product idea into a grounded, buildable SRD suite. Interview \u2192 research @@ -3819,7 +3844,7 @@ Usage: construct analyze --out [--json] construct web|oss|tech|so --out [--q ""] [--url ] [--seeds ] construct render --out [--level light|complex] [--merge] [--no-design] [--prd] - construct check --out [--min-grounding <0-100>] [--semantic] [--json] + construct check --out [--min-grounding <0-100>] [--semantic [--allow-unverified]] [--json] construct review --out [--apply ] [--max-review N] [--json] construct verify --out [--app ] [--run-tests] [--strict] [--json] construct status --out [--json] @@ -3859,6 +3884,9 @@ Options: --level light | complex (default: light) --min-grounding For 'check': fail unless \u2265 n% of claims are grounded (opt-in) --semantic For 'check': fold in the 'review' claim-support verdicts + (fail-closed: no/unreadable VERIFY.json fails the check) + --allow-unverified For 'check --semantic': degrade a missing/unreadable + VERIFY.json to a warning instead of failing --apply For 'review': consume an adjudicated verdicts file + gate --app For 'verify': the built app directory (default: conventions.appDir) --run-tests For 'verify': also execute testCommand + per-task verify commands @@ -3900,7 +3928,7 @@ var VALUE_FLAGS = /* @__PURE__ */ new Set([ "apply", "max-review" ]); -var BOOL_FLAGS = /* @__PURE__ */ new Set(["semantic", "merge", "json", "refresh", "run-tests", "strict", "no-design", "prd"]); +var BOOL_FLAGS = /* @__PURE__ */ new Set(["semantic", "merge", "json", "refresh", "run-tests", "strict", "no-design", "prd", "allow-unverified"]); function fail(message) { process.stderr.write(`construct: ${message} `); @@ -4149,7 +4177,7 @@ ${v.errors.map((e) => " - " + e).join("\n")}`); fail("invalid --min-grounding (expected a number between 0 and 100)"); } } - const res = checkRun(out, { minGrounding, semantic: p.bools.has("semantic") }); + const res = checkRun(out, { minGrounding, semantic: p.bools.has("semantic"), allowUnverified: p.bools.has("allow-unverified") }); if (p.bools.has("json")) { process.stdout.write(JSON.stringify(res, null, 2) + "\n"); } else { diff --git a/src/check.ts b/src/check.ts index 08f34d3..973568e 100644 --- a/src/check.ts +++ b/src/check.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import type { Stats } from "node:fs"; import { join, relative, sep } from "node:path"; +import { reduceVerdicts } from "./review.js"; import { srdManifestPath } from "./srd.js"; import { REQUIRED_NFR, DESIGN_TOKEN_CATEGORIES, DESIGN_TOKENS_SEEDED_BANNER } from "./types.js"; import type { CheckResult, SRD, EvidenceItem, CoverageReport, ClaimVerifyResult } from "./types.js"; @@ -141,22 +142,45 @@ const TEMPLATED_METRIC_RE = /^A measurable target for "/; // Fold the resolved claim-support record (VERIFY.json, written by `review // --apply`) into a check result when `--semantic` is requested. Strictly // additive: it can only ADD a failure (a refuted/unsupported claim), never relax -// the structural gate. Missing VERIFY.json warns (run `review` first), never fails. -function applySemantic(runDir: string, result: CheckResult): void { +// the structural gate. FAIL-CLOSED: passing `--semantic` asserts the support +// gate actually engaged, so a missing/unreadable/verdict-less VERIFY.json fails +// the check unless `--allow-unverified` degrades it to the advisory warning. +// The verdict itself is re-reduced from `verdicts[]` on every check — the +// persisted summary is never trusted, so a stale or hand-tampered `ok` can not +// green-light the gate. +function applySemantic(runDir: string, result: CheckResult, allowUnverified: boolean): void { const p = join(runDir, "VERIFY.json"); + const skip = (reason: string, hint: string) => { + if (allowUnverified) { + result.structural.warnings.push(`--semantic: ${reason} — ${hint}; semantic gate skipped (--allow-unverified).`); + } else { + result.semanticError = `${reason} — ${hint}, or pass --allow-unverified to degrade this to a warning.`; + result.ok = false; + } + }; if (!existsSync(p)) { - result.structural.warnings.push("--semantic: no VERIFY.json — run `construct review` then `review --apply ` first; semantic gate skipped."); + skip("no VERIFY.json", "run `construct review` then `review --apply ` first"); return; } + let sem: ClaimVerifyResult; try { - const sem = JSON.parse(readFileSync(p, "utf8")) as ClaimVerifyResult; - result.semantic = sem; - if (!sem.ok) result.ok = false; - if (sem.unadjudicated?.length) { - result.structural.warnings.push(`${sem.unadjudicated.length} claim(s) not fully adjudicated by review.`); - } + sem = JSON.parse(readFileSync(p, "utf8")) as ClaimVerifyResult; } catch (e) { - result.structural.warnings.push(`--semantic: VERIFY.json is unreadable (${(e as Error).message}).`); + skip(`VERIFY.json is unreadable (${(e as Error).message})`, "re-run `review --apply ` to regenerate it"); + return; + } + if (!Array.isArray(sem.verdicts)) { + skip("VERIFY.json carries no verdicts[] (legacy or hand-edited)", "re-run `review --apply ` to regenerate it"); + return; + } + const reduced = reduceVerdicts(sem.verdicts); + if (reduced.ok !== sem.ok) { + result.structural.warnings.push("VERIFY.json's persisted summary disagreed with its verdicts — recomputed at check time."); + } + result.semantic = { ...reduced, verdicts: sem.verdicts }; + if (!reduced.ok) result.ok = false; + if (reduced.unadjudicated.length) { + result.structural.warnings.push(`${reduced.unadjudicated.length} claim(s) not fully adjudicated by review.`); } } @@ -238,7 +262,7 @@ function checkModules(runDir: string, srd: SRD, errors: string[], warnings: stri // advisory coverage report itself never flips `ok`). With `opts.semantic`, ALSO // folds in the VERIFY.json claim-support verdicts (fails on a refuted/unsupported // claim) — additive: plain `check` (no opts) is byte-for-byte unchanged. -export function checkRun(runDir: string, opts: { minGrounding?: number; semantic?: boolean } = {}): CheckResult { +export function checkRun(runDir: string, opts: { minGrounding?: number; semantic?: boolean; allowUnverified?: boolean } = {}): CheckResult { const errors: string[] = []; const warnings: string[] = []; @@ -373,7 +397,7 @@ export function checkRun(runDir: string, opts: { minGrounding?: number; semantic const ok = structuralOk && (grounding?.ok ?? true); const result: CheckResult = { ok, structural: { ok: structuralOk, errors, warnings }, coverage, grounding }; - if (opts.semantic) applySemantic(runDir, result); + if (opts.semantic) applySemantic(runDir, result, opts.allowUnverified ?? false); return result; } @@ -408,6 +432,11 @@ export function formatCheckReport(r: CheckResult, runDir: string): string { : ` ✗ FAIL — ${g.actualPct}% of groundable claims are grounded, below the ${g.threshold}% threshold`, ); } + if (r.semanticError) { + lines.push(``); + lines.push(`Semantic claim-support gate (--semantic):`); + lines.push(` ✗ FAIL — ${r.semanticError}`); + } if (r.semantic) { const s = r.semantic; lines.push(``); diff --git a/src/cli.ts b/src/cli.ts index b8f9ddf..0ab2baa 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -32,7 +32,7 @@ Usage: construct analyze --out [--json] construct web|oss|tech|so --out [--q ""] [--url ] [--seeds ] construct render --out [--level light|complex] [--merge] [--no-design] [--prd] - construct check --out [--min-grounding <0-100>] [--semantic] [--json] + construct check --out [--min-grounding <0-100>] [--semantic [--allow-unverified]] [--json] construct review --out [--apply ] [--max-review N] [--json] construct verify --out [--app ] [--run-tests] [--strict] [--json] construct status --out [--json] @@ -72,6 +72,9 @@ Options: --level light | complex (default: light) --min-grounding For 'check': fail unless ≥ n% of claims are grounded (opt-in) --semantic For 'check': fold in the 'review' claim-support verdicts + (fail-closed: no/unreadable VERIFY.json fails the check) + --allow-unverified For 'check --semantic': degrade a missing/unreadable + VERIFY.json to a warning instead of failing --apply For 'review': consume an adjudicated verdicts file + gate --app For 'verify': the built app directory (default: conventions.appDir) --run-tests For 'verify': also execute testCommand + per-task verify commands @@ -114,7 +117,7 @@ const VALUE_FLAGS = new Set([ "apply", "max-review", ]); -const BOOL_FLAGS = new Set(["semantic", "merge", "json", "refresh", "run-tests", "strict", "no-design", "prd"]); +const BOOL_FLAGS = new Set(["semantic", "merge", "json", "refresh", "run-tests", "strict", "no-design", "prd", "allow-unverified"]); function fail(message: string): never { process.stderr.write(`construct: ${message}\n`); @@ -397,7 +400,7 @@ async function main(): Promise { fail("invalid --min-grounding (expected a number between 0 and 100)"); } } - const res = checkRun(out, { minGrounding, semantic: p.bools.has("semantic") }); + const res = checkRun(out, { minGrounding, semantic: p.bools.has("semantic"), allowUnverified: p.bools.has("allow-unverified") }); if (p.bools.has("json")) { process.stdout.write(JSON.stringify(res, null, 2) + "\n"); } else { diff --git a/src/types.ts b/src/types.ts index 267155b..90395e3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -405,6 +405,10 @@ export interface CheckResult { // Present ONLY with `check --semantic` (folds the `review` verdicts). Fails the // gate on a refuted/unsupported claim. Undefined — and `ok` unchanged — without it. semantic?: ClaimVerifyResult; + // Present when `--semantic` was requested but the gate could not engage + // (missing/unreadable/verdict-less VERIFY.json) and `--allow-unverified` was + // not passed. Fail-closed: its presence means `ok` is false. + semanticError?: string; } // --------------------------------------------------------------------------- diff --git a/tests/check.test.ts b/tests/check.test.ts index 0361477..5c543ad 100644 --- a/tests/check.test.ts +++ b/tests/check.test.ts @@ -358,22 +358,49 @@ describe("checkRun — manifest & content edge cases", () => { }); describe("checkRun --semantic composition edge cases", () => { - it("folds in unadjudicated claims from VERIFY.json as a warning (still passes)", () => { + // A verdict pair as `review --apply` persists it (verdict: null = unadjudicated). + const pair = (claimId: string, verdict: string | null) => ({ + claimId, + kind: "FR", + claim: `${claimId} claim`, + evidenceId: "E1", + source: "oss", + digest: "d", + verdict, + note: "", + }); + + it("folds in unadjudicated claims from VERIFY.json's verdicts[] as a warning (still passes)", () => { const dir = renderRun(); writeFileSync( join(dir, "VERIFY.json"), - JSON.stringify({ ok: true, pairs: 2, adjudicated: 1, supported: 1, partial: 0, refuted: 0, unsupported: 0, failures: [], unadjudicated: ["FR-002"] }), + JSON.stringify({ + ok: true, + pairs: 2, + adjudicated: 1, + supported: 1, + partial: 0, + refuted: 0, + unsupported: 0, + failures: [], + unadjudicated: ["FR-002"], + verdicts: [pair("FR-001", "supported"), pair("FR-002", null)], + }), ); const r = checkRun(dir, { semantic: true }); expect(r.semantic?.ok).toBe(true); expect(r.structural.warnings.join(" ")).toMatch(/not fully adjudicated/); }); - it("warns when VERIFY.json is unreadable", () => { + it("fails closed when VERIFY.json is unreadable; --allow-unverified degrades to the warning", () => { const dir = renderRun(); writeFileSync(join(dir, "VERIFY.json"), "}broken{"); - const r = checkRun(dir, { semantic: true }); - expect(r.structural.warnings.join(" ")).toMatch(/VERIFY\.json is unreadable/); + const strict = checkRun(dir, { semantic: true }); + expect(strict.ok).toBe(false); + expect(strict.semanticError).toMatch(/VERIFY\.json is unreadable/); + const lax = checkRun(dir, { semantic: true, allowUnverified: true }); + expect(lax.semanticError).toBeUndefined(); + expect(lax.structural.warnings.join(" ")).toMatch(/VERIFY\.json is unreadable/); }); }); diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 1a6ef2d..6345449 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -58,6 +58,12 @@ describe("parseArgs", () => { expect(p.bools.has("json")).toBe(true); }); + it("parses check with --semantic --allow-unverified", () => { + const p = parseArgs(["check", "--out", "run", "--semantic", "--allow-unverified"]); + expect(p.bools.has("semantic")).toBe(true); + expect(p.bools.has("allow-unverified")).toBe(true); + }); + it("parses the analyze command", () => { const p = parseArgs(["analyze", "--out=run", "--json"]); expect(p.command).toBe("analyze"); diff --git a/tests/e2e.test.ts b/tests/e2e.test.ts index 884ee02..d652a10 100644 --- a/tests/e2e.test.ts +++ b/tests/e2e.test.ts @@ -321,13 +321,34 @@ describe("e2e: claim-support gate (review → apply → check --semantic)", () = expect(todo.pairs).toHaveLength(2); }); - it("warns (does not fail) when no VERIFY.json exists", () => { + it("fails closed (exit 1) when --semantic has no VERIFY.json", () => { const run = rendered(); const gate = cli(["check", "--out", run, "--semantic"]); + expect(gate.status).toBe(1); + expect(gate.stdout.toLowerCase()).toContain("verify.json"); + expect(gate.stdout).toContain("--allow-unverified"); + }); + + it("--allow-unverified restores the advisory skip (exit 0) when VERIFY.json is missing", () => { + const run = rendered(); + const gate = cli(["check", "--out", run, "--semantic", "--allow-unverified"]); expect(gate.status).toBe(0); expect(gate.stdout.toLowerCase()).toContain("no verify.json"); }); + it("recomputes the semantic verdict — a hand-tampered ok:true with a refuted verdict still fails", () => { + const run = rendered(); + cli(["review", "--out", run]); + cli(["review", "--out", run, "--apply", verdictsFor(run, () => "supported")]); + const p = join(run, "VERIFY.json"); + const sem = JSON.parse(readFileSync(p, "utf8")); + sem.verdicts[0].verdict = "refuted"; // doctored output, stale green summary + writeFileSync(p, JSON.stringify(sem, null, 2)); + const gate = cli(["check", "--out", run, "--semantic"]); + expect(gate.status).toBe(1); + expect(gate.stdout).toContain("FAIL"); + }); + // --- Regression guards for the hardening fixes (were: leaked ENOENT / silent vacuous pass) --- it("review on a missing SRD gives a clean domain error, not a raw fs error", () => { diff --git a/tests/semantic-review.test.ts b/tests/semantic-review.test.ts index daa1eda..09fb453 100644 --- a/tests/semantic-review.test.ts +++ b/tests/semantic-review.test.ts @@ -160,14 +160,77 @@ describe("check --semantic composition (additive)", () => { rmSync(dir, { recursive: true, force: true }); }); - it("warns (does not add a semantic verdict) when no VERIFY.json exists", () => { + it("fails closed when no VERIFY.json exists (names --allow-unverified)", () => { const dir = scratch(); run(dir, [{ id: "FR-001", ev: ["E1"] }], EVIDENCE); const r = checkRun(dir, { semantic: true }); expect(r.semantic).toBeUndefined(); + expect(r.ok).toBe(false); + expect(r.semanticError).toMatch(/VERIFY\.json/); + expect(r.semanticError).toMatch(/--allow-unverified/); + expect(r.semanticError?.toLowerCase()).toContain("review"); + rmSync(dir, { recursive: true, force: true }); + }); + + it("--allow-unverified degrades a missing VERIFY.json to the advisory warning", () => { + const dir = scratch(); + run(dir, [{ id: "FR-001", ev: ["E1"] }], EVIDENCE); + const r = checkRun(dir, { semantic: true, allowUnverified: true }); + expect(r.semantic).toBeUndefined(); + expect(r.semanticError).toBeUndefined(); expect(r.structural.warnings.join(" ").toLowerCase()).toContain("review"); rmSync(dir, { recursive: true, force: true }); }); + + it("recomputes ok from verdicts[] — a tampered persisted ok:true with a refuted verdict still fails", () => { + const dir = scratch(); + run(dir, [{ id: "FR-001", ev: ["E1"] }], EVIDENCE); + runReview(dir); + applyVerdicts(dir, writeVerdicts(dir, { E1: "supported" })); + // Tamper the OUTPUT: flip the verdict but keep the persisted summary green. + const p = join(dir, "VERIFY.json"); + const sem = JSON.parse(readFileSync(p, "utf8")); + sem.verdicts[0].verdict = "refuted"; + expect(sem.ok).toBe(true); // the doctored summary still claims a pass + writeFileSync(p, JSON.stringify(sem, null, 2)); + + const r = checkRun(dir, { semantic: true }); + expect(r.semantic?.ok).toBe(false); // recomputed from verdicts[], not trusted + expect(r.ok).toBe(false); + expect(r.structural.warnings.join(" ")).toMatch(/recomputed/i); + rmSync(dir, { recursive: true, force: true }); + }); + + it("fails closed on a legacy VERIFY.json without verdicts[] unless --allow-unverified", () => { + const dir = scratch(); + run(dir, [{ id: "FR-001", ev: ["E1"] }], EVIDENCE); + writeFileSync( + join(dir, "VERIFY.json"), + JSON.stringify({ ok: true, pairs: 1, adjudicated: 1, supported: 1, partial: 0, refuted: 0, unsupported: 0, failures: [], unadjudicated: [] }), + ); + const strict = checkRun(dir, { semantic: true }); + expect(strict.ok).toBe(false); + expect(strict.semanticError).toMatch(/verdicts/i); + expect(strict.semanticError).toMatch(/--allow-unverified/); + const lax = checkRun(dir, { semantic: true, allowUnverified: true }); + expect(lax.semanticError).toBeUndefined(); + expect(lax.structural.warnings.join(" ")).toMatch(/verdicts/i); + rmSync(dir, { recursive: true, force: true }); + }); + + it("fails closed on an unreadable VERIFY.json unless --allow-unverified", () => { + const dir = scratch(); + run(dir, [{ id: "FR-001", ev: ["E1"] }], EVIDENCE); + writeFileSync(join(dir, "VERIFY.json"), "}broken{"); + const strict = checkRun(dir, { semantic: true }); + expect(strict.ok).toBe(false); + expect(strict.semanticError).toMatch(/unreadable/i); + expect(strict.semanticError).toMatch(/--allow-unverified/); + const lax = checkRun(dir, { semantic: true, allowUnverified: true }); + expect(lax.semanticError).toBeUndefined(); + expect(lax.structural.warnings.join(" ").toLowerCase()).toContain("unreadable"); + rmSync(dir, { recursive: true, force: true }); + }); }); describe("runReview — claim coverage & error paths", () => { From 7c7b7d127480f92020a88201c81f27e20064bb41 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:41:22 +0200 Subject: [PATCH 02/10] fix(srd,check): competitor grounding requires a literal mention; warn loudly when cited claims skip the semantic gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The renderer auto-attached the same market [E#] to every competitor a listicle merely token-overlapped — citation-washing on by default, and only the opt-in --semantic gate caught it. Competitor grounding now requires the evidence title+snippet to literally name the competitor (word-bounded phrase, punctuation-safe) before matchEvidence may cite it. check also prints a prominent "Semantic gate: SKIPPED" block whenever an SRD carries resolved citations but --semantic was not passed, so an unreviewed citation can no longer read as a verified one. Advisory — plain check exit codes are unchanged. Co-Authored-By: Claude Fable 5 --- scripts/construct.mjs | 31 +++++++++++++- skills/construct/scripts/construct.mjs | 31 +++++++++++++- src/check.ts | 21 +++++++++- src/srd.ts | 25 ++++++++++- src/types.ts | 4 ++ tests/check.test.ts | 37 ++++++++++++++++ tests/srd.test.ts | 58 +++++++++++++++++++++++++- 7 files changed, 200 insertions(+), 7 deletions(-) diff --git a/scripts/construct.mjs b/scripts/construct.mjs index 08da3f9..e935bdb 100755 --- a/scripts/construct.mjs +++ b/scripts/construct.mjs @@ -1611,6 +1611,13 @@ function matchEvidence(text, evidence, n, onlySources) { } return out; } +function mentionsEntity(name, e) { + const phrase = name.trim().toLowerCase().replace(/\s+/g, " "); + if (!phrase) return false; + const hay = `${e.title} ${e.snippet}`.toLowerCase().replace(/\s+/g, " "); + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`(? [e.id, e])); const competitors = brief.competitors.map((name) => { - const ev = matchEvidence(name, evidence, 2, ["market"]); + const ev = matchEvidence( + name, + evidence.filter((e) => mentionsEntity(name, e)), + 2, + ["market"] + ); return { name, note: noteFrom(ev, evById) || `Comparable product / alternative to "${productName}".`, evidence: ev }; }); const ossByKey = /* @__PURE__ */ new Map(); @@ -3497,7 +3509,12 @@ function checkRun(runDir, opts = {}) { } const ok = structuralOk && (grounding?.ok ?? true); const result = { ok, structural: { ok: structuralOk, errors, warnings }, coverage, grounding }; - if (opts.semantic) applySemantic(runDir, result, opts.allowUnverified ?? false); + if (opts.semantic) { + applySemantic(runDir, result, opts.allowUnverified ?? false); + } else if (coverage.resolved.length > 0) { + const citedClaims = coverage.frGrounded + coverage.nfrGrounded + coverage.adrGrounded; + result.semanticSkipped = { citedClaims, verifyExists: existsSync6(join11(runDir, "VERIFY.json")) }; + } return result; } function pct(part, total) { @@ -3528,6 +3545,16 @@ function formatCheckReport(r, runDir) { g.ok ? ` \u2713 PASS \u2014 ${g.actualPct}% of groundable claims are grounded (threshold ${g.threshold}%)` : ` \u2717 FAIL \u2014 ${g.actualPct}% of groundable claims are grounded, below the ${g.threshold}% threshold` ); } + if (r.semanticSkipped) { + const s = r.semanticSkipped; + lines.push(``); + lines.push(`Semantic gate: SKIPPED`); + lines.push(` \u26A0 ${s.citedClaims} cited claim(s) were never adversarially verified \u2014 a citation`); + lines.push( + s.verifyExists ? ` proves nothing until reviewed. A VERIFY.json exists \u2014 re-run with --semantic to gate on it.` : ` proves nothing until reviewed. Run \`construct review --out \`, adjudicate the` + ); + if (!s.verifyExists) lines.push(` worklist, then \`construct check --semantic\`.`); + } if (r.semanticError) { lines.push(``); lines.push(`Semantic claim-support gate (--semantic):`); diff --git a/skills/construct/scripts/construct.mjs b/skills/construct/scripts/construct.mjs index 08da3f9..e935bdb 100755 --- a/skills/construct/scripts/construct.mjs +++ b/skills/construct/scripts/construct.mjs @@ -1611,6 +1611,13 @@ function matchEvidence(text, evidence, n, onlySources) { } return out; } +function mentionsEntity(name, e) { + const phrase = name.trim().toLowerCase().replace(/\s+/g, " "); + if (!phrase) return false; + const hay = `${e.title} ${e.snippet}`.toLowerCase().replace(/\s+/g, " "); + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`(? [e.id, e])); const competitors = brief.competitors.map((name) => { - const ev = matchEvidence(name, evidence, 2, ["market"]); + const ev = matchEvidence( + name, + evidence.filter((e) => mentionsEntity(name, e)), + 2, + ["market"] + ); return { name, note: noteFrom(ev, evById) || `Comparable product / alternative to "${productName}".`, evidence: ev }; }); const ossByKey = /* @__PURE__ */ new Map(); @@ -3497,7 +3509,12 @@ function checkRun(runDir, opts = {}) { } const ok = structuralOk && (grounding?.ok ?? true); const result = { ok, structural: { ok: structuralOk, errors, warnings }, coverage, grounding }; - if (opts.semantic) applySemantic(runDir, result, opts.allowUnverified ?? false); + if (opts.semantic) { + applySemantic(runDir, result, opts.allowUnverified ?? false); + } else if (coverage.resolved.length > 0) { + const citedClaims = coverage.frGrounded + coverage.nfrGrounded + coverage.adrGrounded; + result.semanticSkipped = { citedClaims, verifyExists: existsSync6(join11(runDir, "VERIFY.json")) }; + } return result; } function pct(part, total) { @@ -3528,6 +3545,16 @@ function formatCheckReport(r, runDir) { g.ok ? ` \u2713 PASS \u2014 ${g.actualPct}% of groundable claims are grounded (threshold ${g.threshold}%)` : ` \u2717 FAIL \u2014 ${g.actualPct}% of groundable claims are grounded, below the ${g.threshold}% threshold` ); } + if (r.semanticSkipped) { + const s = r.semanticSkipped; + lines.push(``); + lines.push(`Semantic gate: SKIPPED`); + lines.push(` \u26A0 ${s.citedClaims} cited claim(s) were never adversarially verified \u2014 a citation`); + lines.push( + s.verifyExists ? ` proves nothing until reviewed. A VERIFY.json exists \u2014 re-run with --semantic to gate on it.` : ` proves nothing until reviewed. Run \`construct review --out \`, adjudicate the` + ); + if (!s.verifyExists) lines.push(` worklist, then \`construct check --semantic\`.`); + } if (r.semanticError) { lines.push(``); lines.push(`Semantic claim-support gate (--semantic):`); diff --git a/src/check.ts b/src/check.ts index 973568e..c67bef1 100644 --- a/src/check.ts +++ b/src/check.ts @@ -397,7 +397,14 @@ export function checkRun(runDir: string, opts: { minGrounding?: number; semantic const ok = structuralOk && (grounding?.ok ?? true); const result: CheckResult = { ok, structural: { ok: structuralOk, errors, warnings }, coverage, grounding }; - if (opts.semantic) applySemantic(runDir, result, opts.allowUnverified ?? false); + if (opts.semantic) { + applySemantic(runDir, result, opts.allowUnverified ?? false); + } else if (coverage.resolved.length > 0) { + // Citations exist but the support gate never engaged — surface it loudly + // (advisory): a citation proves nothing until the review adjudicates it. + const citedClaims = coverage.frGrounded + coverage.nfrGrounded + coverage.adrGrounded; + result.semanticSkipped = { citedClaims, verifyExists: existsSync(join(runDir, "VERIFY.json")) }; + } return result; } @@ -432,6 +439,18 @@ export function formatCheckReport(r: CheckResult, runDir: string): string { : ` ✗ FAIL — ${g.actualPct}% of groundable claims are grounded, below the ${g.threshold}% threshold`, ); } + if (r.semanticSkipped) { + const s = r.semanticSkipped; + lines.push(``); + lines.push(`Semantic gate: SKIPPED`); + lines.push(` ⚠ ${s.citedClaims} cited claim(s) were never adversarially verified — a citation`); + lines.push( + s.verifyExists + ? ` proves nothing until reviewed. A VERIFY.json exists — re-run with --semantic to gate on it.` + : ` proves nothing until reviewed. Run \`construct review --out \`, adjudicate the`, + ); + if (!s.verifyExists) lines.push(` worklist, then \`construct check --semantic\`.`); + } if (r.semanticError) { lines.push(``); lines.push(`Semantic claim-support gate (--semantic):`); diff --git a/src/srd.ts b/src/srd.ts index 1437302..a0c48cb 100644 --- a/src/srd.ts +++ b/src/srd.ts @@ -83,6 +83,22 @@ export function matchEvidence(text: string, evidence: EvidenceItem[], n: number, return out; } +// Does the evidence item literally NAME this entity? Word-bounded, +// case-insensitive, whitespace-normalized phrase match over title+snippet. +// Stricter than matchEvidence's token overlap on purpose: a listicle that +// shares tokens with a product name ("a new wave of money apps" vs the +// competitor "Money Wave") is NOT a mention, and auto-citing it would wash the +// citation onto a product the page never discusses. +export function mentionsEntity(name: string, e: EvidenceItem): boolean { + const phrase = name.trim().toLowerCase().replace(/\s+/g, " "); + if (!phrase) return false; + const hay = `${e.title} ${e.snippet}`.toLowerCase().replace(/\s+/g, " "); + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + // Lookarounds instead of \b so punctuation-bearing names (Next.js, C++) + // stay word-bounded at their alphanumeric edges. + return new RegExp(`(? [e.id, e])); const competitors: CompetitorRow[] = brief.competitors.map((name) => { - const ev = matchEvidence(name, evidence, 2, ["market"]); + // Only evidence that literally names the competitor may ground it — token + // overlap alone clones one listicle's [E#] onto every product it brushes. + const ev = matchEvidence( + name, + evidence.filter((e) => mentionsEntity(name, e)), + 2, + ["market"], + ); return { name, note: noteFrom(ev, evById) || `Comparable product / alternative to "${productName}".`, evidence: ev }; }); const ossByKey = new Map(); diff --git a/src/types.ts b/src/types.ts index 90395e3..4c97f02 100644 --- a/src/types.ts +++ b/src/types.ts @@ -409,6 +409,10 @@ export interface CheckResult { // (missing/unreadable/verdict-less VERIFY.json) and `--allow-unverified` was // not passed. Fail-closed: its presence means `ok` is false. semanticError?: string; + // Present when the SRD carries resolved citations but the caller did NOT pass + // `--semantic`: the claim-support gate never engaged, so the citations are + // unverified. Advisory only — reported loudly, never flips `ok`. + semanticSkipped?: { citedClaims: number; verifyExists: boolean }; } // --------------------------------------------------------------------------- diff --git a/tests/check.test.ts b/tests/check.test.ts index 5c543ad..7cfe09d 100644 --- a/tests/check.test.ts +++ b/tests/check.test.ts @@ -357,6 +357,43 @@ describe("checkRun — manifest & content edge cases", () => { }); }); +describe("checkRun — semantic-skip warning (cited claims without --semantic)", () => { + it("reports the skipped semantic gate when the SRD carries resolved citations", () => { + const dir = renderRun(); // sample brief + evidence → citations resolve + const r = checkRun(dir); + expect(r.coverage.resolved.length).toBeGreaterThan(0); // precondition + expect(r.semanticSkipped).toBeDefined(); + expect(r.semanticSkipped!.citedClaims).toBeGreaterThan(0); + expect(r.ok).toBe(true); // never gates + const report = formatCheckReport(r, dir); + expect(report).toMatch(/Semantic gate: SKIPPED/); + expect(report).toMatch(/never .*verified|not .*verified/i); + expect(report).toContain("construct review"); + }); + + it("stays silent when there are no citations to verify", () => { + const dir = renderRun({ withEvidence: false, briefOverride: { competitors: [], ossSeeds: [] } }); + const r = checkRun(dir); + expect(r.coverage.resolved.length).toBe(0); + expect(r.semanticSkipped).toBeUndefined(); + expect(formatCheckReport(r, dir)).not.toMatch(/Semantic gate: SKIPPED/); + }); + + it("stays silent when --semantic engages the gate", () => { + const dir = renderRun(); + const r = checkRun(dir, { semantic: true, allowUnverified: true }); + expect(r.semanticSkipped).toBeUndefined(); + }); + + it("points at VERIFY.json when one already exists", () => { + const dir = renderRun(); + writeFileSync(join(dir, "VERIFY.json"), JSON.stringify({ ok: true, verdicts: [] })); + const r = checkRun(dir); + expect(r.semanticSkipped?.verifyExists).toBe(true); + expect(formatCheckReport(r, dir)).toMatch(/--semantic/); + }); +}); + describe("checkRun --semantic composition edge cases", () => { // A verdict pair as `review --apply` persists it (verdict: null = unadjudicated). const pair = (claimId: string, verdict: string | null) => ({ diff --git a/tests/srd.test.ts b/tests/srd.test.ts index 75bc55f..4c52c8c 100644 --- a/tests/srd.test.ts +++ b/tests/srd.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; -import { buildSRD, matchEvidence, deriveA11yStandard } from "../src/srd.js"; +import { buildSRD, matchEvidence, mentionsEntity, deriveA11yStandard } from "../src/srd.js"; import { DESIGN_TOKEN_CATEGORIES } from "../src/types.js"; import type { Brief, EvidenceItem } from "../src/types.js"; @@ -53,6 +53,62 @@ describe("matchEvidence", () => { }); }); +describe("mentionsEntity", () => { + const item = (text: string) => ({ id: "E1", source: "market", title: "", ref: "r", score: 1, snippet: text }) as EvidenceItem; + + it("matches the whole name as a word-bounded, case-insensitive phrase", () => { + expect(mentionsEntity("Pocket", item("Pocket lets you save articles"))).toBe(true); + expect(mentionsEntity("pocket", item("Compare POCKET and others"))).toBe(true); + expect(mentionsEntity("Money Wave", item("We benchmarked Money Wave against Mint"))).toBe(true); + }); + + it("does not match inside a larger word", () => { + expect(mentionsEntity("Wave", item("microwave ovens on sale"))).toBe(false); + expect(mentionsEntity("Mint", item("badminton rackets"))).toBe(false); + }); + + it("does not treat token overlap as a mention", () => { + expect(mentionsEntity("Money Wave", item("a new wave of money apps"))).toBe(false); + }); + + it("handles punctuation-bearing names literally", () => { + expect(mentionsEntity("Next.js", item("Choosing Next.js for the frontend"))).toBe(true); + expect(mentionsEntity("Next.js", item("what comes next js-wise"))).toBe(false); + }); + + it("checks the title as well as the snippet", () => { + expect(mentionsEntity("Pocket", { id: "E1", source: "market", title: "Pocket review", ref: "r", score: 1, snippet: "save it" } as EvidenceItem)).toBe(true); + }); +}); + +describe("buildSRD — competitor grounding requires a literal mention (no citation washing)", () => { + const listicle = [ + { + id: "E1", + source: "market", + title: "Best read-later apps", + ref: "https://x/listicle", + url: "https://x/listicle", + score: 2, + snippet: "Pocket lets you save articles for later. A new wave of money apps also bundle reading lists.", + }, + ] as EvidenceItem[]; + const washBrief: Brief = { ...brief, competitors: ["Pocket", "Money Wave"] }; + + it("keeps the citation for a competitor the evidence literally names", () => { + const srd = buildSRD(washBrief, listicle, { level: "light", generatedAt: "T" }); + const pocket = srd.competitive.competitors.find((c) => c.name === "Pocket")!; + expect(pocket.evidence).toEqual(["E1"]); + }); + + it("attaches nothing when the name is only token-overlapped, never mentioned", () => { + const srd = buildSRD(washBrief, listicle, { level: "light", generatedAt: "T" }); + const wave = srd.competitive.competitors.find((c) => c.name === "Money Wave")!; + expect(wave.evidence).toEqual([]); // "wave … money" overlap ≠ a mention of Money Wave + expect(wave.note).toMatch(/Comparable product/); // falls back to the generic note, not E1's digest + }); +}); + describe("buildSRD — integration detection is word-bounded", () => { // A neutral brief so ONLY the injected feature titles drive boundary/integration // detection (idea + candidateTech are emptied out of the haystack). From 02ccb858c3653738e17a3da215add56c02f55165 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:45:52 +0200 Subject: [PATCH 03/10] fix(srd,check): templated acceptance criteria fail at complex; adjectival prefixes never become entities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A complex SRD certifies build-readiness, so acceptance criteria still carrying the renderer's template phrasing now HARD-FAIL check at --level complex (they stay an advisory warning at light) — shipping the un-enriched scaffold was previously a silent exit 0. Entity inference no longer promotes adjectival prefixes: keywords() splits "Multi-currency accounts" at the hyphen and the bare "multi" token recurred its way into a data-model entity literally named "Multi". A prefix stop-set (multi/auto/self/cross/…) drops them so the head noun is what survives. Fixture briefs gain concrete-outcome notes on every feature so the rendered sample SRDs clear the new complex gate. Co-Authored-By: Claude Fable 5 --- scripts/construct.mjs | 36 ++++++++- skills/construct/scripts/construct.mjs | 36 ++++++++- src/check.ts | 12 +-- src/srd.ts | 35 ++++++++- tests/check.test.ts | 44 +++++++++-- tests/fixtures/sample-brief-modules.json | 97 ++++++++++++++++++++---- tests/fixtures/sample-brief.json | 68 ++++++++++++++--- tests/srd.test.ts | 31 ++++++++ 8 files changed, 311 insertions(+), 48 deletions(-) diff --git a/scripts/construct.mjs b/scripts/construct.mjs index e935bdb..b27acb6 100755 --- a/scripts/construct.mjs +++ b/scripts/construct.mjs @@ -1721,6 +1721,34 @@ var FEATURE_VERBS = /* @__PURE__ */ new Set([ "stream" ]); var NON_ENTITY_WORDS = /* @__PURE__ */ new Set(["search", "login", "signup", "support", "setup", "offline", "online", "mobile", "desktop", "full", "text", "user", "users"]); +var ADJECTIVAL_PREFIXES = /* @__PURE__ */ new Set([ + "multi", + "auto", + "self", + "cross", + "pre", + "post", + "non", + "anti", + "semi", + "meta", + "mini", + "micro", + "macro", + "mono", + "dual", + "poly", + "omni", + "pseudo", + "quasi", + "ultra", + "hyper", + "super", + "sub", + "inter", + "intra", + "extra" +]); function singularize(w) { if (/ies$/.test(w)) return w.slice(0, -3) + "y"; if (/(?:ches|shes|xes|zes|ses)$/.test(w)) return w.slice(0, -2); @@ -1734,7 +1762,7 @@ function entityTokens(title, exclude) { const words = keywords(title).map((w) => w.toLowerCase()); const verbLed = words.length > 0 && FEATURE_VERBS.has(words[0]); const rest = verbLed ? words.slice(1) : words; - const tokens = rest.filter((w) => w.length >= 3 && !FEATURE_VERBS.has(w) && !NON_ENTITY_WORDS.has(w) && !/(?:ed|ing)$/.test(w)).map(singularize).filter((w) => !exclude.has(w)); + const tokens = rest.filter((w) => w.length >= 3 && !FEATURE_VERBS.has(w) && !NON_ENTITY_WORDS.has(w) && !ADJECTIVAL_PREFIXES.has(w) && !/(?:ed|ing)$/.test(w)).map(singularize).filter((w) => !exclude.has(w)); return { tokens, verbLed }; } function inferEntities(brief, functional) { @@ -3485,9 +3513,9 @@ function checkRun(runDir, opts = {}) { checkModules(runDir, srd, errors, warnings); const templatedThen = srd.functional.reduce((n, fr) => n + fr.acceptance.filter((a) => TEMPLATED_THEN_RE.test(a.then)).length, 0); if (templatedThen) { - warnings.push( - `${templatedThen} acceptance criteria are still renderer-templated \u2014 sharpen them into observable, bounded outcomes (see references/acceptance-criteria.md).` - ); + const msg = `${templatedThen} acceptance criteria are still renderer-templated \u2014 sharpen them into observable, bounded outcomes (see references/acceptance-criteria.md).`; + if (srd.level === "complex") errors.push(msg); + else warnings.push(msg); } const templatedMetrics = srd.nonFunctional.filter((n) => n.metric && TEMPLATED_METRIC_RE.test(n.metric)).length; if (templatedMetrics) { diff --git a/skills/construct/scripts/construct.mjs b/skills/construct/scripts/construct.mjs index e935bdb..b27acb6 100755 --- a/skills/construct/scripts/construct.mjs +++ b/skills/construct/scripts/construct.mjs @@ -1721,6 +1721,34 @@ var FEATURE_VERBS = /* @__PURE__ */ new Set([ "stream" ]); var NON_ENTITY_WORDS = /* @__PURE__ */ new Set(["search", "login", "signup", "support", "setup", "offline", "online", "mobile", "desktop", "full", "text", "user", "users"]); +var ADJECTIVAL_PREFIXES = /* @__PURE__ */ new Set([ + "multi", + "auto", + "self", + "cross", + "pre", + "post", + "non", + "anti", + "semi", + "meta", + "mini", + "micro", + "macro", + "mono", + "dual", + "poly", + "omni", + "pseudo", + "quasi", + "ultra", + "hyper", + "super", + "sub", + "inter", + "intra", + "extra" +]); function singularize(w) { if (/ies$/.test(w)) return w.slice(0, -3) + "y"; if (/(?:ches|shes|xes|zes|ses)$/.test(w)) return w.slice(0, -2); @@ -1734,7 +1762,7 @@ function entityTokens(title, exclude) { const words = keywords(title).map((w) => w.toLowerCase()); const verbLed = words.length > 0 && FEATURE_VERBS.has(words[0]); const rest = verbLed ? words.slice(1) : words; - const tokens = rest.filter((w) => w.length >= 3 && !FEATURE_VERBS.has(w) && !NON_ENTITY_WORDS.has(w) && !/(?:ed|ing)$/.test(w)).map(singularize).filter((w) => !exclude.has(w)); + const tokens = rest.filter((w) => w.length >= 3 && !FEATURE_VERBS.has(w) && !NON_ENTITY_WORDS.has(w) && !ADJECTIVAL_PREFIXES.has(w) && !/(?:ed|ing)$/.test(w)).map(singularize).filter((w) => !exclude.has(w)); return { tokens, verbLed }; } function inferEntities(brief, functional) { @@ -3485,9 +3513,9 @@ function checkRun(runDir, opts = {}) { checkModules(runDir, srd, errors, warnings); const templatedThen = srd.functional.reduce((n, fr) => n + fr.acceptance.filter((a) => TEMPLATED_THEN_RE.test(a.then)).length, 0); if (templatedThen) { - warnings.push( - `${templatedThen} acceptance criteria are still renderer-templated \u2014 sharpen them into observable, bounded outcomes (see references/acceptance-criteria.md).` - ); + const msg = `${templatedThen} acceptance criteria are still renderer-templated \u2014 sharpen them into observable, bounded outcomes (see references/acceptance-criteria.md).`; + if (srd.level === "complex") errors.push(msg); + else warnings.push(msg); } const templatedMetrics = srd.nonFunctional.filter((n) => n.metric && TEMPLATED_METRIC_RE.test(n.metric)).length; if (templatedMetrics) { diff --git a/src/check.ts b/src/check.ts index c67bef1..6f50a98 100644 --- a/src/check.ts +++ b/src/check.ts @@ -362,13 +362,15 @@ export function checkRun(runDir: string, opts: { minGrounding?: number; semantic // Module partition (only when present) — additive structural gate. checkModules(runDir, srd, errors, warnings); - // Advisory: criteria/metrics still carrying the renderer's own template - // phrasing — complete but not yet sharpened into something testable. + // Criteria/metrics still carrying the renderer's own template phrasing — + // complete but not yet sharpened into something testable. A complex SRD + // certifies build-readiness, so surviving templates HARD-FAIL there; at + // light they stay an advisory nudge. const templatedThen = srd.functional.reduce((n, fr) => n + fr.acceptance.filter((a) => TEMPLATED_THEN_RE.test(a.then)).length, 0); if (templatedThen) { - warnings.push( - `${templatedThen} acceptance criteria are still renderer-templated — sharpen them into observable, bounded outcomes (see references/acceptance-criteria.md).`, - ); + const msg = `${templatedThen} acceptance criteria are still renderer-templated — sharpen them into observable, bounded outcomes (see references/acceptance-criteria.md).`; + if (srd.level === "complex") errors.push(msg); + else warnings.push(msg); } const templatedMetrics = srd.nonFunctional.filter((n) => n.metric && TEMPLATED_METRIC_RE.test(n.metric)).length; if (templatedMetrics) { diff --git a/src/srd.ts b/src/srd.ts index a0c48cb..a2d01e7 100644 --- a/src/srd.ts +++ b/src/srd.ts @@ -229,6 +229,39 @@ const FEATURE_VERBS = new Set([ // Words that name actions or qualities, not data — never entities. const NON_ENTITY_WORDS = new Set(["search", "login", "signup", "support", "setup", "offline", "online", "mobile", "desktop", "full", "text", "user", "users"]); +// Adjectival/compound prefixes ("Multi-currency accounts", "Cross-platform +// sync"): keywords() splits the hyphen, and the bare prefix would otherwise +// recur across features and get promoted into a data-model entity literally +// named "Multi". The head noun ("currency", "account") is the real candidate. +const ADJECTIVAL_PREFIXES = new Set([ + "multi", + "auto", + "self", + "cross", + "pre", + "post", + "non", + "anti", + "semi", + "meta", + "mini", + "micro", + "macro", + "mono", + "dual", + "poly", + "omni", + "pseudo", + "quasi", + "ultra", + "hyper", + "super", + "sub", + "inter", + "intra", + "extra", +]); + function singularize(w: string): string { if (/ies$/.test(w)) return w.slice(0, -3) + "y"; if (/(?:ches|shes|xes|zes|ses)$/.test(w)) return w.slice(0, -2); @@ -248,7 +281,7 @@ function entityTokens(title: string, exclude: Set): { tokens: string[]; const verbLed = words.length > 0 && FEATURE_VERBS.has(words[0]!); const rest = verbLed ? words.slice(1) : words; const tokens = rest - .filter((w) => w.length >= 3 && !FEATURE_VERBS.has(w) && !NON_ENTITY_WORDS.has(w) && !/(?:ed|ing)$/.test(w)) + .filter((w) => w.length >= 3 && !FEATURE_VERBS.has(w) && !NON_ENTITY_WORDS.has(w) && !ADJECTIVAL_PREFIXES.has(w) && !/(?:ed|ing)$/.test(w)) .map(singularize) .filter((w) => !exclude.has(w)); return { tokens, verbLed }; diff --git a/tests/check.test.ts b/tests/check.test.ts index 7cfe09d..17ade9a 100644 --- a/tests/check.test.ts +++ b/tests/check.test.ts @@ -218,12 +218,28 @@ describe("checkRun — opt-in grounding threshold (--min-grounding)", () => { }); }); -describe("checkRun — renderer-templated criteria nudges", () => { - it("warns when acceptance criteria still carry the renderer's template", () => { - // The fixture's "Tag and organize saved articles" has no notes, so its - // positive-path Then stays templated. +describe("checkRun — renderer-templated criteria gate", () => { + it("passes the fixture cleanly — every feature note carries a concrete outcome", () => { const r = checkRun(renderRun()); - expect(r.ok).toBe(true); // advisory only + expect(r.ok).toBe(true); + expect(r.structural.errors.join(" ")).not.toMatch(/renderer-templated/); + expect(r.structural.warnings.join(" ")).not.toMatch(/renderer-templated/); + }); + + it("fails at complex when an acceptance criterion still carries the renderer template", () => { + const dir = renderRun(); // renderRun defaults to complex + mutateSRD(dir, (s) => (s.functional[0]!.acceptance[0]!.then = 'the result of "save an article" is persisted and visible to the user')); + const r = checkRun(dir); + expect(r.ok).toBe(false); + expect(r.structural.errors.join(" ")).toMatch(/renderer-templated/); + expect(r.structural.errors.join(" ")).toMatch(/acceptance-criteria\.md/); + }); + + it("stays an advisory warning at light", () => { + const dir = renderRun({ level: "light" }); + mutateSRD(dir, (s) => (s.functional[0]!.acceptance[0]!.then = 'the result of "save an article" is persisted and visible to the user')); + const r = checkRun(dir); + expect(r.ok).toBe(true); expect(r.structural.warnings.join(" ")).toMatch(/renderer-templated/); }); @@ -232,6 +248,7 @@ describe("checkRun — renderer-templated criteria nudges", () => { mutateSRD(dir, (s) => s.functional.forEach((f) => f.acceptance.forEach((a) => (a.then = "the article appears in the reading list within 2 seconds")))); const r = checkRun(dir); expect(r.structural.warnings.join(" ")).not.toMatch(/renderer-templated/); + expect(r.structural.errors.join(" ")).not.toMatch(/renderer-templated/); }); }); @@ -496,7 +513,13 @@ describe("formatCheckReport", () => { describe("checkRun — placeholder words vs decisions", () => { it("does NOT hard-fail when a feature title legitimately contains TODO (advisory only)", () => { - const r = checkRun(renderRun({ briefOverride: { featureWishlist: [{ title: "Manage a shared TODO list", priority: "must" }] } })); + const r = checkRun( + renderRun({ + briefOverride: { + featureWishlist: [{ title: "Manage a shared TODO list", priority: "must", notes: "so that every teammate sees an added item within 2 seconds" }], + }, + }), + ); expect(r.ok).toBe(true); // TODO in a real title must not fail the build expect(r.structural.warnings.join(" ")).toMatch(/TODO\/TBD\/FIXME/); }); @@ -510,7 +533,14 @@ describe("checkRun — placeholder words vs decisions", () => { it("does NOT hard-fail when a 🧠 glyph appears in a feature title (only the rendered Decide callout counts)", () => { // The 🧠 lands in FUNCTIONAL.md, design/SCREENS.md and TRACEABILITY.md, but // none of those is the renderer's `> 🧠 **Decide:**` callout. - const r = checkRun(renderRun({ level: "complex", briefOverride: { featureWishlist: [{ title: "Capture a 🧠 brainstorm", priority: "must" }] } })); + const r = checkRun( + renderRun({ + level: "complex", + briefOverride: { + featureWishlist: [{ title: "Capture a 🧠 brainstorm", priority: "must", notes: "so that a captured idea is never lost on reload" }], + }, + }), + ); expect(r.ok).toBe(true); expect(r.structural.errors.join(" ")).not.toMatch(/Unresolved decision/); }); diff --git a/tests/fixtures/sample-brief-modules.json b/tests/fixtures/sample-brief-modules.json index 9e9bdec..12a40ce 100644 --- a/tests/fixtures/sample-brief-modules.json +++ b/tests/fixtures/sample-brief-modules.json @@ -4,36 +4,103 @@ "product": { "name": "Readpile", "problem": "People save articles to read later but never find them again; hosted services lock data in and die.", - "users": ["self-hosting individuals", "small teams sharing a knowledge base"], + "users": [ + "self-hosting individuals", + "small teams sharing a knowledge base" + ], "valueProp": "Own your reading queue: a private, fast, searchable archive of everything you save." }, "goals": [ "A user can save and reliably re-find any article within seconds", "Reach 1,000 self-hosted installs in the first year" ], - "nonGoals": ["A hosted multi-tenant SaaS offering", "A social/sharing network"], + "nonGoals": [ + "A hosted multi-tenant SaaS offering", + "A social/sharing network" + ], "constraints": { "budget": "side-project, near-zero infra budget", "timeline": "MVP in 8 weeks", "team": "one full-stack developer", - "compliance": ["GDPR (self-hosted, user owns data)"] + "compliance": [ + "GDPR (self-hosted, user owns data)" + ] }, - "candidateTech": ["PostgreSQL", "Meilisearch", "Next.js"], - "competitors": ["Pocket", "Instapaper", "Omnivore"], - "ossSeeds": ["https://github.com/omnivore-app/omnivore", "https://github.com/wallabag/wallabag"], + "candidateTech": [ + "PostgreSQL", + "Meilisearch", + "Next.js" + ], + "competitors": [ + "Pocket", + "Instapaper", + "Omnivore" + ], + "ossSeeds": [ + "https://github.com/omnivore-app/omnivore", + "https://github.com/wallabag/wallabag" + ], "modules": [ - { "id": "capture", "name": "Capture", "description": "Getting content into the archive: saving, extraction, imports." }, - { "id": "search", "name": "Search", "description": "Finding saved content again, fast.", "dependsOn": ["capture"] }, - { "id": "library", "name": "Library", "description": "Organizing and reading the archive.", "dependsOn": ["capture"] } + { + "id": "capture", + "name": "Capture", + "description": "Getting content into the archive: saving, extraction, imports." + }, + { + "id": "search", + "name": "Search", + "description": "Finding saved content again, fast.", + "dependsOn": [ + "capture" + ] + }, + { + "id": "library", + "name": "Library", + "description": "Organizing and reading the archive.", + "dependsOn": [ + "capture" + ] + } ], "featureWishlist": [ - { "title": "Save an article from a URL or browser extension", "priority": "must", "notes": "Extract clean readable content and store it offline.", "module": "capture" }, - { "title": "Full-text search across all saved articles", "priority": "must", "notes": "Sub-second search with typo tolerance.", "module": "search" }, - { "title": "Tag and organize saved articles", "priority": "should", "module": "library" }, - { "title": "Import an existing Pocket or Instapaper export", "priority": "should", "module": "capture" }, - { "title": "Read articles offline on mobile", "priority": "could", "module": "library" } + { + "title": "Save an article from a URL or browser extension", + "priority": "must", + "notes": "Extract clean readable content and store it offline, so that the saved copy renders without the original site being reachable.", + "module": "capture" + }, + { + "title": "Full-text search across all saved articles", + "priority": "must", + "notes": "Sub-second search with typo tolerance, so that a reader re-finds any saved article in under 1 second.", + "module": "search" + }, + { + "title": "Tag and organize saved articles", + "priority": "should", + "module": "library", + "notes": "Tags are freeform and user-owned, so that a user re-finds any article by tag in one search." + }, + { + "title": "Import an existing Pocket or Instapaper export", + "priority": "should", + "module": "capture", + "notes": "Parse the official export formats, so that an import never loses an article's original save date or tags." + }, + { + "title": "Read articles offline on mobile", + "priority": "could", + "module": "library", + "notes": "Cache saved articles locally, so that previously synced articles open without a network connection." + } + ], + "nfrPriorities": [ + "performance", + "security", + "reliability", + "privacy" ], - "nfrPriorities": ["performance", "security", "reliability", "privacy"], "openQuestions": [], "createdAt": "2026-06-08T00:00:00.000Z" } diff --git a/tests/fixtures/sample-brief.json b/tests/fixtures/sample-brief.json index 653cf69..4d965e4 100644 --- a/tests/fixtures/sample-brief.json +++ b/tests/fixtures/sample-brief.json @@ -4,31 +4,75 @@ "product": { "name": "Readpile", "problem": "People save articles to read later but never find them again; hosted services lock data in and die.", - "users": ["self-hosting individuals", "small teams sharing a knowledge base"], + "users": [ + "self-hosting individuals", + "small teams sharing a knowledge base" + ], "valueProp": "Own your reading queue: a private, fast, searchable archive of everything you save." }, "goals": [ "A user can save and reliably re-find any article within seconds", "Reach 1,000 self-hosted installs in the first year" ], - "nonGoals": ["A hosted multi-tenant SaaS offering", "A social/sharing network"], + "nonGoals": [ + "A hosted multi-tenant SaaS offering", + "A social/sharing network" + ], "constraints": { "budget": "side-project, near-zero infra budget", "timeline": "MVP in 8 weeks", "team": "one full-stack developer", - "compliance": ["GDPR (self-hosted, user owns data)"] + "compliance": [ + "GDPR (self-hosted, user owns data)" + ] }, - "candidateTech": ["PostgreSQL", "Meilisearch", "Next.js"], - "competitors": ["Pocket", "Instapaper", "Omnivore"], - "ossSeeds": ["https://github.com/omnivore-app/omnivore", "https://github.com/wallabag/wallabag"], + "candidateTech": [ + "PostgreSQL", + "Meilisearch", + "Next.js" + ], + "competitors": [ + "Pocket", + "Instapaper", + "Omnivore" + ], + "ossSeeds": [ + "https://github.com/omnivore-app/omnivore", + "https://github.com/wallabag/wallabag" + ], "featureWishlist": [ - { "title": "Save an article from a URL or browser extension", "priority": "must", "notes": "Extract clean readable content and store it offline." }, - { "title": "Full-text search across all saved articles", "priority": "must", "notes": "Sub-second search with typo tolerance." }, - { "title": "Tag and organize saved articles", "priority": "should" }, - { "title": "Import an existing Pocket or Instapaper export", "priority": "should" }, - { "title": "Read articles offline on mobile", "priority": "could" } + { + "title": "Save an article from a URL or browser extension", + "priority": "must", + "notes": "Extract clean readable content and store it offline, so that the saved copy renders without the original site being reachable." + }, + { + "title": "Full-text search across all saved articles", + "priority": "must", + "notes": "Sub-second search with typo tolerance, so that a reader re-finds any saved article in under 1 second." + }, + { + "title": "Tag and organize saved articles", + "priority": "should", + "notes": "Tags are freeform and user-owned, so that a user re-finds any article by tag in one search." + }, + { + "title": "Import an existing Pocket or Instapaper export", + "priority": "should", + "notes": "Parse the official export formats, so that an import never loses an article's original save date or tags." + }, + { + "title": "Read articles offline on mobile", + "priority": "could", + "notes": "Cache saved articles locally, so that previously synced articles open without a network connection." + } + ], + "nfrPriorities": [ + "performance", + "security", + "reliability", + "privacy" ], - "nfrPriorities": ["performance", "security", "reliability", "privacy"], "openQuestions": [], "createdAt": "2026-06-08T00:00:00.000Z" } diff --git a/tests/srd.test.ts b/tests/srd.test.ts index 4c52c8c..1a3ff35 100644 --- a/tests/srd.test.ts +++ b/tests/srd.test.ts @@ -81,6 +81,37 @@ describe("mentionsEntity", () => { }); }); +describe("inferEntities — adjectival prefixes never become entities", () => { + const clean = (features: Brief["featureWishlist"]): Brief => ({ + ...brief, + idea: "a simple app", + candidateTech: [], + competitors: [], + featureWishlist: features, + }); + + it("does not promote 'Multi' from recurring multi-* feature titles", () => { + const srd = buildSRD( + clean([ + { title: "Multi-currency accounts", priority: "must" }, + { title: "Multi-currency transfers", priority: "should" }, + ]), + [], + { level: "light", generatedAt: "T" }, + ); + const names = srd.architecture.dataModel.map((e) => e.name); + expect(names).not.toContain("Multi"); + expect(names).toContain("Currency"); // the real recurring noun survives + }); + + it("skips the prefix when picking a verb-led must-have's direct object", () => { + const srd = buildSRD(clean([{ title: "Create multi-currency wallets", priority: "must" }]), [], { level: "light", generatedAt: "T" }); + const names = srd.architecture.dataModel.map((e) => e.name); + expect(names).not.toContain("Multi"); + expect(names.length).toBeGreaterThan(0); // the head noun, not the prefix, is the object + }); +}); + describe("buildSRD — competitor grounding requires a literal mention (no citation washing)", () => { const listicle = [ { From 23657078006b5fdeffbe2b1f336ad105e85610fe Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:48:17 +0200 Subject: [PATCH 04/10] feat(review): adjudicate every cited pair by default; --max-review caps explicitly and loudly The worklist's silent default cap of 40 pairs, ranked by raw retrieval score, dropped exactly the highest-value low-score evidence (issue/PR citations) from adjudication. review now emits EVERY cited pair by default; --max-review N still caps explicitly, and a capped VERIFY.md opens with a DROPPED block naming each excluded pair so the omission can never read as full coverage. Co-Authored-By: Claude Fable 5 --- scripts/construct.mjs | 30 ++++++++++++++------ skills/construct/references/orchestration.md | 13 +++++---- skills/construct/scripts/construct.mjs | 30 ++++++++++++++------ src/cli.ts | 12 ++++++-- src/review.ts | 29 ++++++++++++------- tests/semantic-review.test.ts | 28 ++++++++++++++++++ 6 files changed, 105 insertions(+), 37 deletions(-) diff --git a/scripts/construct.mjs b/scripts/construct.mjs index b27acb6..e2accc3 100755 --- a/scripts/construct.mjs +++ b/scripts/construct.mjs @@ -3035,7 +3035,6 @@ import { join as join11, relative as relative2, sep as sep2 } from "path"; // src/review.ts import { existsSync as existsSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync5 } from "fs"; import { join as join10 } from "path"; -var REVIEW_MAX = 40; var VALID_VERDICTS = ["supported", "partial", "refuted", "unsupported"]; function loadEvidence(path) { if (!existsSync5(path)) return []; @@ -3109,26 +3108,33 @@ function runReview(runDir, opts = {}) { }); } } - const max = Math.max(1, Math.floor(opts.maxReview ?? REVIEW_MAX)); - const kept = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)).slice(0, max) : pairs; + const max = opts.maxReview === void 0 ? Number.POSITIVE_INFINITY : Math.max(1, Math.floor(opts.maxReview)); + const sorted = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)) : pairs; + const kept = sorted.slice(0, Math.min(sorted.length, max)); + const dropped = sorted.slice(kept.length); const worklist = { run: runDir, pairs: kept.map(({ score, ...rest }) => rest) }; const todo = { run: runDir, pairs: worklist.pairs.map((p) => ({ ...p, verdict: null, note: "" })) }; writeFileSync5(join10(runDir, "VERIFY.todo.json"), JSON.stringify(todo, null, 2)); - writeFileSync5(join10(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, kept.length)); + writeFileSync5(join10(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, dropped)); return worklist; } -function renderWorklistMd(wl, total, kept) { +function renderWorklistMd(wl, total, dropped) { const out = []; out.push(`# Claim-support review worklist`); out.push(""); out.push( `For each pair, open the cited evidence and judge whether it **supports** the claim. In \`VERIFY.todo.json\`, set each \`verdict\` to one of supported \xB7 partial \xB7 refuted \xB7 unsupported, add a short \`note\`, save it (e.g. as \`verdicts.json\`), then run \`construct review --apply verdicts.json --out \`.` ); - if (kept < total) out.push(` -_Showing ${kept} of ${total} pair(s) \u2014 capped at the highest-score evidence._`); + if (dropped.length) { + out.push(""); + out.push(`> **DROPPED (--max-review): ${dropped.length} of ${total} pair(s) are NOT in this worklist and will NOT be adjudicated.**`); + out.push(`> Their claims can pass \`check --semantic\` without their evidence ever being judged.`); + out.push(`> Re-run \`construct review\` without --max-review to review everything. Dropped:`); + for (const d of dropped) out.push(`> - ${d.claimId} \xB7 ${d.evidenceId} (${d.source})`); + } out.push(""); for (const p of wl.pairs) { out.push(`## ${p.claimId} \xB7 ${p.evidenceId} (${p.source})`); @@ -3943,6 +3949,9 @@ Options: --allow-unverified For 'check --semantic': degrade a missing/unreadable VERIFY.json to a warning instead of failing --apply For 'review': consume an adjudicated verdicts file + gate + --max-review For 'review': cap the worklist at the n highest-score + pairs (default: review ALL cited pairs; dropped pairs + are named in VERIFY.md) --app For 'verify': the built app directory (default: conventions.appDir) --run-tests For 'verify': also execute testCommand + per-task verify commands --strict For 'verify': a built must-have FR with no referencing test FAILS @@ -4250,8 +4259,11 @@ ${v.errors.map((e) => " - " + e).join("\n")}`); if (!res.ok) process.exit(1); return; } - const maxReview = p.values["max-review"] ? Number(p.values["max-review"]) : REVIEW_MAX; - if (!Number.isFinite(maxReview) || maxReview <= 0) fail("invalid --max-review"); + let maxReview; + if (p.values["max-review"] !== void 0) { + maxReview = Number(p.values["max-review"]); + if (p.values["max-review"].trim() === "" || !Number.isFinite(maxReview) || maxReview <= 0) fail("invalid --max-review"); + } const wl = runReview(out, { maxReview }); if (p.bools.has("json")) { process.stdout.write(JSON.stringify(wl, null, 2) + "\n"); diff --git a/skills/construct/references/orchestration.md b/skills/construct/references/orchestration.md index 97efceb..9cb9422 100644 --- a/skills/construct/references/orchestration.md +++ b/skills/construct/references/orchestration.md @@ -126,8 +126,9 @@ serial reduce (merge fragments → `review --apply`) → deterministic gate (`check --semantic`).* `construct review --out ` mechanises grounding adjudication: it pairs every -grounded SRD claim with each cited `[E#]` item's snippet (capped at -`--max-review`, default 40 highest-score) and writes the worklist to +grounded SRD claim with each cited `[E#]` item's snippet (EVERY cited pair by +default; `--max-review N` caps at the N highest-score pairs and names the +dropped ones in VERIFY.md) and writes the worklist to `VERIFY.todo.json` + `VERIFY.md`. **Each pair is independent** — the ideal fan-out unit, and the engine already accepts the sharded-then-merged shape. @@ -210,9 +211,11 @@ the failure modes live in `references/build-playbook.md`. evidence conflicts, or the decision is hard to reverse). Hard cap: ≤2 panels per SRD; if a third ADR seems contested, take it to the user as a question instead. -- Claim-support (pattern 4): cheap per branch; cap = `--max-review` (default - 40). Batch pairs to keep fan-out modest. Always worth one pass over the - load-bearing FRs/ADRs before presenting. +- Claim-support (pattern 4): cheap per branch; every cited pair is in the + worklist by default (`--max-review N` caps explicitly — the dropped pairs + are named in VERIFY.md and stay unadjudicated). Batch pairs to keep fan-out + modest. Always worth one pass over the load-bearing FRs/ADRs before + presenting. - Build (pattern 5): one agent per ready task, bounded by the frontier width minus the shared-file tasks you serialise. Speedup only — never run the milestone gate (`verify --run-tests --strict`) until the whole frontier is diff --git a/skills/construct/scripts/construct.mjs b/skills/construct/scripts/construct.mjs index b27acb6..e2accc3 100755 --- a/skills/construct/scripts/construct.mjs +++ b/skills/construct/scripts/construct.mjs @@ -3035,7 +3035,6 @@ import { join as join11, relative as relative2, sep as sep2 } from "path"; // src/review.ts import { existsSync as existsSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync5 } from "fs"; import { join as join10 } from "path"; -var REVIEW_MAX = 40; var VALID_VERDICTS = ["supported", "partial", "refuted", "unsupported"]; function loadEvidence(path) { if (!existsSync5(path)) return []; @@ -3109,26 +3108,33 @@ function runReview(runDir, opts = {}) { }); } } - const max = Math.max(1, Math.floor(opts.maxReview ?? REVIEW_MAX)); - const kept = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)).slice(0, max) : pairs; + const max = opts.maxReview === void 0 ? Number.POSITIVE_INFINITY : Math.max(1, Math.floor(opts.maxReview)); + const sorted = pairs.length > max ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)) : pairs; + const kept = sorted.slice(0, Math.min(sorted.length, max)); + const dropped = sorted.slice(kept.length); const worklist = { run: runDir, pairs: kept.map(({ score, ...rest }) => rest) }; const todo = { run: runDir, pairs: worklist.pairs.map((p) => ({ ...p, verdict: null, note: "" })) }; writeFileSync5(join10(runDir, "VERIFY.todo.json"), JSON.stringify(todo, null, 2)); - writeFileSync5(join10(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, kept.length)); + writeFileSync5(join10(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, dropped)); return worklist; } -function renderWorklistMd(wl, total, kept) { +function renderWorklistMd(wl, total, dropped) { const out = []; out.push(`# Claim-support review worklist`); out.push(""); out.push( `For each pair, open the cited evidence and judge whether it **supports** the claim. In \`VERIFY.todo.json\`, set each \`verdict\` to one of supported \xB7 partial \xB7 refuted \xB7 unsupported, add a short \`note\`, save it (e.g. as \`verdicts.json\`), then run \`construct review --apply verdicts.json --out \`.` ); - if (kept < total) out.push(` -_Showing ${kept} of ${total} pair(s) \u2014 capped at the highest-score evidence._`); + if (dropped.length) { + out.push(""); + out.push(`> **DROPPED (--max-review): ${dropped.length} of ${total} pair(s) are NOT in this worklist and will NOT be adjudicated.**`); + out.push(`> Their claims can pass \`check --semantic\` without their evidence ever being judged.`); + out.push(`> Re-run \`construct review\` without --max-review to review everything. Dropped:`); + for (const d of dropped) out.push(`> - ${d.claimId} \xB7 ${d.evidenceId} (${d.source})`); + } out.push(""); for (const p of wl.pairs) { out.push(`## ${p.claimId} \xB7 ${p.evidenceId} (${p.source})`); @@ -3943,6 +3949,9 @@ Options: --allow-unverified For 'check --semantic': degrade a missing/unreadable VERIFY.json to a warning instead of failing --apply For 'review': consume an adjudicated verdicts file + gate + --max-review For 'review': cap the worklist at the n highest-score + pairs (default: review ALL cited pairs; dropped pairs + are named in VERIFY.md) --app For 'verify': the built app directory (default: conventions.appDir) --run-tests For 'verify': also execute testCommand + per-task verify commands --strict For 'verify': a built must-have FR with no referencing test FAILS @@ -4250,8 +4259,11 @@ ${v.errors.map((e) => " - " + e).join("\n")}`); if (!res.ok) process.exit(1); return; } - const maxReview = p.values["max-review"] ? Number(p.values["max-review"]) : REVIEW_MAX; - if (!Number.isFinite(maxReview) || maxReview <= 0) fail("invalid --max-review"); + let maxReview; + if (p.values["max-review"] !== void 0) { + maxReview = Number(p.values["max-review"]); + if (p.values["max-review"].trim() === "" || !Number.isFinite(maxReview) || maxReview <= 0) fail("invalid --max-review"); + } const wl = runReview(out, { maxReview }); if (p.bools.has("json")) { process.stdout.write(JSON.stringify(wl, null, 2) + "\n"); diff --git a/src/cli.ts b/src/cli.ts index 0ab2baa..137ba22 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -17,7 +17,7 @@ import { renderSRD } from "./render.js"; import { checkRun, formatCheckReport } from "./check.js"; import { analyzeRun, formatGapReport } from "./analyze.js"; import { verifyRun, formatVerifyReport } from "./verify.js"; -import { runReview, applyVerdicts, formatReviewReport, REVIEW_MAX } from "./review.js"; +import { runReview, applyVerdicts, formatReviewReport } from "./review.js"; import { loadPlan, readyFrontier } from "./plan.js"; import { semanticControl } from "./research/semantic.js"; @@ -76,6 +76,9 @@ Options: --allow-unverified For 'check --semantic': degrade a missing/unreadable VERIFY.json to a warning instead of failing --apply For 'review': consume an adjudicated verdicts file + gate + --max-review For 'review': cap the worklist at the n highest-score + pairs (default: review ALL cited pairs; dropped pairs + are named in VERIFY.md) --app For 'verify': the built app directory (default: conventions.appDir) --run-tests For 'verify': also execute testCommand + per-task verify commands --strict For 'verify': a built must-have FR with no referencing test FAILS @@ -419,8 +422,11 @@ async function main(): Promise { if (!res.ok) process.exit(1); return; } - const maxReview = p.values["max-review"] ? Number(p.values["max-review"]) : REVIEW_MAX; - if (!Number.isFinite(maxReview) || maxReview <= 0) fail("invalid --max-review"); + let maxReview: number | undefined; + if (p.values["max-review"] !== undefined) { + maxReview = Number(p.values["max-review"]); + if (p.values["max-review"].trim() === "" || !Number.isFinite(maxReview) || maxReview <= 0) fail("invalid --max-review"); + } const wl = runReview(out, { maxReview }); if (p.bools.has("json")) { process.stdout.write(JSON.stringify(wl, null, 2) + "\n"); diff --git a/src/review.ts b/src/review.ts index 0ff7fd4..76cff24 100644 --- a/src/review.ts +++ b/src/review.ts @@ -4,8 +4,6 @@ import { keywords } from "./util.js"; import { srdManifestPath } from "./srd.js"; import type { ClaimEvidencePair, ClaimVerdict, ClaimVerifyResult, EvidenceItem, SRD, VerdictKind } from "./types.js"; -// Bounds the review loop (claim↔evidence pairs adjudicated per run). -export const REVIEW_MAX = 40; const VALID_VERDICTS: VerdictKind[] = ["supported", "partial", "refuted", "unsupported"]; export interface ReviewWorklist { @@ -111,14 +109,17 @@ export function runReview(runDir: string, opts: { maxReview?: number } = {}): Re } } - const max = Math.max(1, Math.floor(opts.maxReview ?? REVIEW_MAX)); - const kept = + // Every cited pair is adjudicated by default — a silent cap ranked by raw + // retrieval score used to drop exactly the highest-value low-score evidence + // (issue/PR citations). `--max-review N` still caps explicitly, and then the + // dropped pairs are named in VERIFY.md so the omission is impossible to miss. + const max = opts.maxReview === undefined ? Number.POSITIVE_INFINITY : Math.max(1, Math.floor(opts.maxReview)); + const sorted = pairs.length > max - ? pairs - .slice() - .sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)) - .slice(0, max) + ? pairs.slice().sort((a, b) => b.score - a.score || a.claimId.localeCompare(b.claimId) || a.evidenceId.localeCompare(b.evidenceId)) : pairs; + const kept = sorted.slice(0, Math.min(sorted.length, max)); + const dropped = sorted.slice(kept.length); const worklist: ReviewWorklist = { run: runDir, pairs: kept.map(({ score, ...rest }) => rest) }; const todo = { @@ -126,11 +127,11 @@ export function runReview(runDir: string, opts: { maxReview?: number } = {}): Re pairs: worklist.pairs.map((p) => ({ ...p, verdict: null as VerdictKind | null, note: "" })), }; writeFileSync(join(runDir, "VERIFY.todo.json"), JSON.stringify(todo, null, 2)); - writeFileSync(join(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, kept.length)); + writeFileSync(join(runDir, "VERIFY.md"), renderWorklistMd(worklist, pairs.length, dropped)); return worklist; } -function renderWorklistMd(wl: ReviewWorklist, total: number, kept: number): string { +function renderWorklistMd(wl: ReviewWorklist, total: number, dropped: (ClaimEvidencePair & { score: number })[]): string { const out: string[] = []; out.push(`# Claim-support review worklist`); out.push(""); @@ -140,7 +141,13 @@ function renderWorklistMd(wl: ReviewWorklist, total: number, kept: number): stri `add a short \`note\`, save it (e.g. as \`verdicts.json\`), then run ` + `\`construct review --apply verdicts.json --out \`.`, ); - if (kept < total) out.push(`\n_Showing ${kept} of ${total} pair(s) — capped at the highest-score evidence._`); + if (dropped.length) { + out.push(""); + out.push(`> **DROPPED (--max-review): ${dropped.length} of ${total} pair(s) are NOT in this worklist and will NOT be adjudicated.**`); + out.push(`> Their claims can pass \`check --semantic\` without their evidence ever being judged.`); + out.push(`> Re-run \`construct review\` without --max-review to review everything. Dropped:`); + for (const d of dropped) out.push(`> - ${d.claimId} · ${d.evidenceId} (${d.source})`); + } out.push(""); for (const p of wl.pairs) { out.push(`## ${p.claimId} · ${p.evidenceId} (${p.source})`); diff --git a/tests/semantic-review.test.ts b/tests/semantic-review.test.ts index 09fb453..cb8095a 100644 --- a/tests/semantic-review.test.ts +++ b/tests/semantic-review.test.ts @@ -106,6 +106,34 @@ describe("runReview (worklist)", () => { expect(r.pairs.length).toBe(1); rmSync(dir, { recursive: true, force: true }); }); + + it("reviews EVERY cited pair by default — no implicit 40-pair cap", () => { + const dir = scratch(); + const many = Array.from({ length: 45 }, (_, i) => ({ id: `FR-${String(i + 1).padStart(3, "0")}`, ev: ["E1"] })); + run(dir, many, EVIDENCE); + const r = runReview(dir); + expect(r.pairs.length).toBe(45); // the old default silently kept the top-40 by score + rmSync(dir, { recursive: true, force: true }); + }); + + it("an explicit cap names the DROPPED pairs loudly in VERIFY.md", () => { + const dir = scratch(); + run( + dir, + [ + { id: "FR-001", ev: ["E1"] }, + { id: "FR-002", ev: ["E2"] }, + ], + EVIDENCE, + ); + runReview(dir, { maxReview: 1 }); + const md = readFileSync(join(dir, "VERIFY.md"), "utf8"); + expect(md).toMatch(/DROPPED/); + expect(md).toMatch(/FR-002 · E2/); // E2 has the lower score → dropped + expect(md).toMatch(/NOT .*adjudicated/i); + expect(md).toMatch(/without --max-review/); + rmSync(dir, { recursive: true, force: true }); + }); }); describe("applyVerdicts (semantic gate)", () => { From c0149d7d90c3e336aae1df6c7e23eaec27f65e3b Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:50:55 +0200 Subject: [PATCH 05/10] fix(brief): coerce bare-string array fields and name unknown constraints keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bare string in an array-typed brief field (product.users, competitors, …) was warned-and-dropped, silently discarding the user's real value; it is now coerced into a one-element array with a warning. An unrecognized constraints key (e.g. constraints.deployment) vanished without a trace; normalizeConstraints now names the offender and lists the four recognized keys. The interview playbook documents the array-typed fields and the constraints contract. Co-Authored-By: Claude Fable 5 --- scripts/construct.mjs | 31 ++++++++++--- .../references/interview-playbook.md | 17 ++++++++ skills/construct/scripts/construct.mjs | 31 ++++++++++--- src/brief.ts | 43 ++++++++++++++++--- tests/brief.test.ts | 31 +++++++++++-- 5 files changed, 132 insertions(+), 21 deletions(-) diff --git a/scripts/construct.mjs b/scripts/construct.mjs index e2accc3..7371cdf 100755 --- a/scripts/construct.mjs +++ b/scripts/construct.mjs @@ -248,6 +248,25 @@ var PRIORITIES = ["must", "should", "could"]; function slugId(s) { return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60); } +var KNOWN_CONSTRAINT_KEYS = ["budget", "timeline", "team", "compliance"]; +function normalizeConstraints(raw, line, arr, warn) { + const c = raw ?? {}; + if (raw && typeof raw === "object" && !Array.isArray(raw)) { + for (const k of Object.keys(c)) { + if (!KNOWN_CONSTRAINT_KEYS.includes(k)) { + warn( + `constraints.${k} is not a recognized constraint (known: ${KNOWN_CONSTRAINT_KEYS.join(", ")}) \u2014 ignored; fold it into the nearest field or openQuestions.` + ); + } + } + } + return { + budget: line(c.budget), + timeline: line(c.timeline), + team: line(c.team), + compliance: arr(c.compliance, "constraints.compliance") + }; +} function normalizeBrief(data, warn = () => { }) { const d = data ?? {}; @@ -255,6 +274,11 @@ function normalizeBrief(data, warn = () => { const arr = (v, field) => { if (v === void 0 || v === null) return []; if (!Array.isArray(v)) { + if (typeof v === "string") { + const s = v.replace(/\s+/g, " ").trim(); + warn(`${field}: expected an array \u2014 coerced the bare string into a one-element array.`); + return s ? [s] : []; + } warn(`${field} is not an array \u2014 ignored.`); return []; } @@ -365,12 +389,7 @@ function normalizeBrief(data, warn = () => { }, goals: arr(d.goals, "goals"), nonGoals: arr(d.nonGoals, "nonGoals"), - constraints: { - budget: line(d.constraints?.budget), - timeline: line(d.constraints?.timeline), - team: line(d.constraints?.team), - compliance: arr(d.constraints?.compliance, "constraints.compliance") - }, + constraints: normalizeConstraints(d.constraints, line, arr, warn), candidateTech: arr(d.candidateTech, "candidateTech"), competitors: arr(d.competitors, "competitors"), ossSeeds: arr(d.ossSeeds, "ossSeeds"), diff --git a/skills/construct/references/interview-playbook.md b/skills/construct/references/interview-playbook.md index dcd8cf2..316c99f 100644 --- a/skills/construct/references/interview-playbook.md +++ b/skills/construct/references/interview-playbook.md @@ -69,6 +69,23 @@ Check the fit before question 1; a wrong fit wastes the whole loop. These render as `🧠 Decide:` callouts and **block the structural gate** until resolved — so only put real, deferred decisions here. +## Field types + +Several brief fields are **arrays of strings** — write them as JSON arrays, not +prose: + +- `product.users`, `goals`, `nonGoals`, `candidateTech`, `competitors`, + `ossSeeds`, `nfrPriorities`, `openQuestions`, +- `constraints.compliance`, `design.platforms`, `design.referenceSystems`. + +A bare string in one of these is **coerced** into a one-element array with a +warning (so nothing is lost), but authoring the array directly is clearer: +`"users": ["solo devs", "small teams"]`, not `"users": "solo devs and teams"`. + +`constraints` reads **only** `budget`, `timeline`, `team`, `compliance` — any +other key (e.g. `constraints.deployment`) is dropped with a named warning. Fold +a stray constraint into the nearest recognized field or into `openQuestions`. + ## Tips - Prefer multiple-choice or "A or B?" phrasing; it's easier to answer. diff --git a/skills/construct/scripts/construct.mjs b/skills/construct/scripts/construct.mjs index e2accc3..7371cdf 100755 --- a/skills/construct/scripts/construct.mjs +++ b/skills/construct/scripts/construct.mjs @@ -248,6 +248,25 @@ var PRIORITIES = ["must", "should", "could"]; function slugId(s) { return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60); } +var KNOWN_CONSTRAINT_KEYS = ["budget", "timeline", "team", "compliance"]; +function normalizeConstraints(raw, line, arr, warn) { + const c = raw ?? {}; + if (raw && typeof raw === "object" && !Array.isArray(raw)) { + for (const k of Object.keys(c)) { + if (!KNOWN_CONSTRAINT_KEYS.includes(k)) { + warn( + `constraints.${k} is not a recognized constraint (known: ${KNOWN_CONSTRAINT_KEYS.join(", ")}) \u2014 ignored; fold it into the nearest field or openQuestions.` + ); + } + } + } + return { + budget: line(c.budget), + timeline: line(c.timeline), + team: line(c.team), + compliance: arr(c.compliance, "constraints.compliance") + }; +} function normalizeBrief(data, warn = () => { }) { const d = data ?? {}; @@ -255,6 +274,11 @@ function normalizeBrief(data, warn = () => { const arr = (v, field) => { if (v === void 0 || v === null) return []; if (!Array.isArray(v)) { + if (typeof v === "string") { + const s = v.replace(/\s+/g, " ").trim(); + warn(`${field}: expected an array \u2014 coerced the bare string into a one-element array.`); + return s ? [s] : []; + } warn(`${field} is not an array \u2014 ignored.`); return []; } @@ -365,12 +389,7 @@ function normalizeBrief(data, warn = () => { }, goals: arr(d.goals, "goals"), nonGoals: arr(d.nonGoals, "nonGoals"), - constraints: { - budget: line(d.constraints?.budget), - timeline: line(d.constraints?.timeline), - team: line(d.constraints?.team), - compliance: arr(d.constraints?.compliance, "constraints.compliance") - }, + constraints: normalizeConstraints(d.constraints, line, arr, warn), candidateTech: arr(d.candidateTech, "candidateTech"), competitors: arr(d.competitors, "competitors"), ossSeeds: arr(d.ossSeeds, "ossSeeds"), diff --git a/src/brief.ts b/src/brief.ts index de90eb9..0f4f29f 100644 --- a/src/brief.ts +++ b/src/brief.ts @@ -65,6 +65,35 @@ function slugId(s: string): string { .slice(0, 60); } +const KNOWN_CONSTRAINT_KEYS = ["budget", "timeline", "team", "compliance"] as const; + +// Rebuild the constraints object from only the four recognized keys, and warn — +// naming the offender — when the brief carries any other key (a common mistake, +// e.g. `constraints.deployment`, that would otherwise vanish silently). +function normalizeConstraints( + raw: Brief["constraints"] | undefined, + line: (v: unknown) => string | undefined, + arr: (v: unknown, field: string) => string[], + warn: (msg: string) => void, +): Brief["constraints"] { + const c = (raw ?? {}) as Record; + if (raw && typeof raw === "object" && !Array.isArray(raw)) { + for (const k of Object.keys(c)) { + if (!(KNOWN_CONSTRAINT_KEYS as readonly string[]).includes(k)) { + warn( + `constraints.${k} is not a recognized constraint (known: ${KNOWN_CONSTRAINT_KEYS.join(", ")}) — ignored; fold it into the nearest field or openQuestions.`, + ); + } + } + } + return { + budget: line(c.budget), + timeline: line(c.timeline), + team: line(c.team), + compliance: arr(c.compliance, "constraints.compliance"), + }; +} + // Coerce a parsed object into a Brief, tolerating missing arrays/objects so a // hand-edited brief never crashes the renderer. Tolerance must not mean silent // loss: anything dropped or rewritten is reported through `warn`. @@ -76,6 +105,13 @@ export function normalizeBrief(data: unknown, warn: (msg: string) => void = () = const arr = (v: unknown, field: string): string[] => { if (v === undefined || v === null) return []; if (!Array.isArray(v)) { + // A bare string is a common brief mistake — coerce it into a one-element + // array (with a warning) rather than dropping the user's real value. + if (typeof v === "string") { + const s = v.replace(/\s+/g, " ").trim(); + warn(`${field}: expected an array — coerced the bare string into a one-element array.`); + return s ? [s] : []; + } warn(`${field} is not an array — ignored.`); return []; } @@ -194,12 +230,7 @@ export function normalizeBrief(data: unknown, warn: (msg: string) => void = () = }, goals: arr(d.goals, "goals"), nonGoals: arr(d.nonGoals, "nonGoals"), - constraints: { - budget: line(d.constraints?.budget), - timeline: line(d.constraints?.timeline), - team: line(d.constraints?.team), - compliance: arr(d.constraints?.compliance, "constraints.compliance"), - }, + constraints: normalizeConstraints(d.constraints, line, arr, warn), candidateTech: arr(d.candidateTech, "candidateTech"), competitors: arr(d.competitors, "competitors"), ossSeeds: arr(d.ossSeeds, "ossSeeds"), diff --git a/tests/brief.test.ts b/tests/brief.test.ts index cc89fef..ca1d9f0 100644 --- a/tests/brief.test.ts +++ b/tests/brief.test.ts @@ -58,7 +58,7 @@ describe("normalizeBrief", () => { idea: "y", goals: ["ok", 42], featureWishlist: [{ title: "Real feature" }, { notes: "no title" }, { title: "Hot one", priority: "high" }], - competitors: "not-an-array", + nfrPriorities: { not: "an array or string" }, }, (w) => warnings.push(w), ); @@ -67,14 +67,39 @@ describe("normalizeBrief", () => { "goals: dropped 1 non-string/empty entry.", "featureWishlist[1] has no usable title — dropped.", 'featureWishlist[2].priority "high" is not must|should|could — treated as should.', - "competitors is not an array — ignored.", + "nfrPriorities is not an array — ignored.", ].sort(), ); // The returned shape is the same as before — tolerant, never crashing. expect(b.goals).toEqual(["ok"]); expect(b.featureWishlist.map((f) => f.title)).toEqual(["Real feature", "Hot one"]); expect(b.featureWishlist[1]!.priority).toBeUndefined(); - expect(b.competitors).toEqual([]); + expect(b.nfrPriorities).toEqual([]); + }); + + it("coerces a bare-string array field into a one-element array (never silently drops it)", () => { + const warnings: string[] = []; + const b = normalizeBrief({ idea: "y", product: { users: "solo developers" }, competitors: "Pocket" }, (w) => warnings.push(w)); + expect(b.product.users).toEqual(["solo developers"]); + expect(b.competitors).toEqual(["Pocket"]); + expect(warnings.join(" ")).toMatch(/product\.users: expected an array — coerced/); + expect(warnings.join(" ")).toMatch(/competitors: expected an array — coerced/); + }); + + it("warns naming an unknown constraints key and lists the known ones", () => { + const warnings: string[] = []; + const b = normalizeBrief({ idea: "y", constraints: { deployment: "on-prem", budget: "none" } }, (w) => warnings.push(w)); + expect(b.constraints.budget).toBe("none"); + const joined = warnings.join(" "); + expect(joined).toMatch(/constraints\.deployment is not a recognized constraint/); + expect(joined).toMatch(/budget, timeline, team, compliance/); + }); + + it("still drops a genuine non-string non-array with the ignored warning", () => { + const warnings: string[] = []; + const b = normalizeBrief({ idea: "y", goals: 42 }, (w) => warnings.push(w)); + expect(b.goals).toEqual([]); + expect(warnings.join(" ")).toMatch(/goals is not an array — ignored/); }); it("stays silent when nothing is coerced", () => { From b80d2e81aa4520dddaf00d04f4242abe61b92aa9 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:53:44 +0200 Subject: [PATCH 06/10] fix(research): tag-scoped StackOverflow queries and an off-topic post-filter The `so` angle issued a free-text StackExchange query with no tag and no filtering, so relevance ranking surfaced high-vote but topically unrelated questions ("The Definitive C++ Book Guide") that lingered uncited in the dossier. The tech angle now scopes each per-candidate query to the tech's StackOverflow tag (soTagFor), retrying untagged once if the tag yields nothing, and every result must now share a keyword with the question (title or tags) or it is dropped with a counted note. Co-Authored-By: Claude Fable 5 --- scripts/construct.mjs | 83 ++++++++++++----- skills/construct/scripts/construct.mjs | 83 ++++++++++++----- src/research/stackoverflow.ts | 119 ++++++++++++++++++------- src/research/tech.ts | 7 +- tests/stackoverflow.test.ts | 65 +++++++++++++- tests/tech.test.ts | 5 +- 6 files changed, 279 insertions(+), 83 deletions(-) diff --git a/scripts/construct.mjs b/scripts/construct.mjs index 7371cdf..ec91fc4 100755 --- a/scripts/construct.mjs +++ b/scripts/construct.mjs @@ -1282,40 +1282,75 @@ ${ex[0].snippet}`; } // src/research/stackoverflow.ts -async function stackoverflow(question, perSource) { +function soTagFor(tech) { + return tech.trim().toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9.+-]/g, "").replace(/^-+|-+$/g, ""); +} +async function soQuery(q, perSource, tag) { + const pat = process.env.STACK_PAT ? `&access_token=${process.env.STACK_PAT}` : ""; + const tagged = tag ? `&tagged=${encodeURIComponent(tag)}` : ""; + const url = `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${q}&site=stackoverflow&filter=withbody&pagesize=${perSource}${tagged}${pat}`; + const r = await httpGet(url, { accept: "application/json" }); + return { ok: r.ok, status: r.status, body: r.body, url }; +} +async function stackoverflow(question, perSource, opts = {}) { const kws = rankedKeywords(question).slice(0, 5).join(" "); if (!kws) return { source: "so", items: [], notes: ["No keywords to search StackOverflow."] }; const q = encodeURIComponent(kws); - const pat = process.env.STACK_PAT ? `&access_token=${process.env.STACK_PAT}` : ""; - const url = `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${q}&site=stackoverflow&filter=withbody&pagesize=${perSource}${pat}`; - const r = await httpGet(url, { accept: "application/json" }); + const notes = []; + let r = await soQuery(q, perSource, opts.tag); if (!r.ok) { return { source: "so", items: [], notes: [`StackOverflow search unavailable (status ${r.status}).`] }; } + let data; try { - const data = JSON.parse(r.body); - const items = (data.items ?? []).map((it) => { - const body = htmlToText(String(it.body ?? "")).slice(0, 1200); - const accepted = it.is_answered ? "answered" : "unanswered"; - return { - source: "so", - title: htmlToText(String(it.title ?? "(question)")).slice(0, 160), - ref: `so:${it.question_id}`, - location: it.link, - score: Number(it.score ?? 0), - snippet: `score: ${it.score ?? 0} \xB7 ${accepted} \xB7 answers: ${it.answer_count ?? 0}` + (it.tags?.length ? ` \xB7 tags: ${it.tags.slice(0, 6).join(", ")}` : "") + ` + data = JSON.parse(r.body); + } catch { + return { source: "so", items: [], notes: ["StackOverflow search returned an unparseable response."] }; + } + if (opts.tag && (data.items ?? []).length === 0) { + r = await soQuery(q, perSource, void 0); + if (r.ok) { + try { + data = JSON.parse(r.body); + notes.push(`No tagged:${opts.tag} results \u2014 retried without the tag.`); + } catch { + } + } + } + const wantKws = new Set(keywords(question).map((k) => k.toLowerCase())); + const items = []; + let filtered = 0; + for (const raw of data.items ?? []) { + const it = raw; + const title = htmlToText(String(it.title ?? "(question)")).slice(0, 160); + const tags = Array.isArray(it.tags) ? it.tags : []; + if (wantKws.size) { + const hay = new Set(keywords(`${title} ${tags.join(" ")}`).map((k) => k.toLowerCase())); + const overlaps = [...wantKws].some((k) => hay.has(k)) || tags.some((t) => wantKws.has(t.toLowerCase())); + if (!overlaps) { + filtered++; + continue; + } + } + const body = htmlToText(String(it.body ?? "")).slice(0, 1200); + const accepted = it.is_answered ? "answered" : "unanswered"; + items.push({ + source: "so", + title, + ref: `so:${it.question_id}`, + location: it.link, + score: Number(it.score ?? 0), + snippet: `score: ${it.score ?? 0} \xB7 ${accepted} \xB7 answers: ${it.answer_count ?? 0}` + (tags.length ? ` \xB7 tags: ${tags.slice(0, 6).join(", ")}` : "") + ` ${body || "(no body)"}`, - url: it.link, - meta: { questionId: it.question_id, isAnswered: it.is_answered, answerCount: it.answer_count } - }; + url: it.link, + meta: { questionId: it.question_id, isAnswered: it.is_answered, answerCount: it.answer_count } }); - const notes = data.quota_remaining !== void 0 && data.quota_remaining < 20 ? [`StackExchange anonymous quota low (${data.quota_remaining} left).`] : []; - if (items.length === 0) notes.push("No StackOverflow questions matched."); - return { source: "so", items, notes }; - } catch { - return { source: "so", items: [], notes: ["StackOverflow search returned an unparseable response."] }; } + if (filtered) notes.push(`Filtered ${filtered} off-topic StackOverflow result(s) (no keyword overlap with the query).`); + if (data.quota_remaining !== void 0 && data.quota_remaining < 20) notes.push(`StackExchange anonymous quota low (${data.quota_remaining} left).`); + if (items.length === 0) notes.push("No StackOverflow questions matched."); + return { source: "so", items, notes }; } // src/research/tech.ts @@ -1352,7 +1387,7 @@ async function techAngle(ctx) { const per = Math.max(2, Math.ceil(ctx.perSource / Math.max(1, techs.length))); for (const tech of techs) { const q = `${tech} ${topKw}`.trim(); - const r = await stackoverflow(q, per); + const r = await stackoverflow(q, per, { tag: soTagFor(tech) }); for (const it of r.items) { if (!seen.has(it.ref)) { seen.add(it.ref); diff --git a/skills/construct/scripts/construct.mjs b/skills/construct/scripts/construct.mjs index 7371cdf..ec91fc4 100755 --- a/skills/construct/scripts/construct.mjs +++ b/skills/construct/scripts/construct.mjs @@ -1282,40 +1282,75 @@ ${ex[0].snippet}`; } // src/research/stackoverflow.ts -async function stackoverflow(question, perSource) { +function soTagFor(tech) { + return tech.trim().toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9.+-]/g, "").replace(/^-+|-+$/g, ""); +} +async function soQuery(q, perSource, tag) { + const pat = process.env.STACK_PAT ? `&access_token=${process.env.STACK_PAT}` : ""; + const tagged = tag ? `&tagged=${encodeURIComponent(tag)}` : ""; + const url = `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${q}&site=stackoverflow&filter=withbody&pagesize=${perSource}${tagged}${pat}`; + const r = await httpGet(url, { accept: "application/json" }); + return { ok: r.ok, status: r.status, body: r.body, url }; +} +async function stackoverflow(question, perSource, opts = {}) { const kws = rankedKeywords(question).slice(0, 5).join(" "); if (!kws) return { source: "so", items: [], notes: ["No keywords to search StackOverflow."] }; const q = encodeURIComponent(kws); - const pat = process.env.STACK_PAT ? `&access_token=${process.env.STACK_PAT}` : ""; - const url = `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${q}&site=stackoverflow&filter=withbody&pagesize=${perSource}${pat}`; - const r = await httpGet(url, { accept: "application/json" }); + const notes = []; + let r = await soQuery(q, perSource, opts.tag); if (!r.ok) { return { source: "so", items: [], notes: [`StackOverflow search unavailable (status ${r.status}).`] }; } + let data; try { - const data = JSON.parse(r.body); - const items = (data.items ?? []).map((it) => { - const body = htmlToText(String(it.body ?? "")).slice(0, 1200); - const accepted = it.is_answered ? "answered" : "unanswered"; - return { - source: "so", - title: htmlToText(String(it.title ?? "(question)")).slice(0, 160), - ref: `so:${it.question_id}`, - location: it.link, - score: Number(it.score ?? 0), - snippet: `score: ${it.score ?? 0} \xB7 ${accepted} \xB7 answers: ${it.answer_count ?? 0}` + (it.tags?.length ? ` \xB7 tags: ${it.tags.slice(0, 6).join(", ")}` : "") + ` + data = JSON.parse(r.body); + } catch { + return { source: "so", items: [], notes: ["StackOverflow search returned an unparseable response."] }; + } + if (opts.tag && (data.items ?? []).length === 0) { + r = await soQuery(q, perSource, void 0); + if (r.ok) { + try { + data = JSON.parse(r.body); + notes.push(`No tagged:${opts.tag} results \u2014 retried without the tag.`); + } catch { + } + } + } + const wantKws = new Set(keywords(question).map((k) => k.toLowerCase())); + const items = []; + let filtered = 0; + for (const raw of data.items ?? []) { + const it = raw; + const title = htmlToText(String(it.title ?? "(question)")).slice(0, 160); + const tags = Array.isArray(it.tags) ? it.tags : []; + if (wantKws.size) { + const hay = new Set(keywords(`${title} ${tags.join(" ")}`).map((k) => k.toLowerCase())); + const overlaps = [...wantKws].some((k) => hay.has(k)) || tags.some((t) => wantKws.has(t.toLowerCase())); + if (!overlaps) { + filtered++; + continue; + } + } + const body = htmlToText(String(it.body ?? "")).slice(0, 1200); + const accepted = it.is_answered ? "answered" : "unanswered"; + items.push({ + source: "so", + title, + ref: `so:${it.question_id}`, + location: it.link, + score: Number(it.score ?? 0), + snippet: `score: ${it.score ?? 0} \xB7 ${accepted} \xB7 answers: ${it.answer_count ?? 0}` + (tags.length ? ` \xB7 tags: ${tags.slice(0, 6).join(", ")}` : "") + ` ${body || "(no body)"}`, - url: it.link, - meta: { questionId: it.question_id, isAnswered: it.is_answered, answerCount: it.answer_count } - }; + url: it.link, + meta: { questionId: it.question_id, isAnswered: it.is_answered, answerCount: it.answer_count } }); - const notes = data.quota_remaining !== void 0 && data.quota_remaining < 20 ? [`StackExchange anonymous quota low (${data.quota_remaining} left).`] : []; - if (items.length === 0) notes.push("No StackOverflow questions matched."); - return { source: "so", items, notes }; - } catch { - return { source: "so", items: [], notes: ["StackOverflow search returned an unparseable response."] }; } + if (filtered) notes.push(`Filtered ${filtered} off-topic StackOverflow result(s) (no keyword overlap with the query).`); + if (data.quota_remaining !== void 0 && data.quota_remaining < 20) notes.push(`StackExchange anonymous quota low (${data.quota_remaining} left).`); + if (items.length === 0) notes.push("No StackOverflow questions matched."); + return { source: "so", items, notes }; } // src/research/tech.ts @@ -1352,7 +1387,7 @@ async function techAngle(ctx) { const per = Math.max(2, Math.ceil(ctx.perSource / Math.max(1, techs.length))); for (const tech of techs) { const q = `${tech} ${topKw}`.trim(); - const r = await stackoverflow(q, per); + const r = await stackoverflow(q, per, { tag: soTagFor(tech) }); for (const it of r.items) { if (!seen.has(it.ref)) { seen.add(it.ref); diff --git a/src/research/stackoverflow.ts b/src/research/stackoverflow.ts index 09cb2b0..9a7ab0c 100644 --- a/src/research/stackoverflow.ts +++ b/src/research/stackoverflow.ts @@ -1,46 +1,105 @@ import type { SourceResult, RawItem } from "../types.js"; -import { rankedKeywords } from "../util.js"; +import { keywords, rankedKeywords } from "../util.js"; import { httpGet, htmlToText } from "./fetch.js"; +// Slugify a candidate-technology name into a StackOverflow tag: lowercase, +// internal whitespace → "-", keep dots/hyphens/plus so "Next.js" → "next.js" +// and "C++" → "c++" survive as their real tag names. +export function soTagFor(tech: string): string { + return tech + .trim() + .toLowerCase() + .replace(/\s+/g, "-") + .replace(/[^a-z0-9.+-]/g, "") + .replace(/^-+|-+$/g, ""); +} + +// One StackExchange `search/advanced` call. Optional STACK_PAT raises the anon +// quota, never required. When `tag` is set the query is scoped to that tag. +async function soQuery(q: string, perSource: number, tag?: string): Promise<{ ok: boolean; status: number; body: string; url: string }> { + const pat = process.env.STACK_PAT ? `&access_token=${process.env.STACK_PAT}` : ""; + const tagged = tag ? `&tagged=${encodeURIComponent(tag)}` : ""; + const url = `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${q}&site=stackoverflow&filter=withbody&pagesize=${perSource}${tagged}${pat}`; + const r = await httpGet(url, { accept: "application/json" }); + return { ok: r.ok, status: r.status, body: r.body, url }; +} + // StackOverflow Q&A via the keyless StackExchange API (anonymous, rate-limited -// but enough for a few targeted lookups). Optional STACK_PAT raises the limit, -// never required. Used by the `tech` angle to surface known pitfalls of the -// candidate technologies. -export async function stackoverflow(question: string, perSource: number): Promise { +// but enough for a few targeted lookups). Used by the `tech` angle to surface +// known pitfalls of the candidate technologies. `opts.tag` scopes the query to +// a StackOverflow tag (the candidate tech); the generic angle passes none and +// relies on the off-topic post-filter alone. Results whose title+tags share no +// keyword with the question are dropped — StackExchange relevance ranking +// otherwise surfaces high-vote but topically-unrelated questions ("The +// Definitive C++ Book Guide") on a generic query. +export async function stackoverflow(question: string, perSource: number, opts: { tag?: string } = {}): Promise { const kws = rankedKeywords(question).slice(0, 5).join(" "); if (!kws) return { source: "so", items: [], notes: ["No keywords to search StackOverflow."] }; const q = encodeURIComponent(kws); - const pat = process.env.STACK_PAT ? `&access_token=${process.env.STACK_PAT}` : ""; - const url = - `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance` + `&q=${q}&site=stackoverflow&filter=withbody&pagesize=${perSource}${pat}`; - const r = await httpGet(url, { accept: "application/json" }); + const notes: string[] = []; + let r = await soQuery(q, perSource, opts.tag); if (!r.ok) { return { source: "so", items: [], notes: [`StackOverflow search unavailable (status ${r.status}).`] }; } + + let data: { items?: unknown[]; quota_remaining?: number }; try { - const data = JSON.parse(r.body); - const items: RawItem[] = (data.items ?? []).map((it: any) => { - const body = htmlToText(String(it.body ?? "")).slice(0, 1200); - const accepted = it.is_answered ? "answered" : "unanswered"; - return { - source: "so", - title: htmlToText(String(it.title ?? "(question)")).slice(0, 160), - ref: `so:${it.question_id}`, - location: it.link, - score: Number(it.score ?? 0), - snippet: - `score: ${it.score ?? 0} · ${accepted} · answers: ${it.answer_count ?? 0}` + - (it.tags?.length ? ` · tags: ${it.tags.slice(0, 6).join(", ")}` : "") + - `\n\n${body || "(no body)"}`, - url: it.link, - meta: { questionId: it.question_id, isAnswered: it.is_answered, answerCount: it.answer_count }, - }; - }); - const notes = data.quota_remaining !== undefined && data.quota_remaining < 20 ? [`StackExchange anonymous quota low (${data.quota_remaining} left).`] : []; - if (items.length === 0) notes.push("No StackOverflow questions matched."); - return { source: "so", items, notes }; + data = JSON.parse(r.body); } catch { return { source: "so", items: [], notes: ["StackOverflow search returned an unparseable response."] }; } + + // A tagged query that returns nothing is often the tag being slightly off + // (a package name vs its SO tag) — retry once untagged before giving up. + if (opts.tag && (data.items ?? []).length === 0) { + r = await soQuery(q, perSource, undefined); + if (r.ok) { + try { + data = JSON.parse(r.body); + notes.push(`No tagged:${opts.tag} results — retried without the tag.`); + } catch { + /* keep the empty tagged result */ + } + } + } + + const wantKws = new Set(keywords(question).map((k) => k.toLowerCase())); + const items: RawItem[] = []; + let filtered = 0; + for (const raw of data.items ?? []) { + const it = raw as Record; + const title = htmlToText(String(it.title ?? "(question)")).slice(0, 160); + const tags: string[] = Array.isArray(it.tags) ? it.tags : []; + // Off-topic filter: the item must share at least one query keyword with its + // title or tags. Skipped when we have no keywords to compare against. + if (wantKws.size) { + const hay = new Set(keywords(`${title} ${tags.join(" ")}`).map((k) => k.toLowerCase())); + const overlaps = [...wantKws].some((k) => hay.has(k)) || tags.some((t) => wantKws.has(t.toLowerCase())); + if (!overlaps) { + filtered++; + continue; + } + } + const body = htmlToText(String(it.body ?? "")).slice(0, 1200); + const accepted = it.is_answered ? "answered" : "unanswered"; + items.push({ + source: "so", + title, + ref: `so:${it.question_id}`, + location: it.link, + score: Number(it.score ?? 0), + snippet: + `score: ${it.score ?? 0} · ${accepted} · answers: ${it.answer_count ?? 0}` + + (tags.length ? ` · tags: ${tags.slice(0, 6).join(", ")}` : "") + + `\n\n${body || "(no body)"}`, + url: it.link, + meta: { questionId: it.question_id, isAnswered: it.is_answered, answerCount: it.answer_count }, + }); + } + + if (filtered) notes.push(`Filtered ${filtered} off-topic StackOverflow result(s) (no keyword overlap with the query).`); + if (data.quota_remaining !== undefined && data.quota_remaining < 20) notes.push(`StackExchange anonymous quota low (${data.quota_remaining} left).`); + if (items.length === 0) notes.push("No StackOverflow questions matched."); + return { source: "so", items, notes }; } diff --git a/src/research/tech.ts b/src/research/tech.ts index a63763d..f60c1e7 100644 --- a/src/research/tech.ts +++ b/src/research/tech.ts @@ -1,7 +1,7 @@ import type { ResearchContext, SourceResult, RawItem } from "../types.js"; import { rankedKeywords } from "../util.js"; import { discover, webFetchUrls } from "./web.js"; -import { stackoverflow } from "./stackoverflow.js"; +import { stackoverflow, soTagFor } from "./stackoverflow.js"; // The `tech` angle: feasibility grounding. For each candidate technology it // (a) fetches the project's official documentation (discovered on the web), and @@ -49,7 +49,10 @@ export async function techAngle(ctx: ResearchContext): Promise { const per = Math.max(2, Math.ceil(ctx.perSource / Math.max(1, techs.length))); for (const tech of techs) { const q = `${tech} ${topKw}`.trim(); - const r = await stackoverflow(q, per); + // Scope the lookup to the tech's StackOverflow tag so a per-candidate query + // stays on-topic (with an untagged retry inside stackoverflow if the tag + // yields nothing). + const r = await stackoverflow(q, per, { tag: soTagFor(tech) }); for (const it of r.items) { if (!seen.has(it.ref)) { seen.add(it.ref); diff --git a/tests/stackoverflow.test.ts b/tests/stackoverflow.test.ts index 226d3d7..40b0e16 100644 --- a/tests/stackoverflow.test.ts +++ b/tests/stackoverflow.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, afterEach } from "vitest"; -import { stackoverflow } from "../src/research/stackoverflow.js"; +import { stackoverflow, soTagFor } from "../src/research/stackoverflow.js"; function res(body: string, ok = true, status = 200) { return { @@ -52,3 +52,66 @@ describe("stackoverflow", () => { expect(r.notes.join(" ")).toMatch(/unavailable/); }); }); + +describe("soTagFor", () => { + it("slugifies a tech name into a StackOverflow tag, keeping dots and plus", () => { + expect(soTagFor("PostgreSQL")).toBe("postgresql"); + expect(soTagFor("Next.js")).toBe("next.js"); + expect(soTagFor("C++")).toBe("c++"); + expect(soTagFor("Ruby on Rails")).toBe("ruby-on-rails"); + }); +}); + +describe("stackoverflow — tag scoping + off-topic filter", () => { + const payloadWith = (items: unknown[]) => JSON.stringify({ quota_remaining: 200, items }); + + it("passes tagged= on the query when a tag is given", async () => { + let seenUrl = ""; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => { + seenUrl = url; + return res(payloadWith([{ question_id: 1, title: "postgres index bloat", body: "

vacuum

", score: 3, tags: ["postgresql"], link: "l" }])); + }), + ); + await stackoverflow("index bloat vacuum postgres", 6, { tag: "postgresql" }); + expect(seenUrl).toContain("tagged=postgresql"); + }); + + it("drops items whose title+tags share no keyword with the query", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + res( + payloadWith([ + { question_id: 1, title: "index bloat in postgres", body: "

x

", score: 3, tags: ["postgresql"], link: "a" }, + { question_id: 2, title: "The Definitive C++ Book Guide", body: "

y

", score: 4226, tags: ["c++", "books"], link: "b" }, + ]), + ), + ), + ); + const r = await stackoverflow("postgres index bloat vacuum", 6); + expect(r.items.map((i) => i.ref)).toEqual(["so:1"]); // the off-topic high-score C++ guide is filtered out + expect(r.notes.join(" ")).toMatch(/off-topic/i); + }); + + it("retries once without the tag when a tagged query returns nothing", async () => { + const calls: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => { + calls.push(url); + // first (tagged) call → empty; second (untagged) → one on-topic hit + return calls.length === 1 + ? res(payloadWith([])) + : res(payloadWith([{ question_id: 9, title: "meilisearch typo tolerance tuning", body: "

z

", score: 5, tags: ["meilisearch"], link: "c" }])); + }), + ); + const r = await stackoverflow("meilisearch typo tolerance", 6, { tag: "meilisearch" }); + expect(calls).toHaveLength(2); + expect(calls[0]).toContain("tagged=meilisearch"); + expect(calls[1]).not.toContain("tagged="); + expect(r.items).toHaveLength(1); + expect(r.notes.join(" ")).toMatch(/without the tag|untagged/i); + }); +}); diff --git a/tests/tech.test.ts b/tests/tech.test.ts index 36f0347..f29596d 100644 --- a/tests/tech.test.ts +++ b/tests/tech.test.ts @@ -8,12 +8,13 @@ function soRes(questionId: number) { items: [ { question_id: questionId, - title: `pitfall ${questionId}`, + // Title shares the idea keyword ("booking") so the off-topic filter keeps it. + title: `booking pitfall ${questionId}`, body: "

watch out

", score: 3, answer_count: 2, is_answered: true, - tags: ["x"], + tags: ["booking"], link: `https://stackoverflow.com/q/${questionId}`, }, ], From 0078d9ac3dcd0f6fb4f0351eb3091aa9729ae098 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:59:42 +0200 Subject: [PATCH 07/10] fix(research): strip consent boilerplate and surface low-signal snippets A pinned --docs-url page whose body did not match the question captured its cookie-consent banner as the "excerpt" and the SRD then cited it as support. fetchAndExtract now strips consent boilerplate before excerpting and extracts the page's meta description as a cleaner low-signal fallback; a zero-keyword-coverage excerpt is flagged meta.lowSignal, which review prefixes in its digest ("adjudicate skeptically") and analyze counts in its gap report. Co-Authored-By: Claude Fable 5 --- scripts/construct.mjs | 81 ++++++++++++++++++++------ skills/construct/scripts/construct.mjs | 81 ++++++++++++++++++++------ src/analyze.ts | 5 ++ src/research/fetch.ts | 59 +++++++++++++++++-- src/research/web.ts | 37 +++++++----- src/review.ts | 6 +- tests/analyze.test.ts | 14 +++++ tests/dossier.test.ts | 8 +++ tests/fetch.test.ts | 36 +++++++++++- tests/semantic-review.test.ts | 11 ++++ tests/web.test.ts | 13 +++++ 11 files changed, 293 insertions(+), 58 deletions(-) diff --git a/scripts/construct.mjs b/scripts/construct.mjs index ec91fc4..e156f4e 100755 --- a/scripts/construct.mjs +++ b/scripts/construct.mjs @@ -539,6 +539,34 @@ function htmlToText(html) { s = s.replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n"); return s.split("\n").map((l) => l.trim()).filter((l) => l.length > 0).join("\n"); } +var CONSENT_PATTERNS = [ + /\bcookies?\b/i, + /\bconsent\b/i, + /\bgdpr\b/i, + /\bccpa\b/i, + /accept all\b/i, + /reject all\b/i, + /manage (?:preferences|choices|cookies|settings)/i, + /privacy (?:policy|preferences|choices)/i, + /tracking technolog/i, + /advertising partners/i, + /legitimate interest/i +]; +function stripConsentBoilerplate(text) { + let dropped = 0; + const kept = text.split("\n").filter((line) => { + const hits = CONSENT_PATTERNS.reduce((n, re) => n + (re.test(line) ? 1 : 0), 0); + const isBanner = hits >= 2 || hits === 1 && line.trim().length < 120; + if (isBanner) dropped++; + return !isBanner; + }); + return { text: kept.join("\n"), dropped }; +} +function metaDescriptionOf(html) { + const m = /]+name=["']description["'][^>]*content=["']([^"']+)["']/i.exec(html) || /]+content=["']([^"']+)["'][^>]*name=["']description["']/i.exec(html) || /]+property=["']og:description["'][^>]*content=["']([^"']+)["']/i.exec(html); + const d = m?.[1]?.replace(/\s+/g, " ").trim(); + return d || void 0; +} async function fetchAndExtract(url) { let res = await httpGet(url, { accept: "text/html,text/plain,*/*" }); if (!res.ok && (res.status === 403 || res.status === 429)) { @@ -551,8 +579,10 @@ async function fetchAndExtract(url) { return { text: "", note: `Could not fetch ${url} (status ${res.status}${res.error ? ", " + res.error : ""}).` }; } const isHtml = /html/i.test(res.contentType) || /^\s* e.meta?.lowSignal).length; + if (lowSignal) { + notes.push(`${lowSignal} low-signal snippet(s) in the dossier \u2014 likely boilerplate; re-drill with a sharper --q or a better --docs-url.`); + } const suggestions = []; const ungroundedFeatures = brief.featureWishlist.filter((f) => matchEvidence(featureText(f), evidence, 1, GROUND_REQUIREMENT).length === 0).map((f) => ({ title: f.title, priority: f.priority ?? "should" })); for (const f of ungroundedFeatures) suggestions.push(drill("web", f.title)); diff --git a/skills/construct/scripts/construct.mjs b/skills/construct/scripts/construct.mjs index ec91fc4..e156f4e 100755 --- a/skills/construct/scripts/construct.mjs +++ b/skills/construct/scripts/construct.mjs @@ -539,6 +539,34 @@ function htmlToText(html) { s = s.replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n"); return s.split("\n").map((l) => l.trim()).filter((l) => l.length > 0).join("\n"); } +var CONSENT_PATTERNS = [ + /\bcookies?\b/i, + /\bconsent\b/i, + /\bgdpr\b/i, + /\bccpa\b/i, + /accept all\b/i, + /reject all\b/i, + /manage (?:preferences|choices|cookies|settings)/i, + /privacy (?:policy|preferences|choices)/i, + /tracking technolog/i, + /advertising partners/i, + /legitimate interest/i +]; +function stripConsentBoilerplate(text) { + let dropped = 0; + const kept = text.split("\n").filter((line) => { + const hits = CONSENT_PATTERNS.reduce((n, re) => n + (re.test(line) ? 1 : 0), 0); + const isBanner = hits >= 2 || hits === 1 && line.trim().length < 120; + if (isBanner) dropped++; + return !isBanner; + }); + return { text: kept.join("\n"), dropped }; +} +function metaDescriptionOf(html) { + const m = /]+name=["']description["'][^>]*content=["']([^"']+)["']/i.exec(html) || /]+content=["']([^"']+)["'][^>]*name=["']description["']/i.exec(html) || /]+property=["']og:description["'][^>]*content=["']([^"']+)["']/i.exec(html); + const d = m?.[1]?.replace(/\s+/g, " ").trim(); + return d || void 0; +} async function fetchAndExtract(url) { let res = await httpGet(url, { accept: "text/html,text/plain,*/*" }); if (!res.ok && (res.status === 403 || res.status === 429)) { @@ -551,8 +579,10 @@ async function fetchAndExtract(url) { return { text: "", note: `Could not fetch ${url} (status ${res.status}${res.error ? ", " + res.error : ""}).` }; } const isHtml = /html/i.test(res.contentType) || /^\s* e.meta?.lowSignal).length; + if (lowSignal) { + notes.push(`${lowSignal} low-signal snippet(s) in the dossier \u2014 likely boilerplate; re-drill with a sharper --q or a better --docs-url.`); + } const suggestions = []; const ungroundedFeatures = brief.featureWishlist.filter((f) => matchEvidence(featureText(f), evidence, 1, GROUND_REQUIREMENT).length === 0).map((f) => ({ title: f.title, priority: f.priority ?? "should" })); for (const f of ungroundedFeatures) suggestions.push(drill("web", f.title)); diff --git a/src/analyze.ts b/src/analyze.ts index a53ac61..34f6d9d 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -54,6 +54,11 @@ export function analyzeRun(runDir: string): GapReport { notes.push("No evidence dossier — run `construct research` first; everything below will render ungrounded."); } + const lowSignal = evidence.filter((e) => e.meta?.lowSignal).length; + if (lowSignal) { + notes.push(`${lowSignal} low-signal snippet(s) in the dossier — likely boilerplate; re-drill with a sharper --q or a better --docs-url.`); + } + const suggestions: string[] = []; // Features with no evidence the renderer could cite (same matcher, same diff --git a/src/research/fetch.ts b/src/research/fetch.ts index 47a782f..964b4eb 100644 --- a/src/research/fetch.ts +++ b/src/research/fetch.ts @@ -169,9 +169,55 @@ export function htmlToText(html: string): string { .join("\n"); } +// Consent/cookie-banner boilerplate that survives htmlToText (it lives in body +//
/, not