diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d203ae..e75ad26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 15.0.0 — 2026-09-07 + +KB improvement can declare a pages directory and include canonical claim ledgers and research events in candidate state. +Snapshots, hashes, resume, selected changes, promotion, and restoration use the same declared scope. +`normalizeKnowledgeStateScope` lets consumers carry the canonical scope without duplicating its defaults or validation. +Existing calls retain the default page, raw-source, and source-registry scope. +Configured tool receipt capture now persists exact visibility artifacts in the run store before recording receipts. +The Eval peer range also admits 0.176 for complete optimization history and final measurement receipts. +The major version follows the package gate for changed exported contract shapes. + ## 14.0.3 — 2026-09-07 The Eval peer range admits both 0.174 and 0.175, restoring compatibility with Eval 0.174 consumers. diff --git a/README.md b/README.md index 9a1a3b3..882d513 100644 --- a/README.md +++ b/README.md @@ -318,6 +318,25 @@ A failed final evaluation ends the run instead of selecting another candidate ag The default evaluator reports only measured dimensions and averages those dimensions with equal weight. It omits `answer_quality` without answer evaluation, `promotion_decision` without a promotion decision, and `blocking_readiness` without blocking readiness requirements. Default evaluator version `2` records this weighting. +Declare additional authoritative local state through `stateScope`: + +```ts +const stateScope = { pagesDirectory: 'kb/pages', researchState: true } +const result = await improveKnowledgeBase({ ...options, stateScope }) +const identity = await hashKnowledgeBase(root, stateScope) +``` + +The default scope remains `knowledge/`, `raw/`, and `.agent-knowledge/sources.json`. +`pagesDirectory` replaces the default page directory for copying, indexing, hashing, and promotion. +`researchState: true` also includes canonical `.agent-knowledge/claim-ledgers/` records and `.agent-knowledge/events.json`. +The persisted run binds this scope, and resume rejects a different scope. +Candidate materialization, selected changes, promotion, and restoration preserve the same declared state. +Use `normalizeKnowledgeStateScope` when carrying this scope into another contract. +Indexes, locks, transaction journals, retrieval artifacts, credentials outside declared content directories, and external memory providers remain outside this identity. +Use the canonical `FileSystemKbStore({ root: candidateRoot })` for research records in update callbacks. +Callbacks must bind external state separately and must not close over a shared mutable store when evaluating isolated candidates. +A scoped KB hash does not identify an entire learner with external state. + A candidate can pass structural checks without any task outcome evaluation; the metric notes state this limit. `candidate-ready` means the configured checks passed and the candidate remains detached from the live knowledge base. diff --git a/api-surface.json b/api-surface.json index 82a86aa..408d2f7 100644 --- a/api-surface.json +++ b/api-surface.json @@ -201,7 +201,7 @@ "KnowledgeControlLoopAction": "value 278d5c67af20", "KnowledgeControlLoopActionResult": "value cf2e78a26708", "KnowledgeControlLoopAdapter": "value 4e5907588233", - "KnowledgeControlLoopAdapterOptions": "value cbc0fec0e6d8", + "KnowledgeControlLoopAdapterOptions": "value eec2b80a8c88", "KnowledgeControlLoopState": "value 8343ad013bdc", "KnowledgeDiscoveryDispatcher": "value a4ed7f82b891", "KnowledgeDiscoveryWorker": "value 704c91abebe2", @@ -235,12 +235,12 @@ "KnowledgeImprovementMetricProvenance": "type 8c9bdc8396db", "KnowledgeImprovementMutationReceipt": "type b7756b2e1432", "KnowledgeImprovementMutationResult": "type 9b65834cf2e3", - "KnowledgeImprovementOptions": "type e10d24cb9e5c", + "KnowledgeImprovementOptions": "type f295268a3492", "KnowledgeImprovementRagOptimizationOptions": "type 57b600037905", "KnowledgeImprovementRagOptimizationRunInput": "type 0bba567069ac", "KnowledgeImprovementResult": "type a457d844fd3d", "KnowledgeImprovementRetrievalOptions": "type ec2bec026699", - "KnowledgeImprovementRunState": "type 90928bbe8ecc", + "KnowledgeImprovementRunState": "type 1ae3b226bbdd", "KnowledgeImprovementRunStateSchema": "value 373728f5643d", "KnowledgeImprovementStatus": "type 0b4aedbceeb7", "KnowledgeImprovementTarget": "type 2a1bc73ffa39", @@ -313,6 +313,7 @@ "KnowledgeSearchHit": "value d13a101a4b4f", "KnowledgeSearchResult": "value c443b7efb49e", "KnowledgeSource": "value 6ebb3f0a0ef2", + "KnowledgeStateScope": "type 61b8ac32e052", "KnowledgeUnit": "value bbd850260f7f", "KnowledgeUseReceipt": "value 95b469ec3c07", "KnowledgeUseRelation": "value b47137088743", @@ -419,8 +420,8 @@ "ResearchSourceVersion": "value f20d55e99f71", "ResearchSourceVersionSchema": "value 373728f5643d", "ResearchWorker": "value 5b69df6c9975", - "ResolvedKnowledgeImprovementCandidate": "type d0a1c8976ee6", - "ResolvedKnowledgeImprovementComparison": "type 45197417f12c", + "ResolvedKnowledgeImprovementCandidate": "type 60be04dfb75d", + "ResolvedKnowledgeImprovementComparison": "type 2d4520e11c17", "ResolvedKnowledgeImprovementComparisonSnapshot": "type 09ca4bdc3f81", "RestoreKnowledgeCandidateBaselineOptions": "type a910d66fbc6b", "RetrievalConfig": "value 0f5f3a0fc625", @@ -459,7 +460,7 @@ "RunDiscoveryLoopOptions": "value bfcdb3617601", "RunKnowledgeBenchmarkSuiteOptions": "type 8f7c9a385609", "RunKnowledgeBenchmarkSuiteResult": "type 46bed11cb991", - "RunKnowledgeResearchLoopOptions": "value b47742289fe2", + "RunKnowledgeResearchLoopOptions": "value ac668b866256", "RunLineageAuthority": "value aa33159b2c96", "RunLineageRecord": "value 9ba2102ed82c", "RunMemoryAdapterBenchmarkOptions": "type 9e6710497bb1", @@ -623,7 +624,7 @@ "gradeFor": "value 79db4a5137a3", "graphitiMemoryAdapterIdentity": "value 203f93807ab2", "groundClaimInText": "value f74ed9ca26c6", - "hashKnowledgeBase": "value 701cd37f4e5b", + "hashKnowledgeBase": "value bdb2d9eea5f9", "htmlToText": "value c58c3f5d8c75", "improveKnowledgeBase": "value a4ebce095bdf", "improveSelectedKnowledgeCandidate": "value 6d48e36af283", @@ -679,6 +680,7 @@ "neighbors": "value 29f67f6cea95", "normalizeClaimText": "value 4c9eca896a37", "normalizeExternalRagScores": "value 1a2d603c9282", + "normalizeKnowledgeStateScope": "value beef86fc6f66", "normalizeLinkTarget": "value 0258f337b3b3", "normalizePageText": "value 9c20decc8888", "normalizePagesDirectory": "value 7af1695dc6cf", diff --git a/docs/architecture.md b/docs/architecture.md index ebdb2e3..3a3c3e9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,6 +67,17 @@ Every write in this layer goes through `durable-fs` (`writeFileDurable`, `writeJ `O_NOFOLLOW` descriptors anchored through `/proc/self/fd` prevent a directory swapped for a symlink during a write from redirecting it outside the root. These are exported from the package entrypoint; consumers that keep their own journals should use them rather than reimplement them. +## Candidate state scope + +KB improvement snapshots include pages, raw evidence, and the source registry by default. +A declared `stateScope.pagesDirectory` selects the same pages for writers, readers, indexing, and promotion. +Opting into `stateScope.researchState` also binds canonical claim ledgers and research events. +The hash includes nondefault scope declarations, so two different scopes cannot silently share an identity when their extra directories are empty. +Resume uses the persisted scope and refuses a changed declaration. +Promotion and crash recovery apply its narrow path allowlist through the existing file transaction journal. +Derived indexes, lock state, retrieval artifacts, and external providers are excluded. +Applications must bind external state through their existing memory branch or evaluation contracts. + ## Runtime Loop 1. Normalize sources into immutable source records. diff --git a/docs/knowledge-use-receipts.md b/docs/knowledge-use-receipts.md index 7a1fe0d..a973bd3 100644 --- a/docs/knowledge-use-receipts.md +++ b/docs/knowledge-use-receipts.md @@ -62,6 +62,16 @@ const artifact = knowledgeVisibilityArtifactRef({ uri: 'artifact://run/visibilit The artifact reference is optional at this contract layer: a caller that retains the snapshot in another durable record may omit it. A production adapter should require a durable locator. +When `createKnowledgeTools` has a `recordRetrieval` sink, it persists canonical visibility bytes before calling that sink. +It stores artifacts under the run's `.agent-knowledge/retrieval-visibility/` directory and attaches a `file:` artifact locator to each receipt. +Searches over the same view reuse that artifact, including concurrent searches. +Changed views receive different artifacts. +A persistence failure or conflicting stored bytes prevents receipt delivery. +The host must retain these artifacts with its receipts and make the locator accessible to later verification. +Searches without a receipt sink still return an in-memory receipt without creating durable evidence. +These artifacts record visibility; they do not promote observations into sourced claims or prove downstream utility. + + ## Retrieval receipt `createKnowledgeRetrievalReceipt()` binds: diff --git a/package.json b/package.json index cdee568..c74266e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "14.0.3", + "version": "15.0.0", "description": "Build, search, evaluate, and improve source-backed knowledge bases.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { @@ -84,7 +84,7 @@ "zod": "4.5.4" }, "peerDependencies": { - "@tangle-network/agent-eval": ">=0.174.0 <0.176.0", + "@tangle-network/agent-eval": ">=0.174.0 <0.177.0", "@tangle-network/agent-interface": "^2.0.0" }, "devDependencies": { diff --git a/scripts/verify-official-optimizers.mjs b/scripts/verify-official-optimizers.mjs index 335184f..8ca44ba 100644 --- a/scripts/verify-official-optimizers.mjs +++ b/scripts/verify-official-optimizers.mjs @@ -18,7 +18,7 @@ const agentEvalVersion = sourcePackage.devDependencies?.['@tangle-network/agent- if (!/^\d+\.\d+\.\d+$/.test(agentEvalVersion)) { throw new Error('@tangle-network/agent-eval must have one exact development pin') } -const expectedEvalPeerRange = '>=0.174.0 <0.176.0' +const expectedEvalPeerRange = '>=0.174.0 <0.177.0' if (sourcePackage.peerDependencies?.['@tangle-network/agent-eval'] !== expectedEvalPeerRange) { throw new Error( `@tangle-network/agent-eval peer range must be ${expectedEvalPeerRange} to match the development pin`, diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 301626b..53599ff 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -25,6 +25,7 @@ const publicImports = [ ] const requiredRootExports = [ 'createFileSystemSearchProvider', + 'normalizeKnowledgeStateScope', 'optimizeKnowledgeBasePolicy', 'runRagOptimization', 'runRetrievalImprovementLoop', @@ -68,10 +69,10 @@ const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') const sourcePackage = JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')) const agentEvalVersion = process.env.AGENT_KNOWLEDGE_EVAL_VERSION ?? exactDevelopmentPin(sourcePackage, agentEvalPackage) -if (!['0.174.0', '0.175.0'].includes(agentEvalVersion)) { +if (!['0.174.0', '0.175.0', '0.176.0'].includes(agentEvalVersion)) { throw new Error(`unsupported Eval compatibility test version: ${agentEvalVersion}`) } -const agentEvalPeerRange = '>=0.174.0 <0.176.0' +const agentEvalPeerRange = '>=0.174.0 <0.177.0' const agentInterfaceVersion = exactDevelopmentPin(sourcePackage, agentInterfacePackage) const agentInterfacePeerRange = expectedPeerRange(agentInterfaceVersion) const zodVersion = exactVersion(sourcePackage.dependencies?.zod, 'zod runtime dependency') diff --git a/src/file-transaction.ts b/src/file-transaction.ts index cac62ac..29c05aa 100644 --- a/src/file-transaction.ts +++ b/src/file-transaction.ts @@ -56,6 +56,7 @@ const transactionSchema = z purpose: z.string().min(1), recoveryOwner: z.string().min(1).max(256).optional(), pagesDirectory: pagesDirectorySchema.optional(), + researchState: z.boolean().optional(), createdAt: z.string().min(1), entries: z.array(transactionEntrySchema).min(1), }) @@ -85,9 +86,10 @@ export interface KnowledgeFileTransactionPlanEntry { export function knowledgeFileTransactionPlanHash( entries: readonly KnowledgeFileTransactionPlanEntry[], pagesDirectory: string, + researchState = false, ): string { const normalized = entries - .map((entry) => normalizePlanEntry(entry, pagesDirectory)) + .map((entry) => normalizePlanEntry(entry, pagesDirectory, researchState)) .sort((left, right) => left.path.localeCompare(right.path)) if (new Set(normalized.map((entry) => entry.path)).size !== normalized.length) { throw new Error('knowledge transaction plan repeats a path') @@ -109,6 +111,8 @@ export async function prepareKnowledgeFileTransaction(input: { mutations: readonly KnowledgeFileMutation[] /** Pages directory the mutations may write under; defaults to `knowledge`. */ pagesDirectory?: string + /** Explicitly permit authoritative claim-ledger and research-event records. */ + researchState?: boolean includeUnchanged?: boolean now?: () => Date }): Promise { @@ -129,7 +133,11 @@ export async function prepareKnowledgeFileTransaction(input: { const paths = new Set() const prepared = await Promise.all( input.mutations.map(async (mutation, index) => { - const path = assertKnowledgeMutationPath(mutation.path, boundPagesDirectory) + const path = assertKnowledgeMutationPath( + mutation.path, + boundPagesDirectory, + input.researchState, + ) if (paths.has(path)) throw new Error(`knowledge file transaction repeats path: ${path}`) if (mutation.content === null && mutation.mode !== undefined) { throw new Error(`deleted knowledge file cannot declare a mode: ${path}`) @@ -192,6 +200,7 @@ export async function prepareKnowledgeFileTransaction(input: { purpose: input.purpose, ...(input.recoveryOwner ? { recoveryOwner: input.recoveryOwner } : {}), ...(pagesDirectory === undefined ? {} : { pagesDirectory }), + ...(input.researchState === undefined ? {} : { researchState: input.researchState }), createdAt: (input.now ?? (() => new Date()))().toISOString(), entries: changed.map((item) => item.entry), }) @@ -222,6 +231,8 @@ export async function commitKnowledgeFileMutations(input: { mutations: readonly KnowledgeFileMutation[] /** Pages directory the mutations may write under; defaults to `knowledge`. */ pagesDirectory?: string + /** Explicitly permit authoritative claim-ledger and research-event records. */ + researchState?: boolean assertOwned?: () => void now?: () => Date }): Promise { @@ -238,6 +249,7 @@ export async function commitKnowledgeFileMutations(input: { transactionRoot: input.transactionRoot, purpose: input.purpose, mutations: input.mutations, + ...(input.researchState === undefined ? {} : { researchState: input.researchState }), ...(input.pagesDirectory === undefined ? {} : { pagesDirectory: input.pagesDirectory }), now: input.now, }) @@ -630,7 +642,11 @@ function assertTransactionEntries(transaction: KnowledgeFileTransaction): void { const paths = new Set() const boundPagesDirectory = normalizePagesDirectory(transaction.pagesDirectory) for (const entry of transaction.entries) { - const normalized = assertKnowledgeMutationPath(entry.path, boundPagesDirectory) + const normalized = assertKnowledgeMutationPath( + entry.path, + boundPagesDirectory, + transaction.researchState, + ) if (entry.path !== normalized || indexes.has(entry.index) || paths.has(entry.path)) { throw new Error('knowledge file transaction has duplicate or unsafe entries') } @@ -641,8 +657,12 @@ function assertTransactionEntries(transaction: KnowledgeFileTransaction): void { } } -function normalizePlanEntry(entry: KnowledgeFileTransactionPlanEntry, pagesDirectory: string) { - const path = assertKnowledgeMutationPath(entry.path, pagesDirectory) +function normalizePlanEntry( + entry: KnowledgeFileTransactionPlanEntry, + pagesDirectory: string, + researchState: boolean, +) { + const path = assertKnowledgeMutationPath(entry.path, pagesDirectory, researchState) assertHashModePair(path, 'before', entry.beforeHash, entry.beforeMode) assertHashModePair(path, 'after', entry.afterHash, entry.afterMode) return { @@ -683,10 +703,17 @@ export function knowledgeMutationPathPrefixes(pagesDirectory: string): readonly return [`${normalizePagesDirectory(pagesDirectory)}/`, 'raw/'] } -export function assertKnowledgeMutationPath(path: string, pagesDirectory: string): string { +export function assertKnowledgeMutationPath( + path: string, + pagesDirectory: string, + researchState = false, +): string { const normalized = normalizeTransactionPath(path) if ( normalized === SOURCE_REGISTRY_PATH || + (researchState && + (normalized === '.agent-knowledge/events.json' || + /^\.agent-knowledge\/claim-ledgers\/[^/]+\.json$/.test(normalized))) || knowledgeMutationPathPrefixes(pagesDirectory).some((prefix) => normalized.startsWith(prefix)) ) { return normalized diff --git a/src/index.ts b/src/index.ts index 51ee2f4..2e3de16 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,6 +30,10 @@ export * from './investment-thesis-task' export * from './kb-improvement' export * from './kb-store' export * from './knowledge-brief' +export { + type KnowledgeStateScope, + normalizeKnowledgeStateScope, +} from './knowledge-state-scope' export * from './knowledge-tools' export * from './knowledge-use-receipts' export * from './lexical-index' diff --git a/src/kb-improvement/contracts.ts b/src/kb-improvement/contracts.ts index 3cafb39..748f568 100644 --- a/src/kb-improvement/contracts.ts +++ b/src/kb-improvement/contracts.ts @@ -10,6 +10,7 @@ import type { EvalKnowledgeBundleBuildResult, KnowledgeReadinessSpec, } from '../eval-readiness' +import { type KnowledgeStateScope, knowledgeStateScopeSchema } from '../knowledge-state-scope' import { DEFAULT_PAGES_DIRECTORY } from '../pages-directory' import type { KnowledgeBaseQualityOptions, @@ -102,6 +103,7 @@ export interface KnowledgeImprovementCandidateRecord { } export interface KnowledgeImprovementRunState { + stateScope?: KnowledgeStateScope runId: string root: string goal: string @@ -280,6 +282,7 @@ const candidateRecordSchema = z export const KnowledgeImprovementRunStateSchema = z .object({ + stateScope: knowledgeStateScopeSchema.optional(), runId: runIdSchema, root: z.string().min(1), goal: z.string().min(1), @@ -455,6 +458,7 @@ export interface ResolvedKnowledgeImprovementComparisonSnapshot { } export interface ResolvedKnowledgeImprovementComparison { + stateScope?: KnowledgeStateScope reference: KnowledgeImprovementCandidateRef evaluation: KnowledgeImprovementMetric baseline: ResolvedKnowledgeImprovementComparisonSnapshot @@ -462,6 +466,7 @@ export interface ResolvedKnowledgeImprovementComparison { } export interface ResolvedKnowledgeImprovementCandidate { + stateScope?: KnowledgeStateScope root: string candidate: KnowledgeImprovementCandidateRef evaluation: KnowledgeImprovementMetric @@ -506,6 +511,8 @@ export type KnowledgeImprovementUpdate = ( ) => Promise | RagKnowledgeUpdateResult export interface KnowledgeImprovementOptions { + /** Declared authoritative local state, frozen and restored with each candidate. */ + stateScope?: KnowledgeStateScope root: string goal: string /** diff --git a/src/kb-improvement/evaluation.ts b/src/kb-improvement/evaluation.ts index 3514ccc..c448aee 100644 --- a/src/kb-improvement/evaluation.ts +++ b/src/kb-improvement/evaluation.ts @@ -6,6 +6,7 @@ import { knowledgeFileTransactionPlanHash } from '../file-transaction' import { sha256 } from '../ids' import { assertImmutableRef } from '../immutable-ref' import { buildKnowledgeIndex } from '../indexer' +import { normalizeKnowledgeStateScope } from '../knowledge-state-scope' import { ragAnswerEvidenceRejectionReasons } from '../rag-answer-evidence' import { type KnowledgeBaseQualityReport, scoreKnowledgeBaseIndex } from '../rag-eval' import { @@ -28,7 +29,6 @@ import type { import { EVALUATION_PHASES, improvementMetricSchema, - KB_IMPROVEMENT_PAGES_DIRECTORY, KnowledgeImprovementEvidenceSchema, UPDATE_PHASES, } from './contracts' @@ -50,6 +50,7 @@ import { export function assertKnowledgeImprovementOptions(options: KnowledgeImprovementOptions): void { assertImmutableRef(options.implementationRef, 'knowledge improvement implementationRef') + normalizeKnowledgeStateScope(options.stateScope) for (const phase of options.requiredPhases ?? []) { if (options.enabledPhases && !options.enabledPhases.includes(phase)) { throw new Error(`required phase ${phase} is not enabled`) @@ -103,7 +104,7 @@ export async function measureCandidate( finalEvaluated: boolean }> { return withCandidateWorkspace(runDir, candidate, async (candidateRoot) => { - const currentCandidateHash = await hashKnowledgeBase(candidateRoot) + const currentCandidateHash = await hashKnowledgeBase(candidateRoot, options.stateScope) if ( candidate.status === 'candidate-ready' && candidate.candidateHash === currentCandidateHash && @@ -130,60 +131,66 @@ export async function measureCandidate( if (candidate.status === 'running') { lifecycle = await runCandidateUpdateLifecycle(runId, candidate, candidateRoot, options, now) } - return withFrozenCandidateWorkspace(runDir, candidate, candidateRoot, async (snapshot) => { - const development = await evaluateCandidate( - runDir, - state, - candidate, - snapshot, - lifecycle, - options, - now, - false, - ) - if (!development.evaluation.passed || !shouldRunEvaluationStage(options)) { + return withFrozenCandidateWorkspace( + runDir, + candidate, + candidateRoot, + async (snapshot) => { + const development = await evaluateCandidate( + runDir, + state, + candidate, + snapshot, + lifecycle, + options, + now, + false, + ) + if (!development.evaluation.passed || !shouldRunEvaluationStage(options)) { + return { + ...development, + ...(lifecycle ? { lifecycle } : {}), + finalEvaluated: false, + } + } + + candidate.finalEvaluationStartedAt = now().toISOString() + candidate.updatedAt = candidate.finalEvaluationStartedAt + state.updatedAt = candidate.finalEvaluationStartedAt + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.final-evaluation-started', + runId: state.runId, + candidateId: candidate.candidateId, + }) + lifecycle = await runCandidateEvaluationLifecycle( + runId, + runDir, + candidate, + snapshot.root, + snapshot.hash, + lifecycle, + options, + now, + ) + const measured = await evaluateCandidate( + runDir, + state, + candidate, + snapshot, + lifecycle, + options, + now, + true, + ) return { - ...development, + ...measured, ...(lifecycle ? { lifecycle } : {}), - finalEvaluated: false, + finalEvaluated: true, } - } - - candidate.finalEvaluationStartedAt = now().toISOString() - candidate.updatedAt = candidate.finalEvaluationStartedAt - state.updatedAt = candidate.finalEvaluationStartedAt - await saveState(runDir, state, options.onState) - await appendLedger(runDir, { - type: 'candidate.final-evaluation-started', - runId: state.runId, - candidateId: candidate.candidateId, - }) - lifecycle = await runCandidateEvaluationLifecycle( - runId, - runDir, - candidate, - snapshot.root, - snapshot.hash, - lifecycle, - options, - now, - ) - const measured = await evaluateCandidate( - runDir, - state, - candidate, - snapshot, - lifecycle, - options, - now, - true, - ) - return { - ...measured, - ...(lifecycle ? { lifecycle } : {}), - finalEvaluated: true, - } - }) + }, + options.stateScope, + ) }) } @@ -220,53 +227,58 @@ async function runCandidateEvaluationLifecycle( now: () => Date, ): Promise { if (!shouldRunEvaluationStage(options)) return undefined - const candidateIndex = await buildKnowledgeIndex(candidateRoot) - return withBaselineSnapshot(runDir, candidate.baseHash, (baselineRoot) => - runRagKnowledgeImprovementPhases( - { - goal: options.goal, - optimization: options.ragOptimization - ? { - ...options.ragOptimization, - executionRef: candidateExecutionRef( - options.ragOptimization.executionRef, - candidateHash, - ), - runDir: - options.ragOptimization.runDir ?? - join(runDir, 'rag-optimization', candidate.candidateId), - run: (input) => - options.ragOptimization!.run({ - ...input, - runId, - iteration: candidate.iteration, - candidateId: candidate.candidateId, - root: candidateRoot, - baselineRoot, - candidateRoot, - candidateIndex, - baseHash: candidate.baseHash, - }), - } - : undefined, - retrieval: options.retrieval - ? { - ...options.retrieval, - executionRef: candidateExecutionRef(options.retrieval.executionRef, candidateHash), - index: candidateIndex, - runDir: options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), - } - : undefined, - evaluateAnswers: options.evaluateAnswers, - answerQualityCostCeiling: options.answerQualityCostCeiling, - decidePromotion: options.decidePromotion, - enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), - requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), - signal: options.signal, - now, - }, - lifecycle, - ), + const candidateIndex = await buildKnowledgeIndex(candidateRoot, options.stateScope) + return withBaselineSnapshot( + runDir, + candidate.baseHash, + (baselineRoot) => + runRagKnowledgeImprovementPhases( + { + goal: options.goal, + optimization: options.ragOptimization + ? { + ...options.ragOptimization, + executionRef: candidateExecutionRef( + options.ragOptimization.executionRef, + candidateHash, + ), + runDir: + options.ragOptimization.runDir ?? + join(runDir, 'rag-optimization', candidate.candidateId), + run: (input) => + options.ragOptimization!.run({ + ...input, + runId, + iteration: candidate.iteration, + candidateId: candidate.candidateId, + root: candidateRoot, + baselineRoot, + candidateRoot, + candidateIndex, + baseHash: candidate.baseHash, + }), + } + : undefined, + retrieval: options.retrieval + ? { + ...options.retrieval, + executionRef: candidateExecutionRef(options.retrieval.executionRef, candidateHash), + index: candidateIndex, + runDir: + options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), + } + : undefined, + evaluateAnswers: options.evaluateAnswers, + answerQualityCostCeiling: options.answerQualityCostCeiling, + decidePromotion: options.decidePromotion, + enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), + requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), + signal: options.signal, + now, + }, + lifecycle, + ), + options.stateScope, ) } @@ -284,6 +296,7 @@ function candidateKnowledgeResearchOptions( return { ...rest, root: candidateRoot, + pagesDirectory: normalizeKnowledgeStateScope(options.stateScope).pagesDirectory, step, maxIterations: rest.maxIterations ?? (step ? (options.candidateResearchIterations ?? 3) : undefined), @@ -367,98 +380,104 @@ async function evaluateCandidate( candidate: KnowledgeImprovementCandidateRecord evaluation: KnowledgeImprovementMetric }> { - return withBaselineSnapshot(runDir, state.baseHash, async (baselineRoot) => { - const [baselineIndex, candidateIndex] = await Promise.all([ - buildKnowledgeIndex(baselineRoot), - buildKnowledgeIndex(snapshot.root), - ]) - const validation = validateKnowledgeIndex(candidateIndex, { strict: options.strict }) - const readiness = readinessFor(options, candidateIndex) - const kbQuality = scoreKnowledgeBaseIndex(candidateIndex, { - strict: options.strict, - ...options.kbQuality, - }) - const candidateHash = snapshot.hash - const evaluator = useConfiguredEvaluator ? options.evaluate : options.evaluateDevelopment - const configuredMetric = evaluator - ? evaluator({ - runId: state.runId, - iteration: candidate.iteration, - root: options.root, - baselineRoot, - candidateRoot: snapshot.root, - baselineIndex, - candidateIndex, - baseHash: state.baseHash, - candidateHash, + return withBaselineSnapshot( + runDir, + state.baseHash, + async (baselineRoot) => { + const [baselineIndex, candidateIndex] = await Promise.all([ + buildKnowledgeIndex(baselineRoot, options.stateScope), + buildKnowledgeIndex(snapshot.root, options.stateScope), + ]) + const validation = validateKnowledgeIndex(candidateIndex, { strict: options.strict }) + const readiness = readinessFor(options, candidateIndex) + const kbQuality = scoreKnowledgeBaseIndex(candidateIndex, { + strict: options.strict, + ...options.kbQuality, + }) + const candidateHash = snapshot.hash + const evaluator = useConfiguredEvaluator ? options.evaluate : options.evaluateDevelopment + const configuredMetric = evaluator + ? evaluator({ + runId: state.runId, + iteration: candidate.iteration, + root: options.root, + baselineRoot, + candidateRoot: snapshot.root, + baselineIndex, + candidateIndex, + baseHash: state.baseHash, + candidateHash, + validation, + readiness, + kbQuality, + lifecycle, + signal: options.signal, + }) + : undefined + const metric = + configuredMetric ?? + defaultKnowledgeImprovementMetric( validation, readiness, + options.readinessSpecs, kbQuality, lifecycle, - signal: options.signal, - }) - : undefined - const metric = - configuredMetric ?? - defaultKnowledgeImprovementMetric( - validation, - readiness, - options.readinessSpecs, - kbQuality, + ) + const evaluation = applyLifecycleFailures( + normalizeMetric(await metric), lifecycle, + options.answerQualityCostCeiling, ) - const evaluation = applyLifecycleFailures( - normalizeMetric(await metric), - lifecycle, - options.answerQualityCostCeiling, - ) - const measuredHash = await hashKnowledgeBase(snapshot.root) - if (measuredHash !== candidateHash) { - throw new Error( - `knowledge candidate changed during evaluation: expected ${candidateHash}, got ${measuredHash}`, + const measuredHash = await hashKnowledgeBase(snapshot.root, options.stateScope) + if (measuredHash !== candidateHash) { + throw new Error( + `knowledge candidate changed during evaluation: expected ${candidateHash}, got ${measuredHash}`, + ) + } + candidate.candidateHash = candidateHash + candidate.promotionPlanHash = knowledgeFileTransactionPlanHash( + await knowledgeFilePlanEntries(baselineRoot, snapshot.root, options.stateScope), + normalizeKnowledgeStateScope(options.stateScope).pagesDirectory, + options.stateScope?.researchState, ) - } - candidate.candidateHash = candidateHash - candidate.promotionPlanHash = knowledgeFileTransactionPlanHash( - await knowledgeFilePlanEntries(baselineRoot, snapshot.root), - KB_IMPROVEMENT_PAGES_DIRECTORY, - ) - const evidence = KnowledgeImprovementEvidenceSchema.parse( - JSON.parse( - JSON.stringify({ - kind: 'knowledge-improvement-evidence', - runId: state.runId, - candidateId: candidate.candidateId, - iteration: candidate.iteration, - goalHash: sha256(state.goal), - implementationRef: state.implementationRef, - baseHash: candidate.baseHash, - candidateHash, - promotionPlanHash: candidate.promotionPlanHash, - validation, - readiness: readiness ?? null, - kbQuality, - evaluation, - lifecycle: lifecycle ?? null, - }), - ), - ) - candidate.evidenceHash = contentHash(evidence) - candidate.updatedAt = now().toISOString() - await writeJsonDurableWithinRoot( - runDir, - candidateEvidenceRelativePath(candidate.candidateId), - evidence, - ) - await appendLedger(runDir, { - type: 'candidate.evaluated', - runId: state.runId, - candidateId: candidate.candidateId, - score: evaluation.score, - passed: evaluation.passed, - }) - return { candidate, evaluation } - }) + const evidence = KnowledgeImprovementEvidenceSchema.parse( + JSON.parse( + JSON.stringify({ + kind: 'knowledge-improvement-evidence', + runId: state.runId, + candidateId: candidate.candidateId, + iteration: candidate.iteration, + goalHash: sha256(state.goal), + implementationRef: state.implementationRef, + baseHash: candidate.baseHash, + candidateHash, + promotionPlanHash: candidate.promotionPlanHash, + validation, + readiness: readiness ?? null, + kbQuality, + evaluation, + lifecycle: lifecycle ?? null, + }), + ), + ) + candidate.evidenceHash = contentHash(evidence) + candidate.updatedAt = now().toISOString() + await writeJsonDurableWithinRoot( + runDir, + candidateEvidenceRelativePath(candidate.candidateId), + evidence, + ) + await appendLedger(runDir, { + type: 'candidate.evaluated', + runId: state.runId, + candidateId: candidate.candidateId, + score: evaluation.score, + passed: evaluation.passed, + }) + return { candidate, evaluation } + }, + options.stateScope, + ) } function defaultKnowledgeImprovementMetric( diff --git a/src/kb-improvement/optimization.ts b/src/kb-improvement/optimization.ts index 0069268..0748ca6 100644 --- a/src/kb-improvement/optimization.ts +++ b/src/kb-improvement/optimization.ts @@ -119,7 +119,7 @@ export async function optimizeKnowledgeBasePolicy< ) { throw new Error('optimizeKnowledgeBasePolicy candidateRunLabel must be non-empty') } - const baseHash = await hashKnowledgeBase(root) + const baseHash = await hashKnowledgeBase(root, candidate?.stateScope) const optimization = await runSerializedKnowledgeOptimization({ ...optimizationOptions, executionRef: policyApplicationRef, @@ -130,7 +130,7 @@ export async function optimizeKnowledgeBasePolicy< finalScenarios, }) const winner = optimization.winner - const currentBaseHash = await hashKnowledgeBase(root) + const currentBaseHash = await hashKnowledgeBase(root, candidate?.stateScope) if (currentBaseHash !== baseHash) { throw new Error( `knowledge base changed during policy optimization: expected ${baseHash}, got ${currentBaseHash}`, diff --git a/src/kb-improvement/run.ts b/src/kb-improvement/run.ts index 953fef7..ad6a26f 100644 --- a/src/kb-improvement/run.ts +++ b/src/kb-improvement/run.ts @@ -1,4 +1,6 @@ +import { canonicalJson } from '@tangle-network/agent-eval' import { isMissingFile } from '../durable-fs' +import { normalizeKnowledgeStateScope } from '../knowledge-state-scope' import { withKnowledgeMutation } from '../mutation-lock' import type { RunRagKnowledgeImprovementLoopResult } from '../rag-improvement-loop' import type { @@ -34,6 +36,12 @@ export async function improveKnowledgeBase( options: KnowledgeImprovementOptions, ): Promise { assertExactCandidatePlatform() + options = { + ...options, + ...(options.stateScope === undefined + ? {} + : { stateScope: Object.freeze(normalizeKnowledgeStateScope(options.stateScope)) }), + } assertKnowledgeImprovementOptions(options) const now = options.now ?? (() => new Date()) const runId = runIdSchema.parse( @@ -65,13 +73,16 @@ async function improveKnowledgeBaseInRun( throw error }) if (!state) { - const baseHash = await hashKnowledgeBase(options.root) - await createBaselineSnapshot(runDir, options.root, baseHash) + const baseHash = await hashKnowledgeBase(options.root, options.stateScope) + await createBaselineSnapshot(runDir, options.root, baseHash, options.stateScope) state = { runId, root: options.root, goal: options.goal, implementationRef: options.implementationRef, + ...(options.stateScope === undefined + ? {} + : { stateScope: normalizeKnowledgeStateScope(options.stateScope) }), status: 'running', baseHash, createdAt: now().toISOString(), @@ -82,6 +93,12 @@ async function improveKnowledgeBaseInRun( await saveState(runDir, state, options.onState) await appendLedger(runDir, { type: 'run.created', runId, baseHash }) } + if ( + canonicalJson(normalizeKnowledgeStateScope(state.stateScope)) !== + canonicalJson(normalizeKnowledgeStateScope(options.stateScope)) + ) { + throw new Error('knowledge improvement state does not match the requested stateScope') + } if (state.goal !== options.goal) { throw new Error('knowledge improvement state does not match the requested goal') } @@ -100,7 +117,7 @@ async function improveKnowledgeBaseInRun( const promoted = promotedCandidate! const promotedState = state return withKnowledgeMutation(options.root, async () => { - const currentHash = await hashKnowledgeBase(options.root) + const currentHash = await hashKnowledgeBase(options.root, options.stateScope) if (currentHash !== promoted.candidateHash) { throw new Error( `promoted knowledge base changed: expected ${promoted.candidateHash}, got ${currentHash}`, @@ -122,7 +139,7 @@ async function improveKnowledgeBaseInRun( }) } await withKnowledgeMutation(options.root, () => undefined) - await ensureBaselineSnapshot(runDir, options.root, state.baseHash) + await ensureBaselineSnapshot(runDir, options.root, state.baseHash, state.stateScope) if (state.status === 'blocked') { return { runId, state, promoted: false, blocked: true } @@ -152,7 +169,7 @@ async function improveKnowledgeBaseInRun( } } if (!candidate) { - const currentHash = await hashKnowledgeBase(options.root) + const currentHash = await hashKnowledgeBase(options.root, options.stateScope) if (currentHash !== state.baseHash) { state = await blockRun( runDir, @@ -164,8 +181,11 @@ async function improveKnowledgeBaseInRun( return { runId, state, promoted: false, blocked: true } } const activeState = state - candidate = await withBaselineSnapshot(runDir, activeState.baseHash, (baselineRoot) => - createCandidateWorkspace(runDir, activeState, baselineRoot, now), + candidate = await withBaselineSnapshot( + runDir, + activeState.baseHash, + (baselineRoot) => createCandidateWorkspace(runDir, activeState, baselineRoot, now), + state.stateScope, ) state.candidates.push(candidate) state.status = 'running' diff --git a/src/kb-improvement/selected-candidate.ts b/src/kb-improvement/selected-candidate.ts index dd23233..7361b59 100644 --- a/src/kb-improvement/selected-candidate.ts +++ b/src/kb-improvement/selected-candidate.ts @@ -16,7 +16,9 @@ import { } from '../file-transaction' import { stableId } from '../ids' import { writeKnowledgeIndex } from '../indexer' +import { type KnowledgeStateScope, normalizeKnowledgeStateScope } from '../knowledge-state-scope' import { withKnowledgeMutation } from '../mutation-lock' +import { normalizePagesDirectory } from '../pages-directory' import type { RagKnowledgeImprovementPhase } from '../rag-improvement-loop' import type { KnowledgeImprovementCandidateRef, @@ -35,13 +37,16 @@ import { knowledgeImprovementRunDir } from './state' import { knowledgeFilePlanEntries } from './transition' import { hashKnowledgeBase, withKnowledgeImprovementComparison } from './workspace' -const DERIVED_KNOWLEDGE_PATHS = new Set([`${KB_IMPROVEMENT_PAGES_DIRECTORY}/index.md`]) const selectionPathSchema = z .string() .min(1) .transform((path, context) => { try { - return assertKnowledgeMutationPath(path, KB_IMPROVEMENT_PAGES_DIRECTORY) + try { + return assertKnowledgeMutationPath(path, KB_IMPROVEMENT_PAGES_DIRECTORY, true) + } catch { + return normalizePagesDirectory(path) + } } catch (error) { context.addIssue({ code: 'custom', @@ -163,16 +168,30 @@ export async function improveSelectedKnowledgeCandidate( return withKnowledgeImprovementComparison( { root: options.root, candidate: sourceCandidate }, async (source) => { - const sourcePlan = await knowledgeFilePlanEntries(source.baseline.root, source.candidate.root) + const scope = normalizeKnowledgeStateScope(source.stateScope) + if ( + options.stateScope !== undefined && + canonicalJson(normalizeKnowledgeStateScope(options.stateScope)) !== canonicalJson(scope) + ) { + throw new Error('selected knowledge candidate stateScope differs from its source') + } + const sourcePlan = await knowledgeFilePlanEntries( + source.baseline.root, + source.candidate.root, + scope, + ) const sourcePlanHash = knowledgeFileTransactionPlanHash( sourcePlan, - KB_IMPROVEMENT_PAGES_DIRECTORY, + scope.pagesDirectory, + scope.researchState, ) if (sourcePlanHash !== sourceCandidate.promotionPlanHash) { throw new Error('source knowledge candidate plan no longer matches its measured identity') } - const changedSourcePlan = sourcePlan.filter(planEntryChanged).filter(notDerivedPath) - const selectedPaths = normalizeSelectedPaths(options.selectedPaths, changedSourcePlan) + const changedSourcePlan = sourcePlan + .filter(planEntryChanged) + .filter((entry) => notDerivedPath(entry, scope)) + const selectedPaths = normalizeSelectedPaths(options.selectedPaths, changedSourcePlan, scope) const selectionMaterial = immutableJson({ kind: 'measured-knowledge-change-selection' as const, version: 1 as const, @@ -208,13 +227,15 @@ export async function improveSelectedKnowledgeCandidate( }) const selectionMutationPlanHash = knowledgeFileTransactionPlanHash( selectedEntries, - KB_IMPROVEMENT_PAGES_DIRECTORY, + scope.pagesDirectory, + scope.researchState, ) let lifecycleSelection: z.infer | undefined const result = await improveKnowledgeBase({ ...improvementOptions(options), root: options.root, + stateScope: scope, goal: options.goal, implementationRef: derivedImplementationRef, runId, @@ -232,6 +253,7 @@ export async function improveSelectedKnowledgeCandidate( root: input.candidateRoot, transactionRoot: lock.transactionRoot, purpose, + ...scope, recoveryOwner, mutations: await selectionMutations(source.candidate.root, selectedEntries), now: options.now, @@ -252,7 +274,7 @@ export async function improveSelectedKnowledgeCandidate( }) } } - await writeKnowledgeIndex(input.candidateRoot) + await writeKnowledgeIndex(input.candidateRoot, scope) lock.assertOwned() }, { @@ -268,13 +290,15 @@ export async function improveSelectedKnowledgeCandidate( const selectedPlan = await knowledgeFilePlanEntries( source.baseline.root, input.candidateRoot, + scope, ) - assertExactSelectedChanges(selectedPlan, selectedPaths) + assertExactSelectedChanges(selectedPlan, selectedPaths, scope) const selectedPlanHash = knowledgeFileTransactionPlanHash( selectedPlan, - KB_IMPROVEMENT_PAGES_DIRECTORY, + scope.pagesDirectory, + scope.researchState, ) - const selectedCandidateHash = await hashKnowledgeBase(input.candidateRoot) + const selectedCandidateHash = await hashKnowledgeBase(input.candidateRoot, scope) lifecycleSelection = measuredSelectionLifecycleSchema.parse({ kind: 'measured-knowledge-change-selection', version: 1, @@ -387,13 +411,18 @@ function normalizeEvaluationPhases( function normalizeSelectedPaths( paths: readonly string[], changedPlan: readonly KnowledgeFileTransactionPlanEntry[], + scope: KnowledgeStateScope, ): string[] { const available = new Set(changedPlan.map((entry) => entry.path)) const selected: string[] = [] const seen = new Set() for (const input of paths) { - const path = selectionPathSchema.parse(input) - if (DERIVED_KNOWLEDGE_PATHS.has(path)) { + const path = assertKnowledgeMutationPath( + selectionPathSchema.parse(input), + normalizeKnowledgeStateScope(scope).pagesDirectory, + scope.researchState, + ) + if (!notDerivedPath({ path }, scope)) { throw new Error(`derived knowledge path cannot be selected directly: ${path}`) } if (seen.has(path)) throw new Error(`selected knowledge path is repeated: ${path}`) @@ -413,8 +442,8 @@ function planEntryChanged(entry: KnowledgeFileTransactionPlanEntry): boolean { ) } -function notDerivedPath(entry: KnowledgeFileTransactionPlanEntry): boolean { - return !DERIVED_KNOWLEDGE_PATHS.has(entry.path) +function notDerivedPath(entry: { path: string }, scope: KnowledgeStateScope): boolean { + return entry.path !== `${normalizeKnowledgeStateScope(scope).pagesDirectory}/index.md` } async function selectionMutations( @@ -439,8 +468,11 @@ function assertSelectionTransaction( expectedPlanHash: string, ): void { if ( - knowledgeFileTransactionPlanHash(transaction.entries, KB_IMPROVEMENT_PAGES_DIRECTORY) !== - expectedPlanHash + knowledgeFileTransactionPlanHash( + transaction.entries, + normalizePagesDirectory(transaction.pagesDirectory), + transaction.researchState, + ) !== expectedPlanHash ) { throw new Error('selected knowledge transaction does not match its approved path set') } @@ -449,10 +481,11 @@ function assertSelectionTransaction( function assertExactSelectedChanges( plan: readonly KnowledgeFileTransactionPlanEntry[], selectedPaths: readonly string[], + scope: KnowledgeStateScope, ): void { const actual = plan .filter(planEntryChanged) - .filter(notDerivedPath) + .filter((entry) => notDerivedPath(entry, scope)) .map((entry) => entry.path) .sort((left, right) => left.localeCompare(right)) if (canonicalJson(actual) !== canonicalJson(selectedPaths)) { diff --git a/src/kb-improvement/transition.ts b/src/kb-improvement/transition.ts index bcc49a8..67810b4 100644 --- a/src/kb-improvement/transition.ts +++ b/src/kb-improvement/transition.ts @@ -14,6 +14,7 @@ import { rollbackKnowledgeFileTransaction, } from '../file-transaction' import { writeKnowledgeIndex } from '../indexer' +import { type KnowledgeStateScope, normalizeKnowledgeStateScope } from '../knowledge-state-scope' import { withKnowledgeMutation } from '../mutation-lock' import type { RunRagKnowledgeImprovementLoopResult } from '../rag-improvement-loop' import type { ResolvedKnowledgeImprovementActivationPersistence } from './activation' @@ -34,11 +35,7 @@ import type { PromoteKnowledgeCandidateOptions, RestoreKnowledgeCandidateBaselineOptions, } from './contracts' -import { - DEFAULT_LEASE_TTL_MS, - KB_IMPROVEMENT_PAGES_DIRECTORY, - KnowledgeImprovementCandidateRefSchema, -} from './contracts' +import { DEFAULT_LEASE_TTL_MS, KnowledgeImprovementCandidateRefSchema } from './contracts' import { acquireRunLease, appendLedger, @@ -178,7 +175,7 @@ async function applyKnowledgeCandidateTarget( throw new Error('stored knowledge activation result conflicts with a pending rollback') } if (recovery) { - const recoveredHash = await hashKnowledgeBase(input.root) + const recoveredHash = await hashKnowledgeBase(input.root, state.stateScope) if (recoveredHash !== existingActivation.mutation.afterHash) { throw new Error('stored knowledge activation result does not match the recovered files') } @@ -224,7 +221,7 @@ async function applyKnowledgeCandidateTarget( const recovered = recovery?.direction === 'apply' ? recovery : undefined let pending: KnowledgeFileTransaction | null = input.activation && recovered ? recovered.transaction : null - const currentHash = await hashKnowledgeBase(input.root) + const currentHash = await hashKnowledgeBase(input.root, state.stateScope) let transactionId = recovered?.transactionId ?? null if (recovered && currentHash !== desiredHash) { throw new Error(`recovered knowledge ${action} does not match the approved target`) @@ -287,28 +284,38 @@ async function applyKnowledgeCandidateTarget( state, candidateRef, (resolved) => - withBaselineSnapshot(runDir, state.baseHash, async (baselineRoot) => { - const sourceRoot = target === 'candidate' ? baselineRoot : resolved.root - const targetRoot = target === 'candidate' ? resolved.root : baselineRoot - const plan = await knowledgeFilePlanEntries(sourceRoot, targetRoot) - assertCandidateTransitionPlan(plan, candidateRef, target) - return prepareKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - purpose, - recoveryOwner, - mutations: await knowledgePlanMutations(targetRoot, plan), - includeUnchanged: true, - now: input.now, - }) - }), + withBaselineSnapshot( + runDir, + state.baseHash, + async (baselineRoot) => { + const sourceRoot = target === 'candidate' ? baselineRoot : resolved.root + const targetRoot = target === 'candidate' ? resolved.root : baselineRoot + const plan = await knowledgeFilePlanEntries( + sourceRoot, + targetRoot, + state.stateScope, + ) + assertCandidateTransitionPlan(plan, candidateRef, target, state.stateScope) + return prepareKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + purpose, + ...normalizeKnowledgeStateScope(state.stateScope), + recoveryOwner, + mutations: await knowledgePlanMutations(targetRoot, plan), + includeUnchanged: true, + now: input.now, + }) + }, + state.stateScope, + ), ) if (!pending) { throw new Error(`knowledge ${action} plan unexpectedly contained no file changes`) } transactionId = pending.transactionId try { - assertCandidateTransitionTransaction(pending, candidateRef, target) + assertCandidateTransitionTransaction(pending, candidateRef, target, state.stateScope) } catch (error) { try { await rollbackKnowledgeFileTransaction({ @@ -342,10 +349,10 @@ async function applyKnowledgeCandidateTarget( }) } assertOwned() - if ((await hashKnowledgeBase(input.root)) !== desiredHash) { + if ((await hashKnowledgeBase(input.root, state.stateScope)) !== desiredHash) { throw new Error(`knowledge ${action} content does not match the approved target`) } - await writeKnowledgeIndex(input.root) + await writeKnowledgeIndex(input.root, state.stateScope) } catch (error) { if (!pending) throw error if (recovered && input.activation) throw error @@ -370,7 +377,7 @@ async function applyKnowledgeCandidateTarget( transaction: pending, assertOwned, }) - await writeKnowledgeIndex(input.root) + await writeKnowledgeIndex(input.root, state.stateScope) } catch (rollbackError) { throw new AggregateError( [error, rollbackError], @@ -388,7 +395,7 @@ async function applyKnowledgeCandidateTarget( state.updatedAt = input.now().toISOString() await saveState(runDir, state, input.onState) await ensureCandidateTransitionEvent(runDir, candidateRef, target) - let finalHash = await hashKnowledgeBase(input.root) + let finalHash = await hashKnowledgeBase(input.root, state.stateScope) if (finalHash !== desiredHash) { throw new Error(`knowledge ${action} changed before its result was returned`) } @@ -409,7 +416,7 @@ async function applyKnowledgeCandidateTarget( mutation, ) : undefined - if ((await hashKnowledgeBase(input.root)) !== finalHash) { + if ((await hashKnowledgeBase(input.root, state.stateScope)) !== finalHash) { throw new Error(`knowledge ${action} changed while its result was persisted`) } if (pending) { @@ -420,7 +427,7 @@ async function applyKnowledgeCandidateTarget( assertOwned, }) } - finalHash = await hashKnowledgeBase(input.root) + finalHash = await hashKnowledgeBase(input.root, state.stateScope) if (finalHash !== desiredHash) { throw new Error(`knowledge ${action} changed before its result was returned`) } @@ -439,7 +446,7 @@ async function applyKnowledgeCandidateTarget( purpose, recoveryOwner, validate: (transaction) => - assertCandidateTransitionTransaction(transaction, candidateRef, target), + assertCandidateTransitionTransaction(transaction, candidateRef, target, state.stateScope), deferFinish: input.activation !== undefined, }, }, @@ -517,10 +524,11 @@ function knowledgeCandidateTransitionPurpose( export async function knowledgeFilePlanEntries( sourceRoot: string, targetRoot: string, + scope?: KnowledgeStateScope, ): Promise { const [before, after] = await Promise.all([ - knowledgeHashEntries(sourceRoot), - knowledgeHashEntries(targetRoot), + knowledgeHashEntries(sourceRoot, scope), + knowledgeHashEntries(targetRoot, scope), ]) const beforeByPath = new Map(before.map((entry) => [entry.path, entry])) const afterByPath = new Map(after.map((entry) => [entry.path, entry])) @@ -528,7 +536,11 @@ export async function knowledgeFilePlanEntries( ...new Set([...before.map((entry) => entry.path), ...after.map((entry) => entry.path)]), ].sort((left, right) => left.localeCompare(right)) return paths.map((path) => { - assertKnowledgeMutationPath(path, KB_IMPROVEMENT_PAGES_DIRECTORY) + assertKnowledgeMutationPath( + path, + normalizeKnowledgeStateScope(scope).pagesDirectory, + scope?.researchState, + ) const beforeEntry = beforeByPath.get(path) const afterEntry = afterByPath.get(path) return { @@ -562,11 +574,13 @@ function assertCandidateTransitionPlan( plan: readonly KnowledgeFileTransactionPlanEntry[], candidate: KnowledgeImprovementCandidateRef, target: KnowledgeImprovementTarget, + scope?: KnowledgeStateScope, ): void { const approvedDirection = target === 'candidate' ? plan : reverseKnowledgeFilePlan(plan) const actualPlanHash = knowledgeFileTransactionPlanHash( approvedDirection, - KB_IMPROVEMENT_PAGES_DIRECTORY, + normalizeKnowledgeStateScope(scope).pagesDirectory, + scope?.researchState, ) if (actualPlanHash !== candidate.promotionPlanHash) { throw new Error( @@ -579,8 +593,9 @@ function assertCandidateTransitionTransaction( transaction: KnowledgeFileTransaction, candidate: KnowledgeImprovementCandidateRef, target: KnowledgeImprovementTarget, + scope?: KnowledgeStateScope, ): void { - assertCandidateTransitionPlan(transaction.entries, candidate, target) + assertCandidateTransitionPlan(transaction.entries, candidate, target, scope) } function reverseKnowledgeFilePlan( diff --git a/src/kb-improvement/workspace.ts b/src/kb-improvement/workspace.ts index 2cadd24..71ccd3f 100644 --- a/src/kb-improvement/workspace.ts +++ b/src/kb-improvement/workspace.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto' import { cp, lstat, mkdir, mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { dirname, join, relative } from 'node:path' +import { dirname, join } from 'node:path' import { canonicalJson, contentHash } from '@tangle-network/agent-eval' import { canonicalPathsEqual, @@ -13,8 +13,12 @@ import { } from '../durable-fs' import { sha256, stableId } from '../ids' import { writeKnowledgeIndex } from '../indexer' +import { + KNOWLEDGE_RESEARCH_STATE_PATHS, + type KnowledgeStateScope, + normalizeKnowledgeStateScope, +} from '../knowledge-state-scope' import { withKnowledgeRead } from '../mutation-lock' -import { layoutFor } from '../store' import { immutableJsonValue } from './activation' import type { KnowledgeImprovementCandidateRecord, @@ -57,23 +61,34 @@ export async function withKnowledgeImprovementComparison( return withKnowledgeImprovementRun(options.root, reference.runId, false, async (runDir) => { const state = await loadKnowledgeImprovementStateFromRun(options.root, reference.runId, runDir) return withMeasuredCandidateSnapshot(options.root, runDir, state, reference, (resolved) => - withBaselineSnapshot(runDir, reference.baseHash, (baselineRoot) => - withIsolatedKnowledgeCopy(baselineRoot, reference.baseHash, 'baseline', (baseline) => + withBaselineSnapshot( + runDir, + reference.baseHash, + (baselineRoot) => withIsolatedKnowledgeCopy( - resolved.root, - reference.candidateHash, - 'candidate', - (candidate) => - use( - Object.freeze({ - reference, - evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), - baseline: Object.freeze({ root: baseline, hash: reference.baseHash }), - candidate: Object.freeze({ root: candidate, hash: reference.candidateHash }), - }), + baselineRoot, + reference.baseHash, + 'baseline', + (baseline) => + withIsolatedKnowledgeCopy( + resolved.root, + reference.candidateHash, + 'candidate', + (candidate) => + use( + Object.freeze({ + reference, + stateScope: Object.freeze(normalizeKnowledgeStateScope(state.stateScope)), + evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), + baseline: Object.freeze({ root: baseline, hash: reference.baseHash }), + candidate: Object.freeze({ root: candidate, hash: reference.candidateHash }), + }), + ), + state.stateScope, ), + state.stateScope, ), - ), + state.stateScope, ), ) }) @@ -95,14 +110,20 @@ export async function withKnowledgeImprovementCandidate( runDir, ) return withMeasuredCandidateSnapshot(options.root, runDir, state, candidateRef, (resolved) => - withIsolatedKnowledgeCopy(resolved.root, candidateRef.candidateHash, 'candidate', (root) => - use( - Object.freeze({ - root, - candidate: candidateRef, - evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), - }), - ), + withIsolatedKnowledgeCopy( + resolved.root, + candidateRef.candidateHash, + 'candidate', + (root) => + use( + Object.freeze({ + root, + candidate: candidateRef, + stateScope: Object.freeze(normalizeKnowledgeStateScope(state.stateScope)), + evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), + }), + ), + state.stateScope, ), ) }) @@ -173,11 +194,11 @@ export async function withMeasuredCandidateSnapshot( candidateRef.candidateHash, ) return withSafeDirectory(runDir, relativePath, false, async (root) => { - if ((await hashKnowledgeBase(root)) !== candidateRef.candidateHash) { + if ((await hashKnowledgeBase(root, state.stateScope)) !== candidateRef.candidateHash) { throw new Error('knowledge candidate snapshot changed after approval') } const result = await use({ root, candidate, evidence }) - if ((await hashKnowledgeBase(root)) !== candidateRef.candidateHash) { + if ((await hashKnowledgeBase(root, state.stateScope)) !== candidateRef.candidateHash) { throw new Error('knowledge candidate snapshot changed during use') } return result @@ -189,16 +210,17 @@ async function withIsolatedKnowledgeCopy( expectedHash: string, target: KnowledgeImprovementTarget, use: (root: string) => Promise | T, + scope?: KnowledgeStateScope, ): Promise { const isolationRoot = await mkdtemp(join(tmpdir(), 'agent-knowledge-snapshot-')) const snapshotRoot = join(isolationRoot, 'snapshot') try { - await copyKnowledgeWorkspace(sourceRoot, snapshotRoot) - if ((await hashKnowledgeBase(snapshotRoot)) !== expectedHash) { + await copyKnowledgeWorkspace(sourceRoot, snapshotRoot, scope) + if ((await hashKnowledgeBase(snapshotRoot, scope)) !== expectedHash) { throw new Error(`isolated knowledge ${target} does not match its measured content`) } const result = await use(snapshotRoot) - if ((await hashKnowledgeBase(snapshotRoot)) !== expectedHash) { + if ((await hashKnowledgeBase(snapshotRoot, scope)) !== expectedHash) { throw new Error(`knowledge ${target} snapshot changed during use`) } return result @@ -271,7 +293,7 @@ export async function createCandidateWorkspace( const iteration = state.candidates.length + 1 const candidateId = stableId('kcand', `${state.runId}:${iteration}:${now().toISOString()}`) const candidateRoot = candidateWorkspacePath(runDir, candidateId) - await copyKnowledgeWorkspace(root, candidateRoot) + await copyKnowledgeWorkspace(root, candidateRoot, state.stateScope) const createdAt = now().toISOString() return { iteration, @@ -295,10 +317,11 @@ export async function createBaselineSnapshot( runDir: string, root: string, expectedHash: string, + scope?: KnowledgeStateScope, ): Promise { const target = baselineSnapshotPath(runDir) try { - await assertBaselineSnapshot(runDir, expectedHash) + await assertBaselineSnapshot(runDir, expectedHash, scope) return } catch (error) { if (!isMissingFile(error)) throw error @@ -306,8 +329,8 @@ export async function createBaselineSnapshot( const preparation = await mkdtemp(join(runDir, 'baseline-prepare-')) let activated = false try { - await copyKnowledgeWorkspace(root, preparation) - const actualHash = await hashKnowledgeBase(preparation) + await copyKnowledgeWorkspace(root, preparation, scope) + const actualHash = await hashKnowledgeBase(preparation, scope) if (actualHash !== expectedHash) { throw new Error( `knowledge base changed while baseline was frozen: expected ${expectedHash}, got ${actualHash}`, @@ -324,32 +347,38 @@ export async function ensureBaselineSnapshot( runDir: string, root: string, expectedHash: string, + scope?: KnowledgeStateScope, ): Promise { try { - await assertBaselineSnapshot(runDir, expectedHash) + await assertBaselineSnapshot(runDir, expectedHash, scope) } catch (error) { if (!isMissingFile(error)) throw error - const liveHash = await hashKnowledgeBase(root) + const liveHash = await hashKnowledgeBase(root, scope) if (liveHash !== expectedHash) { throw new Error( 'knowledge improvement baseline snapshot is missing and cannot be reconstructed', ) } - await createBaselineSnapshot(runDir, root, expectedHash) + await createBaselineSnapshot(runDir, root, expectedHash, scope) } } -async function assertBaselineSnapshot(runDir: string, expectedHash: string): Promise { - await withBaselineSnapshot(runDir, expectedHash, () => undefined) +async function assertBaselineSnapshot( + runDir: string, + expectedHash: string, + scope?: KnowledgeStateScope, +): Promise { + await withBaselineSnapshot(runDir, expectedHash, () => undefined, scope) } export async function withBaselineSnapshot( runDir: string, expectedHash: string, use: (baselineRoot: string) => Promise | T, + scope?: KnowledgeStateScope, ): Promise { return withSafeDirectory(runDir, 'baseline', false, async (baselineRoot) => { - const actualHash = await hashKnowledgeBase(baselineRoot) + const actualHash = await hashKnowledgeBase(baselineRoot, scope) if (actualHash !== expectedHash) { throw new Error( `knowledge improvement baseline changed: expected ${expectedHash}, got ${actualHash}`, @@ -364,6 +393,7 @@ export async function withFrozenCandidateWorkspace( candidate: KnowledgeImprovementCandidateRecord, candidateRoot: string, use: (snapshot: { root: string; hash: string }) => Promise | T, + scope?: KnowledgeStateScope, ): Promise { const snapshotsPath = join( 'candidates', @@ -374,11 +404,11 @@ export async function withFrozenCandidateWorkspace( const preparation = await mkdtemp(join(snapshotsDir, 'prepare-')) let activated = false try { - await copyKnowledgeWorkspace(candidateRoot, preparation) - const hash = await hashKnowledgeBase(preparation) + await copyKnowledgeWorkspace(candidateRoot, preparation, scope) + const hash = await hashKnowledgeBase(preparation, scope) try { const result = await withSafeDirectory(snapshotsDir, hash, false, async (existing) => { - if ((await hashKnowledgeBase(existing)) !== hash) { + if ((await hashKnowledgeBase(existing, scope)) !== hash) { throw new Error('knowledge candidate snapshot does not match its content identity') } return use({ root: existing, hash }) @@ -404,17 +434,31 @@ export function clearCandidateMeasurement(candidate: KnowledgeImprovementCandida delete candidate.promotionPlanHash } -async function copyKnowledgeWorkspace(sourceRoot: string, targetRoot: string): Promise { - await rm(targetRoot, { recursive: true, force: true }) - await mkdir(join(targetRoot, 'knowledge'), { recursive: true }) - await mkdir(join(targetRoot, 'raw', 'sources'), { recursive: true }) - await copyIfExists(join(sourceRoot, 'knowledge'), join(targetRoot, 'knowledge')) - await copyIfExists(join(sourceRoot, 'raw'), join(targetRoot, 'raw')) - await copyIfExists( - join(layoutFor(sourceRoot).cacheDir, 'sources.json'), - join(layoutFor(targetRoot).cacheDir, 'sources.json'), - ) - await writeKnowledgeIndex(targetRoot) +async function copyKnowledgeWorkspace( + sourceRoot: string, + targetRoot: string, + scope?: KnowledgeStateScope, +): Promise { + const normalized = normalizeKnowledgeStateScope(scope) + await withKnowledgeRead(sourceRoot, async () => { + await rm(targetRoot, { recursive: true, force: true }) + await mkdir(join(targetRoot, normalized.pagesDirectory), { recursive: true }) + await mkdir(join(targetRoot, 'raw', 'sources'), { recursive: true }) + for (const path of knowledgeScopePaths(normalized)) { + await copyIfExists(join(sourceRoot, path), join(targetRoot, path)) + } + }) + await writeKnowledgeIndex(targetRoot, { pagesDirectory: normalized.pagesDirectory }) +} + +function knowledgeScopePaths(scope: KnowledgeStateScope): string[] { + const normalized = normalizeKnowledgeStateScope(scope) + return [ + normalized.pagesDirectory, + 'raw', + '.agent-knowledge/sources.json', + ...(normalized.researchState ? KNOWLEDGE_RESEARCH_STATE_PATHS : []), + ] } async function copyIfExists(source: string, target: string): Promise { @@ -432,13 +476,27 @@ async function copyIfExists(source: string, target: string): Promise { await cp(source, target, { recursive: sourceStat.isDirectory(), dereference: false }) } -export async function hashKnowledgeBase(root: string): Promise { - return withKnowledgeRead(root, () => hashKnowledgeBaseUnlocked(root)) +export async function hashKnowledgeBase( + root: string, + scope?: KnowledgeStateScope, +): Promise { + return withKnowledgeRead(root, () => hashKnowledgeBaseUnlocked(root, scope)) } -async function hashKnowledgeBaseUnlocked(root: string): Promise { - const entries = await knowledgeHashEntries(root) - return sha256(JSON.stringify(entries.map(({ path, hash, mode }) => ({ path, hash, mode })))) +async function hashKnowledgeBaseUnlocked( + root: string, + scope?: KnowledgeStateScope, +): Promise { + const normalized = normalizeKnowledgeStateScope(scope) + const entries = await knowledgeHashEntries(root, normalized) + const files = entries.map(({ path, hash, mode }) => ({ path, hash, mode })) + return sha256( + JSON.stringify( + normalized.pagesDirectory === 'knowledge' && !normalized.researchState + ? files + : { scope: normalized, files }, + ), + ) } export interface KnowledgeFileIdentity { @@ -448,24 +506,23 @@ export interface KnowledgeFileIdentity { mode: number } -export async function knowledgeHashEntries(root: string): Promise { +export async function knowledgeHashEntries( + root: string, + scope?: KnowledgeStateScope, +): Promise { const entries: KnowledgeFileIdentity[] = [] - for (const rel of ['knowledge', 'raw']) { + for (const path of knowledgeScopePaths(scope ?? {})) { try { - for (const file of await listRegularFilesWithinRoot(root, rel)) { + const files = + path === '.agent-knowledge/sources.json' || path === '.agent-knowledge/events.json' + ? [{ ...(await readRegularFileWithinRoot(root, path)), path }] + : await listRegularFilesWithinRoot(root, path) + for (const file of files) entries.push(knowledgeFileIdentity(file.path, file.bytes, file.mode)) - } } catch (error) { if (!isMissingFile(error)) throw error } } - const sourceRegistry = relative(root, layoutFor(root).sourceRegistryPath).replace(/\\/g, '/') - try { - const file = await readRegularFileWithinRoot(root, sourceRegistry) - entries.push(knowledgeFileIdentity(sourceRegistry, file.bytes, file.mode)) - } catch (error) { - if (!isMissingFile(error)) throw error - } entries.sort((a, b) => a.path.localeCompare(b.path)) return entries } diff --git a/src/knowledge-state-scope.ts b/src/knowledge-state-scope.ts new file mode 100644 index 0000000..b1255e2 --- /dev/null +++ b/src/knowledge-state-scope.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' +import { normalizePagesDirectory } from './pages-directory' + +/** Authoritative local state included in a KB improvement candidate. */ +export interface KnowledgeStateScope { + /** Root-relative Markdown pages directory. Defaults to `knowledge`. */ + pagesDirectory?: string + /** Include claim ledgers and research events. Excludes indexes, locks, and receipt artifacts. */ + researchState?: boolean +} + +export const knowledgeStateScopeSchema = z + .object({ + pagesDirectory: z + .string() + .transform((value) => normalizePagesDirectory(value)) + .optional(), + researchState: z.boolean().optional(), + }) + .strict() + +export function normalizeKnowledgeStateScope(scope: KnowledgeStateScope = {}) { + const parsed = knowledgeStateScopeSchema.parse(scope) + return { + pagesDirectory: normalizePagesDirectory(parsed.pagesDirectory), + researchState: parsed.researchState ?? false, + } +} + +export const KNOWLEDGE_RESEARCH_STATE_PATHS = [ + '.agent-knowledge/claim-ledgers', + '.agent-knowledge/events.json', +] as const diff --git a/src/knowledge-tools.test.ts b/src/knowledge-tools.test.ts index ddc2c22..530cc05 100644 --- a/src/knowledge-tools.test.ts +++ b/src/knowledge-tools.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, readdir, readFile, realpath, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { ToolDefinition } from '@tangle-network/agent-interface' @@ -7,6 +7,7 @@ import { KnowledgeCitationResolutionError } from './citation-resolution' import { createKnowledgeTools } from './knowledge-tools' import { assertKnowledgeRetrievalMatchesVisibility, + assertKnowledgeRetrievalMatchesVisibilityArtifact, createKnowledgeRetrievalDisposition, createKnowledgeVisibilitySnapshot, type KnowledgeRetrievalReceipt, @@ -53,6 +54,39 @@ async function writePage(storeRoot: string, id: string, body: string) { } describe('createKnowledgeTools', () => { + it('persists one exact visibility artifact for concurrent receipts and verifies it after mutation', async () => { + await writePage(stores.storePath('run-a'), 'budget', 'Retry budget is three attempts.') + await Promise.all([ + call('knowledge_search', { question: 'retry budget' }), + call('knowledge_search', { question: 'three attempts' }), + ]) + const first = recorded[0]! + expect(first.visibility.artifact).toBeDefined() + expect(recorded[1]!.visibility.artifact).toEqual(first.visibility.artifact) + expect( + await readdir(join(stores.storePath('run-a'), '.agent-knowledge/retrieval-visibility')), + ).toHaveLength(1) + await writePage(stores.storePath('run-a'), 'budget', 'Retry budget is now five attempts.') + await assertKnowledgeRetrievalMatchesVisibilityArtifact(first, (ref) => + readFile(new URL(ref.uri)), + ) + await call('knowledge_search', { question: 'retry budget' }) + expect(recorded[2]!.visibility.artifact).not.toEqual(first.visibility.artifact) + await assertKnowledgeRetrievalMatchesVisibilityArtifact(recorded[2]!, (ref) => + readFile(new URL(ref.uri)), + ) + }) + + it('refuses corrupt stored visibility bytes before recording another receipt', async () => { + await writePage(stores.storePath('run-a'), 'budget', 'Retry budget is three attempts.') + await call('knowledge_search', { question: 'retry budget' }) + await writeFile(new URL(recorded[0]!.visibility.artifact!.uri), 'corrupt') + await expect(call('knowledge_search', { question: 'retry budget' })).rejects.toThrow( + 'content identity', + ) + expect(recorded).toHaveLength(1) + }) + it('mints a retrieval receipt the visibility snapshot verifies', async () => { await writePage( stores.storePath('run-a'), diff --git a/src/knowledge-tools.ts b/src/knowledge-tools.ts index cd8f2ff..737cbb9 100644 --- a/src/knowledge-tools.ts +++ b/src/knowledge-tools.ts @@ -6,9 +6,12 @@ * so no provider vocabulary reaches this package and no knowledge behavior * reaches a provider adapter. * - * Every search mints a retrieval receipt, so what an actor was shown is - * recorded by the infrastructure rather than claimed by the run. + * Every search mints a retrieval receipt. Configured receipt capture also + * persists the exact visibility snapshot before delivering the receipt. */ + +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' import type { ToolDefinition } from '@tangle-network/agent-interface' import { z } from 'zod' import { @@ -16,12 +19,16 @@ import { resolveKnowledgeCitation, resolveRunScopedCitations, } from './citation-resolution' +import { isMissingFile, readRegularFileWithinRoot, writeFileDurableWithinRoot } from './durable-fs' import { buildKnowledgeBrief, type KnowledgeBriefOptions } from './knowledge-brief' import { createKnowledgeRetrievalReceipt, createKnowledgeVisibilitySnapshot, + encodeKnowledgeVisibilitySnapshot, type KnowledgeRetrievalReceipt, + knowledgeVisibilityArtifactRef, } from './knowledge-use-receipts' +import { withKnowledgeMutation } from './mutation-lock' import { applyKnowledgeWriteBlocks, type KnowledgeWriteIntakeRequest } from './proposals' import type { OriginatedPage, RunScopedStores } from './run-scoped' @@ -41,7 +48,7 @@ export interface CreateKnowledgeToolsOptions { readonly intake?: Omit /** Brief settings for `knowledge_search`, overridden per call by the tool input. */ readonly brief?: Omit - /** Durable sink for every minted retrieval receipt. */ + /** Receipt sink. Exact visibility bytes are persisted in the run store before this is called. */ readonly recordRetrieval?: (receipt: KnowledgeRetrievalReceipt) => Promise | void readonly now?: () => Date } @@ -83,6 +90,10 @@ export function createKnowledgeTools(options: CreateKnowledgeToolsOptions): Tool ...options.brief, ...(input.limit === undefined ? {} : { limit: input.limit }), }) + const visibility = createKnowledgeVisibilitySnapshot(chain) + const visibilityArtifact = options.recordRetrieval + ? await persistVisibility(stores.storePath(runId), visibility) + : undefined const receipt = createKnowledgeRetrievalReceipt({ runId, ...(options.actorId === undefined ? {} : { actorId: options.actorId }), @@ -92,7 +103,8 @@ export function createKnowledgeTools(options: CreateKnowledgeToolsOptions): Tool version: options.retrieverVersion, configDigest: brief.retrieverConfigDigest, }, - visibility: createKnowledgeVisibilitySnapshot(chain), + visibility, + ...(visibilityArtifact ? { visibilityArtifact } : {}), results: brief.results, createdAt: options.now?.(), }) @@ -201,3 +213,24 @@ function tool( handler: (input: unknown) => handler(inputSchema.parse(input)), } } + +/** Snapshot artifacts are immutable evidence, separate from authoritative KB state. */ +async function persistVisibility( + root: string, + snapshot: ReturnType, +) { + const bytes = encodeKnowledgeVisibilitySnapshot(snapshot) + const path = `.agent-knowledge/retrieval-visibility/${snapshot.snapshotDigest.replace('sha256:', '')}.json` + await withKnowledgeMutation(root, async () => { + try { + const existing = await readRegularFileWithinRoot(root, path) + if (!Buffer.from(existing.bytes).equals(Buffer.from(bytes))) { + throw new Error('stored knowledge visibility artifact does not match its content identity') + } + } catch (error) { + if (!isMissingFile(error)) throw error + await writeFileDurableWithinRoot(root, path, Buffer.from(bytes)) + } + }) + return knowledgeVisibilityArtifactRef({ uri: pathToFileURL(join(root, path)).href, bytes }) +} diff --git a/src/research-loop.ts b/src/research-loop.ts index ae689ff..c9ff7da 100644 --- a/src/research-loop.ts +++ b/src/research-loop.ts @@ -83,6 +83,7 @@ export interface KnowledgeResearchLoopStep { } export interface RunKnowledgeResearchLoopOptions { + pagesDirectory?: string root: string goal: string maxIterations?: number @@ -116,6 +117,7 @@ export type KnowledgeControlLoopAction = KnowledgeResearchLoopDecision export type KnowledgeControlLoopActionResult = KnowledgeResearchLoopStep export interface KnowledgeControlLoopAdapterOptions { + pagesDirectory?: string root: string goal: string actor?: string @@ -155,7 +157,7 @@ export function createKnowledgeControlLoopAdapter( await initKnowledgeBase(options.root) initialized = true } - const index = await buildKnowledgeIndex(options.root) + const index = await buildKnowledgeIndex(options.root, options) const validation = validateKnowledgeIndex(index, { strict: options.strict }) const lintFindings = lintKnowledgeIndex(index) const readiness = readinessFor(options, index) @@ -223,7 +225,7 @@ export async function runKnowledgeResearchLoop( const maxIterations = Math.max(1, options.maxIterations ?? 3) await initKnowledgeBase(options.root) const steps: KnowledgeResearchLoopStep[] = [] - let index = await buildKnowledgeIndex(options.root) + let index = await buildKnowledgeIndex(options.root, options) let validation = validateKnowledgeIndex(index, { strict: options.strict }) let lintFindings = lintKnowledgeIndex(index) let readiness = readinessFor(options, index) @@ -245,7 +247,7 @@ export async function runKnowledgeResearchLoop( done = Boolean(decision.done) const step = await applyKnowledgeResearchDecision(options, decision, iteration) - index = await buildKnowledgeIndex(options.root) + index = await buildKnowledgeIndex(options.root, options) validation = step.validation lintFindings = step.lintFindings readiness = step.readiness @@ -283,10 +285,12 @@ async function applyKnowledgeResearchDecision( } const applied = decision.proposalText - ? await applyKnowledgeWriteBlocks(options.root, decision.proposalText) + ? await applyKnowledgeWriteBlocks(options.root, decision.proposalText, { + pagesDirectory: options.pagesDirectory, + }) : undefined - const index = await buildKnowledgeIndex(options.root) + const index = await buildKnowledgeIndex(options.root, options) const validation = validateKnowledgeIndex(index, { strict: options.strict }) const lintFindings = lintKnowledgeIndex(index) const readiness = readinessFor(options, index) diff --git a/tests/kb-improvement/optimization.test.ts b/tests/kb-improvement/optimization.test.ts index 352767b..f41a275 100644 --- a/tests/kb-improvement/optimization.test.ts +++ b/tests/kb-improvement/optimization.test.ts @@ -42,306 +42,316 @@ function immutableRef(value: string): string { } describe('optimizeKnowledgeBasePolicy', () => { - it('runs full RAG evaluation against the isolated candidate KB', async () => { - await withKb(async (root) => { - const method: OptimizationMethod = { - name: 'fixture-candidate-rag-method', - async optimize(input) { - expect('testScenarios' in input).toBe(false) - return { - winnerSurface: '{"mode":"grounded"}', - cost: { - totalCostUsd: 0, - costProvenance: { kind: 'observed', usd: 0 }, - accountingComplete: true, - incompleteReasons: [], + it.skipIf(process.platform !== 'linux')( + 'runs full RAG evaluation against the isolated candidate KB', + async () => { + await withKb(async (root) => { + const method: OptimizationMethod = { + name: 'fixture-candidate-rag-method', + async optimize(input) { + expect('testScenarios' in input).toBe(false) + return { + winnerSurface: '{"mode":"grounded"}', + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, + } + }, + } + const scenario = (id: string): RagAnswerEvalScenario => ({ + id, + kind: 'rag-answer-eval', + query: `${id} candidate policy`, + }) + const seenCandidateRoots = new Set() + + const result = await improveKnowledgeBase({ + root, + goal: 'Evaluate RAG against candidate knowledge', + implementationRef: immutableRef('candidate-rag-improvement'), + runId: 'candidate-rag-optimization', + async updateKnowledge({ candidateRoot }) { + const path = join(candidateRoot, 'knowledge', 'candidate-policy.md') + await mkdir(dirname(path), { recursive: true }) + await writeFile( + path, + [ + '---', + 'id: candidate-policy', + 'title: Candidate Policy', + '---', + '# Candidate Policy', + 'Candidate-only evidence.', + ].join('\n'), + ) + return { applied: true, summary: 'wrote candidate knowledge' } + }, + ragOptimization: { + executionRef: immutableRef('candidate-rag-execution'), + baseline: { mode: 'unsupported' }, + method, + trainScenarios: [scenario('candidate-rag-train')], + selectionScenarios: [scenario('candidate-rag-selection')], + finalScenarios: [scenario('candidate-rag-final-a'), scenario('candidate-rag-final-b')], + async run({ + config, + scenario: item, + baseHash, + baselineRoot, + candidateRoot, + candidateIndex, + }) { + seenCandidateRoots.add(candidateRoot) + expect(candidateRoot).not.toBe(root) + expect(baselineRoot).not.toBe(root) + expect(await hashKnowledgeBase(baselineRoot)).toBe(baseHash) + expect(candidateIndex.pages.map((page) => page.id)).toContain('candidate-policy') + const score = config.mode === 'grounded' ? 1 : 0 + return { + query: item.query, + answer: score ? 'Candidate-only evidence.' : 'Unsupported answer.', + contexts: [], + metadata: { score }, + } }, - } - }, - } - const scenario = (id: string): RagAnswerEvalScenario => ({ - id, - kind: 'rag-answer-eval', - query: `${id} candidate policy`, + judges: [ + { + name: 'candidate-rag-quality', + dimensions: [{ key: 'quality', description: 'candidate RAG quality' }], + score: ({ artifact }) => { + const score = Number(artifact.metadata?.score ?? 0) + return { composite: score, dimensions: { quality: score } } + }, + }, + ], + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + }, + requiredPhases: ['rag-optimization'], + evaluate: passingMetric, + }) + + expect(seenCandidateRoots.size).toBe(1) + expect(result.lifecycle?.optimization?.winner.value).toEqual({ mode: 'grounded' }) + expect(result.lifecycle?.optimization?.comparison.testScenarioIds).toEqual([ + 'candidate-rag-final-a', + 'candidate-rag-final-b', + ]) }) - const seenCandidateRoots = new Set() + }, + ) - const result = await improveKnowledgeBase({ - root, - goal: 'Evaluate RAG against candidate knowledge', - implementationRef: immutableRef('candidate-rag-improvement'), - runId: 'candidate-rag-optimization', - async updateKnowledge({ candidateRoot }) { - const path = join(candidateRoot, 'knowledge', 'candidate-policy.md') - await mkdir(dirname(path), { recursive: true }) - await writeFile( - path, - [ - '---', - 'id: candidate-policy', - 'title: Candidate Policy', - '---', - '# Candidate Policy', - 'Candidate-only evidence.', - ].join('\n'), - ) - return { applied: true, summary: 'wrote candidate knowledge' } - }, - ragOptimization: { - executionRef: immutableRef('candidate-rag-execution'), - baseline: { mode: 'unsupported' }, - method, - trainScenarios: [scenario('candidate-rag-train')], - selectionScenarios: [scenario('candidate-rag-selection')], - finalScenarios: [scenario('candidate-rag-final-a'), scenario('candidate-rag-final-b')], - async run({ - config, - scenario: item, - baseHash, - baselineRoot, - candidateRoot, - candidateIndex, - }) { - seenCandidateRoots.add(candidateRoot) - expect(candidateRoot).not.toBe(root) - expect(baselineRoot).not.toBe(root) - expect(await hashKnowledgeBase(baselineRoot)).toBe(baseHash) - expect(candidateIndex.pages.map((page) => page.id)).toContain('candidate-policy') - const score = config.mode === 'grounded' ? 1 : 0 + it.skipIf(process.platform !== 'linux')( + 'uses development checks for retries and runs final evaluation once', + async () => { + await withKb(async (root) => { + let methodCalls = 0 + let promotionCalls = 0 + let developmentEvaluatorCalls = 0 + let finalEvaluatorCalls = 0 + const updatedIterations: number[] = [] + const finalDispatches: string[] = [] + const scenario = (id: string): RagAnswerEvalScenario => ({ + id, + kind: 'rag-answer-eval', + query: id, + }) + const method: OptimizationMethod = { + name: 'single-final-method', + async optimize() { + methodCalls += 1 return { - query: item.query, - answer: score ? 'Candidate-only evidence.' : 'Unsupported answer.', - contexts: [], - metadata: { score }, + winnerSurface: '{"mode":"candidate"}', + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, - judges: [ - { - name: 'candidate-rag-quality', - dimensions: [{ key: 'quality', description: 'candidate RAG quality' }], - score: ({ artifact }) => { - const score = Number(artifact.metadata?.score ?? 0) - return { composite: score, dimensions: { quality: score } } - }, - }, - ], - storage: inMemoryCampaignStorage(), - expectUsage: 'off', - resamples: 200, - }, - requiredPhases: ['rag-optimization'], - evaluate: passingMetric, - }) + } - expect(seenCandidateRoots.size).toBe(1) - expect(result.lifecycle?.optimization?.winner.value).toEqual({ mode: 'grounded' }) - expect(result.lifecycle?.optimization?.comparison.testScenarioIds).toEqual([ - 'candidate-rag-final-a', - 'candidate-rag-final-b', - ]) - }) - }) + const result = await improveKnowledgeBase({ + root, + goal: 'Retry development candidates without reusing final cases', + implementationRef: immutableRef('single-final-improvement'), + runId: 'single-final-improvement', + maxCandidates: 3, + async updateKnowledge({ candidateRoot, iteration }) { + updatedIterations.push(iteration) + if (iteration === 1) + return { applied: false, summary: 'left required knowledge absent' } + const source = refundSource() + const added = await addSourceText(candidateRoot, source) + await applyKnowledgeWriteBlocks(candidateRoot, refundProposal(added.id)) + return { applied: true, summary: `updated candidate ${iteration}` } + }, + readinessSpecs: [refundSpec], + strict: true, + ragOptimization: { + executionRef: immutableRef('single-final-rag'), + baseline: { mode: 'baseline' }, + method, + trainScenarios: [scenario('single-final-train')], + selectionScenarios: [scenario('single-final-selection')], + finalScenarios: [scenario('single-final-a'), scenario('single-final-b')], + async run({ scenario: item }) { + if (item.id.startsWith('single-final-') && !item.id.endsWith('train')) { + if (item.id === 'single-final-a' || item.id === 'single-final-b') { + finalDispatches.push(item.id) + } + } + return { query: item.query, answer: 'answer', contexts: [] } + }, + judges: [ + { + name: 'single-final-quality', + dimensions: [{ key: 'quality', description: 'answer quality' }], + score: () => ({ composite: 1, dimensions: { quality: 1 } }), + }, + ], + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + }, + requiredPhases: ['rag-optimization', 'promotion'], + evaluateDevelopment({ iteration }) { + developmentEvaluatorCalls += 1 + return { + score: iteration >= 2 ? 1 : 0, + passed: iteration >= 2, + provenance: { + evaluator: 'single-final-development', + version: '1', + method: 'deterministic', + }, + } + }, + evaluate() { + finalEvaluatorCalls += 1 + return passingMetric() + }, + decidePromotion() { + promotionCalls += 1 + return { promoted: false, reason: 'adversarial final rejection' } + }, + }) - it('uses development checks for retries and runs final evaluation once', async () => { - await withKb(async (root) => { - let methodCalls = 0 - let promotionCalls = 0 - let developmentEvaluatorCalls = 0 - let finalEvaluatorCalls = 0 - const updatedIterations: number[] = [] - const finalDispatches: string[] = [] - const scenario = (id: string): RagAnswerEvalScenario => ({ - id, - kind: 'rag-answer-eval', - query: id, + expect(updatedIterations).toEqual([1, 2]) + expect(methodCalls).toBe(1) + expect(promotionCalls).toBe(1) + expect(developmentEvaluatorCalls).toBe(2) + expect(finalEvaluatorCalls).toBe(1) + expect(new Set(finalDispatches)).toEqual(new Set(['single-final-a', 'single-final-b'])) + expect(result.state.status).toBe('rejected') + expect(result.state.candidates).toHaveLength(2) }) - const method: OptimizationMethod = { - name: 'single-final-method', - async optimize() { - methodCalls += 1 - return { - winnerSurface: '{"mode":"candidate"}', - cost: { - totalCostUsd: 0, - costProvenance: { kind: 'observed', usd: 0 }, - accountingComplete: true, - incompleteReasons: [], - }, - } - }, - } + }, + ) - const result = await improveKnowledgeBase({ - root, - goal: 'Retry development candidates without reusing final cases', - implementationRef: immutableRef('single-final-improvement'), - runId: 'single-final-improvement', - maxCandidates: 3, - async updateKnowledge({ candidateRoot, iteration }) { - updatedIterations.push(iteration) - if (iteration === 1) return { applied: false, summary: 'left required knowledge absent' } - const source = refundSource() - const added = await addSourceText(candidateRoot, source) - await applyKnowledgeWriteBlocks(candidateRoot, refundProposal(added.id)) - return { applied: true, summary: `updated candidate ${iteration}` } - }, - readinessSpecs: [refundSpec], - strict: true, - ragOptimization: { - executionRef: immutableRef('single-final-rag'), - baseline: { mode: 'baseline' }, - method, - trainScenarios: [scenario('single-final-train')], - selectionScenarios: [scenario('single-final-selection')], - finalScenarios: [scenario('single-final-a'), scenario('single-final-b')], - async run({ scenario: item }) { - if (item.id.startsWith('single-final-') && !item.id.endsWith('train')) { - if (item.id === 'single-final-a' || item.id === 'single-final-b') { - finalDispatches.push(item.id) - } + it.skipIf(process.platform !== 'linux')( + 'runs a complete method and applies only the exact winner to an isolated candidate', + async () => { + await withKb(async (root) => { + const methodInputs: string[][] = [] + const method: OptimizationMethod = { + name: 'fixture-kb-policy-method', + async optimize(input) { + methodInputs.push([ + ...input.trainScenarios.map((scenario) => scenario.id), + ...input.selectionScenarios.map((scenario) => scenario.id), + ]) + expect('testScenarios' in input).toBe(false) + return { + winnerSurface: '{"evidence":"required","maxSources":4}', + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } - return { query: item.query, answer: 'answer', contexts: [] } }, + } + const scenario = (id: string): PolicyScenario => ({ + id, + kind: 'kb-policy-eval', + prompt: `${id} source-backed update`, + }) + + const result = await optimizeKnowledgeBasePolicy({ + root, + goal: 'Select a source-backed KB maintenance policy', + baselinePolicy: { evidence: 'none', maxSources: 1 }, + method, + trainScenarios: [scenario('policy-train')], + selectionScenarios: [scenario('policy-selection')], + finalScenarios: [scenario('policy-final-a'), scenario('policy-final-b')], + policyApplicationRef: immutableRef('write-maintenance-policy'), + dispatchCandidate: async ({ candidate }) => ({ + score: candidate.evidence === 'required' && candidate.maxSources >= 2 ? 1 : 0, + }), judges: [ { - name: 'single-final-quality', - dimensions: [{ key: 'quality', description: 'answer quality' }], - score: () => ({ composite: 1, dimensions: { quality: 1 } }), + name: 'policy-quality', + dimensions: [{ key: 'quality', description: 'policy satisfies evidence rules' }], + score: ({ artifact }) => ({ + composite: artifact.score, + dimensions: { quality: artifact.score }, + }), }, ], + scenarioFingerprint: scenarioContentFingerprint, + runDir: 'memory://kb-policy-optimization-test', storage: inMemoryCampaignStorage(), expectUsage: 'off', resamples: 200, - }, - requiredPhases: ['rag-optimization', 'promotion'], - evaluateDevelopment({ iteration }) { - developmentEvaluatorCalls += 1 - return { - score: iteration >= 2 ? 1 : 0, - passed: iteration >= 2, - provenance: { - evaluator: 'single-final-development', - version: '1', - method: 'deterministic', - }, - } - }, - evaluate() { - finalEvaluatorCalls += 1 - return passingMetric() - }, - decidePromotion() { - promotionCalls += 1 - return { promoted: false, reason: 'adversarial final rejection' } - }, - }) - - expect(updatedIterations).toEqual([1, 2]) - expect(methodCalls).toBe(1) - expect(promotionCalls).toBe(1) - expect(developmentEvaluatorCalls).toBe(2) - expect(finalEvaluatorCalls).toBe(1) - expect(new Set(finalDispatches)).toEqual(new Set(['single-final-a', 'single-final-b'])) - expect(result.state.status).toBe('rejected') - expect(result.state.candidates).toHaveLength(2) - }) - }) - - it('runs a complete method and applies only the exact winner to an isolated candidate', async () => { - await withKb(async (root) => { - const methodInputs: string[][] = [] - const method: OptimizationMethod = { - name: 'fixture-kb-policy-method', - async optimize(input) { - methodInputs.push([ - ...input.trainScenarios.map((scenario) => scenario.id), - ...input.selectionScenarios.map((scenario) => scenario.id), - ]) - expect('testScenarios' in input).toBe(false) - return { - winnerSurface: '{"evidence":"required","maxSources":4}', - cost: { - totalCostUsd: 0, - costProvenance: { kind: 'observed', usd: 0 }, - accountingComplete: true, - incompleteReasons: [], - }, - } - }, - } - const scenario = (id: string): PolicyScenario => ({ - id, - kind: 'kb-policy-eval', - prompt: `${id} source-backed update`, - }) - - const result = await optimizeKnowledgeBasePolicy({ - root, - goal: 'Select a source-backed KB maintenance policy', - baselinePolicy: { evidence: 'none', maxSources: 1 }, - method, - trainScenarios: [scenario('policy-train')], - selectionScenarios: [scenario('policy-selection')], - finalScenarios: [scenario('policy-final-a'), scenario('policy-final-b')], - policyApplicationRef: immutableRef('write-maintenance-policy'), - dispatchCandidate: async ({ candidate }) => ({ - score: candidate.evidence === 'required' && candidate.maxSources >= 2 ? 1 : 0, - }), - judges: [ - { - name: 'policy-quality', - dimensions: [{ key: 'quality', description: 'policy satisfies evidence rules' }], - score: ({ artifact }) => ({ - composite: artifact.score, - dimensions: { quality: artifact.score }, - }), + candidate: { evaluate: passingMetric }, + async applyPolicy({ candidateRoot, policy, policySurfaceHash, optimizationMethod }) { + expect(policy).toEqual({ evidence: 'required', maxSources: 4 }) + expect(optimizationMethod).toBe('fixture-kb-policy-method') + const path = join(candidateRoot, 'knowledge', 'maintenance-policy.md') + await mkdir(dirname(path), { recursive: true }) + await writeFile( + path, + `# Maintenance Policy\n\n${policySurfaceHash}: require source evidence.\n`, + ) + return { applied: true, summary: 'wrote selected maintenance policy' } }, - ], - scenarioFingerprint: scenarioContentFingerprint, - runDir: 'memory://kb-policy-optimization-test', - storage: inMemoryCampaignStorage(), - expectUsage: 'off', - resamples: 200, - candidate: { evaluate: passingMetric }, - async applyPolicy({ candidateRoot, policy, policySurfaceHash, optimizationMethod }) { - expect(policy).toEqual({ evidence: 'required', maxSources: 4 }) - expect(optimizationMethod).toBe('fixture-kb-policy-method') - const path = join(candidateRoot, 'knowledge', 'maintenance-policy.md') - await mkdir(dirname(path), { recursive: true }) - await writeFile( - path, - `# Maintenance Policy\n\n${policySurfaceHash}: require source evidence.\n`, - ) - return { applied: true, summary: 'wrote selected maintenance policy' } - }, - }) + }) - expect(methodInputs).toEqual([['policy-train', 'policy-selection']]) - expect(result.optimization.winner.value).toEqual({ - evidence: 'required', - maxSources: 4, - }) - expect(result.optimization.comparison.testScenarioIds).toEqual([ - 'policy-final-a', - 'policy-final-b', - ]) - expect(result.improvement.state.status).toBe('candidate-ready') - expect(result.improvement.promoted).toBe(false) - expect(result.improvement.lifecycle?.knowledgeUpdate?.metadata?.optimization).toEqual({ - method: 'fixture-kb-policy-method', - policySurfaceHash: result.optimization.winner.surfaceHash, - policyApplicationRef: immutableRef('write-maintenance-policy'), - }) - await expect( - readFile(join(root, 'knowledge', 'maintenance-policy.md'), 'utf8'), - ).rejects.toMatchObject({ code: 'ENOENT' }) + expect(methodInputs).toEqual([['policy-train', 'policy-selection']]) + expect(result.optimization.winner.value).toEqual({ + evidence: 'required', + maxSources: 4, + }) + expect(result.optimization.comparison.testScenarioIds).toEqual([ + 'policy-final-a', + 'policy-final-b', + ]) + expect(result.improvement.state.status).toBe('candidate-ready') + expect(result.improvement.promoted).toBe(false) + expect(result.improvement.lifecycle?.knowledgeUpdate?.metadata?.optimization).toEqual({ + method: 'fixture-kb-policy-method', + policySurfaceHash: result.optimization.winner.surfaceHash, + policyApplicationRef: immutableRef('write-maintenance-policy'), + }) + await expect( + readFile(join(root, 'knowledge', 'maintenance-policy.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) - const candidateRoot = mutableCandidateRoot(root, result.improvement) - await expect( - readFile(join(candidateRoot, 'knowledge', 'maintenance-policy.md'), 'utf8'), - ).resolves.toContain(result.optimization.winner.surfaceHash) - }) - }) + const candidateRoot = mutableCandidateRoot(root, result.improvement) + await expect( + readFile(join(candidateRoot, 'knowledge', 'maintenance-policy.md'), 'utf8'), + ).resolves.toContain(result.optimization.winner.surfaceHash) + }) + }, + ) it('does not materialize a policy winner after the live knowledge base changes', async () => { await withKb(async (root) => { diff --git a/tests/kb-improvement/state-scope.test.ts b/tests/kb-improvement/state-scope.test.ts new file mode 100644 index 0000000..db18b94 --- /dev/null +++ b/tests/kb-improvement/state-scope.test.ts @@ -0,0 +1,219 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { assertKnowledgeMutationPath } from '../../src/file-transaction' +import { + createKnowledgeEvent, + FileSystemKbStore, + hashKnowledgeBase, + improveSelectedKnowledgeCandidate, + knowledgeImprovementCandidateRef, + promoteKnowledgeCandidate, + restoreKnowledgeCandidateBaseline, + withKnowledgeImprovementCandidate, + withKnowledgeImprovementComparison, +} from '../../src/index' +import { + improveTestKnowledgeBase, + passingMetric, + TEST_KNOWLEDGE_IMPLEMENTATION_REF, + withKb, +} from '../support/kb-improvement' + +const stateScope = { pagesDirectory: 'kb/pages', researchState: true } +const ledgerPath = '.agent-knowledge/claim-ledgers/episode.json' + +async function recordResearch(root: string, rounds: number) { + const store = new FileSystemKbStore({ root }) + await store.putClaimLedger({ + schemaVersion: 2, + id: 'episode', + goal: 'Learn retry policy', + updatedAt: '2026-09-07T00:00:00.000Z', + rounds, + claimEvidence: [], + registeredSources: [], + claims: [], + questions: [], + }) + await store.putEvent( + createKnowledgeEvent({ + type: 'research.iteration', + metadata: { rounds }, + now: () => new Date('2026-09-07T00:00:00.000Z'), + }), + ) +} + +async function prepare(root: string) { + await mkdir(join(root, 'kb/pages'), { recursive: true }) + await writeFile(join(root, 'kb/pages/retry.md'), '---\nid: retry\n---\nRetry three times.\n') + await recordResearch(root, 1) +} + +const readRounds = async (root: string) => + (await new FileSystemKbStore({ root }).getClaimLedger('episode'))?.rounds + +describe('declared candidate state', () => { + it.skipIf(process.platform !== 'linux')( + 'uses the declared pages directory in the maintained research writer and evaluator', + async () => { + await withKb(async (root) => { + const scope = { pagesDirectory: 'kb/pages.json' } + const result = await improveTestKnowledgeBase({ + root, + goal: 'Learn custom-layout retry policy', + stateScope: scope, + step: () => ({ + done: true, + proposalText: + '---FILE: kb/pages.json/retry.md---\n---\nid: retry\n---\nRetry three times.\n---END FILE---', + }), + evaluate: ({ candidateIndex }) => { + expect(candidateIndex.pages.map((page) => page.id)).toEqual(['retry']) + return passingMetric() + }, + }) + await promoteKnowledgeCandidate({ + root, + candidate: knowledgeImprovementCandidateRef(result), + }) + expect(await readFile(join(root, 'kb/pages.json/retry.md'), 'utf8')).toContain('three') + }) + }, + ) + + it('binds declared state and scope while excluding caches and unrelated files from default identity', async () => { + await withKb(async (root) => { + await prepare(root) + const baseline = await hashKnowledgeBase(root) + const declared = await hashKnowledgeBase(root, stateScope) + await recordResearch(root, 2) + expect(await hashKnowledgeBase(root)).toBe(baseline) + expect(await hashKnowledgeBase(root, stateScope)).not.toBe(declared) + const changed = await hashKnowledgeBase(root, stateScope) + await writeFile(join(root, '.agent-knowledge/index.json'), '{}') + await writeFile(join(root, '.env'), 'TOKEN=not-a-real-secret') + expect(await hashKnowledgeBase(root, stateScope)).toBe(changed) + expect(await hashKnowledgeBase(root, { pagesDirectory: 'kb/pages' })).not.toBe(changed) + await writeFile(join(root, 'kb/pages/retry.md'), 'Retry five times.') + expect(await hashKnowledgeBase(root, stateScope)).not.toBe(changed) + }) + }) + + it.skipIf(process.platform !== 'linux')( + 'isolates siblings, resumes exact state, and promotes and restores declared research and custom pages', + async () => { + await withKb(async (root) => { + await prepare(root) + const baseline = await hashKnowledgeBase(root, stateScope) + let updates = 0 + const options = { + root, + goal: 'Learn retry policy', + stateScope, + async updateKnowledge({ candidateRoot }: { candidateRoot: string }) { + updates++ + expect(await readRounds(candidateRoot)).toBe(1) + await recordResearch(candidateRoot, 2) + await writeFile( + join(candidateRoot, 'kb/pages/retry.md'), + '---\nid: retry\n---\nRetry five times.\n', + ) + return { applied: true, summary: 'Updated retry evidence' } + }, + evaluate: passingMetric, + } + const first = await improveTestKnowledgeBase({ ...options, runId: 'first' }) + const sibling = await improveTestKnowledgeBase({ ...options, runId: 'sibling' }) + expect(await hashKnowledgeBase(root, stateScope)).toBe(baseline) + expect(await readRounds(root)).toBe(1) + const resumed = await improveTestKnowledgeBase({ ...options, runId: 'first' }) + expect(updates).toBe(2) + expect(knowledgeImprovementCandidateRef(resumed)).toEqual( + knowledgeImprovementCandidateRef(first), + ) + await expect( + improveTestKnowledgeBase({ ...options, runId: 'first', stateScope: {} }), + ).rejects.toThrow('stateScope') + const reference = knowledgeImprovementCandidateRef(first) + await expect( + withKnowledgeImprovementCandidate( + { root, candidate: reference }, + async ({ root: isolated }) => { + await recordResearch(isolated, 99) + }, + ), + ).rejects.toThrow('snapshot changed during use') + await withKnowledgeImprovementComparison( + { root, candidate: knowledgeImprovementCandidateRef(sibling) }, + async (comparison) => { + expect(comparison.stateScope).toEqual(stateScope) + expect(await readRounds(comparison.baseline.root)).toBe(1) + expect(await readRounds(comparison.candidate.root)).toBe(2) + }, + ) + await expect( + promoteKnowledgeCandidate({ + root, + candidate: reference, + onState() { + throw new Error('interrupted after durable promotion state') + }, + }), + ).rejects.toThrow('interrupted after durable promotion state') + await promoteKnowledgeCandidate({ root, candidate: reference }) + expect(await hashKnowledgeBase(root, stateScope)).toBe(reference.candidateHash) + expect(await readRounds(root)).toBe(2) + expect(await readFile(join(root, 'kb/pages/retry.md'), 'utf8')).toContain('five') + await restoreKnowledgeCandidateBaseline({ root, candidate: reference }) + expect(await hashKnowledgeBase(root, stateScope)).toBe(baseline) + expect(await readRounds(root)).toBe(1) + }) + }, + ) + + it.skipIf(process.platform !== 'linux')( + 'remeasures selected custom pages and research records using the source scope', + async () => { + await withKb(async (root) => { + await prepare(root) + const result = await improveTestKnowledgeBase({ + root, + goal: 'Learn retry policy', + stateScope, + async updateKnowledge({ candidateRoot }) { + await recordResearch(candidateRoot, 2) + return { applied: true, summary: 'Second research round' } + }, + evaluate: passingMetric, + }) + const selected = await improveSelectedKnowledgeCandidate({ + root, + goal: 'Select the claim state', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate: knowledgeImprovementCandidateRef(result), + selectedPaths: [ledgerPath], + evaluate: passingMetric, + }) + const reference = knowledgeImprovementCandidateRef(selected) + await promoteKnowledgeCandidate({ root, candidate: reference }) + expect(await readRounds(root)).toBe(2) + expect(await new FileSystemKbStore({ root }).listEvents()).toHaveLength(1) + }) + }, + ) + + it('requires explicit research permission and rejects credentials and derived state in transaction paths', () => { + expect(() => assertKnowledgeMutationPath(ledgerPath, 'kb/pages')).toThrow('unsupported') + expect(assertKnowledgeMutationPath(ledgerPath, 'kb/pages', true)).toBe(ledgerPath) + for (const path of [ + '.env', + '.agent-knowledge/index.json', + '.agent-knowledge/mutation-epoch.json', + '.agent-knowledge/claim-ledgers/../token.json', + ]) { + expect(() => assertKnowledgeMutationPath(path, 'kb/pages', true)).toThrow() + } + }) +})