diff --git a/.dockerignore b/.dockerignore index e3ae0d7..0cd2a2f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,7 @@ ** !CodeMesh.sln +!Directory.Build.props +!VERSION !NuGet.config !src/ !src/CodeMesh.Domain/ diff --git a/.github/workflows/baseline-standards.yml b/.github/workflows/baseline-standards.yml index 4446e82..ad5cf7e 100644 --- a/.github/workflows/baseline-standards.yml +++ b/.github/workflows/baseline-standards.yml @@ -93,8 +93,10 @@ jobs: run: | git diff --check "$CHECK_RANGE" npx --yes markdownlint-cli2@0.23.1 "**/*.md" + python tools/check_version.py python tools/check_markdown_links.py python tools/check_publication_safety.py + python tools/check_github_workflows.py .github/workflows/baseline-standards.yml if [[ "$EVENT_NAME" == "pull_request" ]]; then python tools/check_conventional_commit.py --message "$PULL_REQUEST_TITLE" else diff --git a/AGENTS.md b/AGENTS.md index 6f12432..08d96dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,8 @@ Before planning larger CodeMesh work, review these repository notes: - `docs/README.md` is the documentation hub and authority map. - `docs/guides/agent-quickstart.md` routes AI agents through the repository. +- `COMMANDS.md` is the canonical repository command index, while + `docs/evaluation/testing.md` defines risk-based check selection. - `docs/current/project-status.md` records current implementation and reviewed evidence. - `docs/planning/next-steps.md` is the execution-order authority for future product work. - `docs/current/architecture.md`, `docs/current/agent-access-contracts.md`, and source/tests @@ -21,6 +23,8 @@ Before planning larger CodeMesh work, review these repository notes: - `docs/evaluation/testing.md`, `docs/current/security-and-redaction.md`, and `docs/engineering/standards.md` define verification, safety, and completion discipline. +- `docs/engineering/methodology.md` defines evidence, authority, and + reusable-control admission boundaries. Treat `docs/planning/next-steps.md`, `docs/planning/agent-integration-contract.md`, @@ -33,22 +37,23 @@ Treat `docs/planning/next-steps.md`, ## Agent Defaults - Inspect `git status --short --branch` before editing and preserve unrelated - work. Stage only task-related files or hunks when a commit is requested. + work. Stage only task-related files or hunks for each commit. - Prefer the smallest coherent change and existing repository patterns. Keep implemented behavior, passing tests, reviewed evidence, release, deployment, and production authority distinct. - Do not print `.env` contents, credentials, tokens, private keys, certificates, or local secret values. -- For implementation changes, run `dotnet run --project tests/CodeMesh.Tests` from the repo root unless clearly irrelevant. -- For Python Agent Access changes, also run `uv run pytest` from `agent-access`. +- Use the supported invocations in `COMMANDS.md` and the risk map in + `docs/evaluation/testing.md`; do not improvise weaker substitutes for an + unavailable required check. - For changes crossing .NET/Python contracts, serialization, lifecycle, or integration boundaries, run both suites. -- Use `.\scripts\e2e-smoke.ps1` when changes affect live store wiring, Docker services, repository lifecycle, cleanup, or Agent Access integration. -- Documentation-only changes usually do not require application tests. Run - `npx --yes markdownlint-cli2@0.23.1 "**/*.md"`, - `python tools/check_markdown_links.py`, and `git diff --check`. -- Use Conventional Commits for requested commits. Validate a proposed range - with `python tools/check_conventional_commit.py --range BASE..HEAD` when - relevant; do not install or alter Git hooks implicitly. +- Use the end-to-end smoke path when changes affect live store wiring, Docker + services, repository lifecycle, cleanup, or Agent Access integration. +- Documentation-only changes usually do not require application tests; run the + repository standards checks listed in `COMMANDS.md`. +- Use Conventional Commits. Validate a proposed range + with the checker listed in `COMMANDS.md` when relevant; do not install or + alter Git hooks implicitly. - Record consequential implemented architecture decisions under `docs/decisions/`; do not use decision records to present planning as current behavior. @@ -56,14 +61,37 @@ Treat `docs/planning/next-steps.md`, `docs/planning/next-steps.md`, and detailed proposals in their dedicated planning documents. +## Development Feedback Sessions + +- Development-feedback MCP profiles are available only through the explicit, + reviewed session procedure in `docs/guides/mcp-setup.md`. Do not create or + activate a session unless the user asks for that operational action. +- When acting as the CodeMesh maintainer in an active session, start with + `codemesh_list_feedback`, retrieve the selected exact id with + `codemesh_get_feedback`, and reproduce the report read-only. +- Use `codemesh_prepare_feedback_resolution` to validate and hash the smallest + proposed change. Do not edit source until a human explicitly approves the + exact feedback ids and returned plan hash. An agent-supplied confirmation is + not human approval. +- After approval, preserve normal completion discipline. A passing client + recheck must append a new packet that names the original failure in + `supersedes_feedback_ids`; do not edit or delete either packet. + ## Completion And Authority +- Once a unit of work has been completed, commit the relevant files. Work is + complete when: + - Implementation is complete. + - Tests have been updated, and changes verified through testing. + - Documentation has been updated. - Update canonical documentation when behavior, commands, contracts, setup, architecture, or verified status changes. +- Use the narrowest truthful evidence state defined in + `docs/engineering/methodology.md`; one state never implies a later state. - Report passed, failed, skipped, and unavailable checks separately. - Review the final diff for accidental or unrelated changes and report the final worktree state. -- A completed implementation does not authorize a commit, tag, release, +- A completed unit of work and its commit do not authorize a tag, release, publication, deployment, production mutation, external spend, or other live side effect unless the user explicitly requests it and the repository's relevant gates are satisfied. diff --git a/COMMANDS.md b/COMMANDS.md new file mode 100644 index 0000000..0a0b9a0 --- /dev/null +++ b/COMMANDS.md @@ -0,0 +1,252 @@ +# CodeMesh repository commands + +This is the canonical index for supported contributor and coding-agent +commands. Use [Testing](docs/evaluation/testing.md) for risk-based check +selection, end-to-end variants, evaluation gates, and result interpretation. +Use the linked operating guides for ordered product or evaluation procedures. + +Run commands from the repository root unless a section says otherwise. Run +.NET commands serially because projects share intermediate output directories. + +## Deterministic verification + +### .NET + +```powershell +dotnet restore CodeMesh.sln --locked-mode +dotnet build CodeMesh.sln --no-restore +dotnet run --project tests/CodeMesh.Tests --no-restore +dotnet format CodeMesh.sln --verify-no-changes --no-restore +``` + +### Python Agent Access + +Run from `agent-access`: + +```powershell +uv sync --locked +uv run --no-sync pytest +uv run --no-sync ruff check . ../tools +uv run --no-sync ruff format --check . ../tools +uv run --no-sync python -m codemesh_agent_access eval +``` + +The last command is the deterministic in-process MCP fixture suite. It uses no +Docker service or model provider. Run both .NET and Python checks after changes +to shared contracts, serialization, lifecycle, integration, or end-to-end +behavior. A focused change may use only its affected surface when the handoff +states why the other suite was not required. + +For the development-feedback session boundary and its real stdio rehearsal, +run from `agent-access`: + +```powershell +uv run --no-sync pytest tests/test_feedback.py tests/test_feedback_session.py tests/test_development_feedback.py tests/test_development_feedback_integration.py tests/test_installer_probe.py tests/test_mcp_contract.py +``` + +The integration test uses disposable Git state, exact local MCP stdio launches, +and `CODEMESH_MODEL_PROVIDER=none`. It records only bounded fixture packets and +does not edit CodeMesh. Use [MCP Setup](docs/guides/mcp-setup.md#agent-feedback) +for the human-reviewed session, installation, approval, recheck, and revocation +procedure. + +### Repository standards + +```powershell +npx --yes markdownlint-cli2@0.23.1 "**/*.md" +python tools/check_version.py +python tools/check_markdown_links.py +python tools/check_publication_safety.py +python tools/check_github_workflows.py .github/workflows/baseline-standards.yml +git diff --check +``` + +Validate a proposed commit range when relevant: + +```powershell +python tools/check_conventional_commit.py --range BASE..HEAD +``` + +The documentation, publication-safety, and workflow-policy checks are local +and read-only. The GitHub workflow definition and a local workflow-policy pass +are not evidence that hosted GitHub Actions passed. + +## End-to-end verification + +Use the PowerShell smoke path when changes affect live store wiring, Docker +services, repository lifecycle, cleanup, REST endpoints, or Agent Access +integration: + +```powershell +.\scripts\e2e-smoke.ps1 +``` + +The script uses disposable fixture state, but it starts local containers and +services. See [Testing](docs/evaluation/testing.md#end-to-end-smoke) for its +selection rules and variants. + +For unpublished security corrections and candidate review, the +[local CodeQL procedure](docs/current/security-and-redaction.md#local-codeql-verification) +pins query packs, separates threat models, and forces evaluation after a model +change. Local scanning does not authorize SARIF upload or alert dismissal. + +## Local product health + +```powershell +dotnet run --project src/CodeMesh.Cli -- doctor +dotnet run --project src/CodeMesh.Cli -- status +docker compose ps +Invoke-RestMethod http://127.0.0.1:8088/health +docker compose logs --tail 100 agent-access csharp-parser +``` + +These commands inspect the configured local environment. They do not establish +release, deployment, external acceptance, or production fitness. + +## Operational and evaluation procedures + +Use the controlling guide instead of treating isolated commands as authority: + +- [MCP Setup](docs/guides/mcp-setup.md) for local services, ingestion, refresh, + health, and MCP configuration; +- [Self-Analysis](docs/guides/self-analysis.md) for indexing and querying this + repository; and +- [MCP Effectiveness Evaluation](docs/evaluation/mcp-effectiveness.md) for live, + agent, comparison, and model-backed evaluation. + +For provider-free context-package latency attribution, add +`--capture-context-package-timings` to `eval live`; the evaluation guide defines +the sanitized sideband and report contract. + +Live services, provider-backed runs, model spend, raw-trace retention, external +accounts, publication, release, deployment, and production mutation require +their own authority and evidence gates. The presence of a command in a guide +does not grant that authority. + +Agent campaigns additionally require `--max-reported-tokens` and a selected +runner that declares compatible hard-cap enforcement. The default +`--runner codex` reports usage only after completion and therefore fails +provider-free capability preflight before any model call. Select +`--runner capped-codex` explicitly for the loopback Responses proxy after the +evaluation guide's remaining identity, retrieval, safety, authority, and spend +gates pass; a repetition limit or post-run audit does not satisfy this gate. + +### Summary-model qualification + +The qualification runner uses the production summary prompt and parser. Start +from the checked-in synthetic suite and profile examples, then replace the +profile placeholders and use a separately reviewed, frozen corpus for real +qualification: + +```powershell +dotnet run --project src/CodeMesh.Cli -- summaries qualify run ` + --suite docs/evaluation/assets/summary-qualification-suite.example.json ` + --profile ` + --private-output ` + --review-output ` + --summary-provider ` + --summary-model + +dotnet run --project src/CodeMesh.Cli -- summaries qualify bind-retrieval ` + --archive ` + --baseline-live ` + --candidate-live ` + --assessment ` + --ranking-identity ` + --confirm-isolated-indexes ` + --output + +dotnet run --project src/CodeMesh.Cli -- summaries qualify compile ` + --suite ` + --archive ` + --review ` + --review ` + --retrieval ` + --deployment ` + --output + +dotnet run --project src/CodeMesh.Cli -- summaries qualify compare ` + --candidate ` + --reference +``` + +`run` refuses dirty CodeMesh checkouts and mismatched suite, profile, provider, +or model identities. Online runs additionally require +`--authorize-online-source`; that flag records the operator's explicit action +but does not replace source-governance, account, or spend approval. Output +paths use create-new semantics. Private archives and review packets contain +source or generated text and must remain in restricted evidence storage. +`compile` requires two complete blinded reviews bound to the same non-empty +disagreement-resolution record hash, a reviewed query-level retrieval +assessment bound to comparable live reports, and complete resource or online +cost evidence bound to the deployment profile. Missing or incompatible +evidence produces `invalid-run`, never a qualified result. The example suite is +a smoke fixture, not a sufficient qualification corpus. Example schemas for +review, retrieval-assessment, and deployment evidence are under +`docs/evaluation/assets/`. + +### Local backup and recovery + +Use [Backup and Recovery](docs/guides/backup-and-recovery.md) for the reviewed +sequence, ownership checks, stopped-set copy, new-volume restoration, image +identity verification, and complete data/retrieval comparison. The commands +apply only to the explicitly selected environment; the retained example uses +disposable stores and does not authorize changes to existing user data. + +## External advisory checks + +Before publication or dependency changes, and only when network access is +appropriate, run: + +```powershell +dotnet list CodeMesh.sln package --vulnerable --include-transitive +Set-Location agent-access +uv audit --locked +``` + +These results are time-dependent because they query external advisory data. + +## Application version + +```powershell +dotnet run --project src/CodeMesh.Cli -- --version +``` + +From `agent-access`: + +```powershell +uv run --no-sync python -m codemesh_agent_access --version +``` + +`VERSION` is the canonical source version. .NET builds consume it; Python +package metadata and exports are synchronized by `tools/check_version.py`. +The .NET informational version may include a build commit suffix. Version +output is available without opening stores or loading repository `.env` files. +It identifies software, not a published release or an accepted deployment. +See [Release Preparation](docs/engineering/release-preparation.md). + +### Local package verification + +Use new disposable output directories and record their exact paths. From the +repository root: + +```powershell +dotnet publish src/CodeMesh.Cli/CodeMesh.Cli.csproj --no-restore -c Release -p:PublishDir=/ +dotnet /CodeMesh.Cli.dll --version +``` + +From `agent-access`, build the Python artifacts and install the wheel into a +fresh environment with the locked runtime dependency set: + +```powershell +uv build --out-dir +uv export --locked --no-dev --no-emit-project --format requirements-txt --output-file +uv venv +uv pip sync --python --require-hashes +uv pip install --python --no-deps +``` + +Run the installed `codemesh-agent-access --version` outside the source checkout, +and inspect REST OpenAPI and the normal MCP manifest from that environment. +Record SHA-256 checksums for the artifacts and the source SHA. These local +commands do not publish packages or establish release acceptance. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6362432..ce1a5ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,17 +7,21 @@ the owner explicitly activates a deferred area. The [Documentation Hub](docs/README.md) identifies current references, operating guides, status, planning documents, and historical evidence. +Supported repository commands are indexed in [`COMMANDS.md`](COMMANDS.md). + +Start with the [Developer Handbook](docs/guides/developer-handbook.md) to learn +the architecture, domain vocabulary, and implementation flows. Use +[Development and Debugging](docs/guides/development-and-debugging.md) for local +diagnosis and [Making Changes](docs/guides/making-changes.md) to trace a change +across producers, consumers, contracts, and tests. ## Development setup Required tools are the .NET 10 SDK, Python 3.12 or newer, `uv`, and Git. Docker Compose is needed only for live-store and end-to-end work. -```powershell -dotnet restore CodeMesh.sln --locked-mode -Set-Location agent-access -uv sync --locked -``` +Use the locked .NET restore and Python Agent Access environment commands in +[`COMMANDS.md`](COMMANDS.md#deterministic-verification). Copy `.env.example` to `.env` only for local services, set a unique development password, and never commit the resulting file. @@ -45,12 +49,14 @@ password, and never commit the resulting file. ## Validation -Use the canonical risk-scaled commands and selection rules in -[Testing](docs/evaluation/testing.md). Run .NET commands serially because the solution -shares intermediate outputs. Use the end-to-end smoke path only for changes -that cross live stores, repository lifecycle, cleanup, Docker, or Agent Access -integration boundaries. Do not run provider-backed evaluation without explicit -authorization. +Use the supported invocations in [`COMMANDS.md`](COMMANDS.md) and the +risk-scaled selection rules in [Testing](docs/evaluation/testing.md). Run .NET +commands serially because the solution shares intermediate outputs. Use the +end-to-end smoke path only for changes that cross live stores, repository +lifecycle, cleanup, Docker, or Agent Access integration boundaries. Do not run +provider-backed evaluation without explicit authorization. The full engineering policy is in -[Engineering Standards](docs/engineering/standards.md). +[Engineering Standards](docs/engineering/standards.md), with evidence and +authority boundaries in the +[Software Engineering Methodology](docs/engineering/methodology.md). diff --git a/CodeMesh.sln b/CodeMesh.sln index 7bb3173..aa889b6 100644 --- a/CodeMesh.sln +++ b/CodeMesh.sln @@ -25,9 +25,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeMesh.Parser.Python", "s EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeMesh.Parser.Markdown", "src\CodeMesh.Parser.Markdown\CodeMesh.Parser.Markdown.csproj", "{2A78BC09-E879-45FC-BA76-F0A72ADC423F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeMesh.Parser.Deployment", "src\CodeMesh.Parser.Deployment\CodeMesh.Parser.Deployment.csproj", "{E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD}" -EndProject -Global +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeMesh.Parser.Deployment", "src\CodeMesh.Parser.Deployment\CodeMesh.Parser.Deployment.csproj", "{E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeMesh.Parser.Rust", "src\CodeMesh.Parser.Rust\CodeMesh.Parser.Rust.csproj", "{B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}" +EndProject +Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 @@ -156,8 +158,20 @@ Global {E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD}.Release|x64.ActiveCfg = Release|Any CPU {E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD}.Release|x64.Build.0 = Release|Any CPU {E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD}.Release|x86.ActiveCfg = Release|Any CPU - {E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection + {E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD}.Release|x86.Build.0 = Release|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Debug|x64.ActiveCfg = Debug|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Debug|x64.Build.0 = Debug|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Debug|x86.ActiveCfg = Debug|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Debug|x86.Build.0 = Debug|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Release|Any CPU.Build.0 = Release|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Release|x64.ActiveCfg = Release|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Release|x64.Build.0 = Release|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Release|x86.ActiveCfg = Release|Any CPU + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection @@ -171,6 +185,7 @@ Global {F6E289BC-AF8D-4067-945F-531EDFEA3641} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {0D23D644-B1F7-427B-A090-8954529A058F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {2A78BC09-E879-45FC-BA76-F0A72ADC423F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} - {E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E30BC1B7-F5D4-4C39-B3F1-1AC6544071FD} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {B8B8F5B8-9DAE-4F24-BB25-4DF030E55771} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..95e4446 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,5 @@ + + + $([System.IO.File]::ReadAllText('$(MSBuildThisFileDirectory)VERSION').Trim()) + + diff --git a/README.md b/README.md index 0fc81aa..ba260d8 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ is available under the [MIT License](LICENSE). | Status | Capability | | --- | --- | -| Implemented and locally verified | C# ingestion through Roslyn; graph/content persistence; stable checkout identity and bounded local snapshots; repository freshness; REST, MCP, CLI, and .NET Agent Access; default normal and diagnostic MCP profiles; canonical manual onboarding; source spans; relationship-aware context packages; deterministic fixture evaluation; security redaction tests. | -| Optional and environment-dependent | Neo4j, MongoDB, and Qdrant stores; Docker Compose; embeddings through a configured local provider; Python, Markdown, and deployment-artifact parsing with narrower relationships than C#. | -| Retained, bounded evidence | Summary-free live retrieval passed on two repositories; assisted campaigns replicated utility after explicit onboarding; spontaneous and prompt-parity campaigns recorded zero CodeMesh adoption. These results do not establish broad generalization. | +| Implemented and locally verified | C# ingestion through Roslyn; atomic multi-language ingestion; bounded Python and Rust parsing with PyO3 links; graph/content persistence; stable checkout identity and bounded local snapshots; exact checkout binding and fail-closed freshness; REST, MCP, CLI, and .NET Agent Access; normal and diagnostic MCP profiles; reviewable installation, configured-evaluation preflight, provider-free positive and rejection probes, and task-adaptive context-package guidance; agent feedback recording and summarization; source-backed context packages; deterministic fixture evaluation; pre-persistence redaction, default known-secret-file exclusion, repository-relative path filters, and the provider-neutral summary qualification runner/report compiler. | +| Optional and environment-dependent | Neo4j, MongoDB, and Qdrant stores; Docker Compose; embeddings through a configured local provider; Python, Rust, Markdown, and deployment-artifact parsing with narrower relationships than C#. | +| Retained, bounded evidence | Summary-free live retrieval passed for CodeMesh, YoutubeDownloader, and Config.Net, and the provider-free one\|nine Primary pilot passed its local retrieval, safety, and feedback gates. Configured campaigns include two insufficient status-first one\|nine results, a neutral one-step one\|nine result, regressed and improved Config.Net results, and two correctness-preserving but duration-regressed YoutubeDownloader results. Exact candidate `fbc1433` reduced treatment package attempts to one and reduced median tokens by 6.63%, but crossed the duration threshold by 0.0192 percentage points and exceeded its authorized aggregate reported-token ceiling. These results do not establish repeatable product benefit or broad generalization. | | Experimental or unqualified | Generated summaries, online model providers, model-selection benchmarks, and the read-only local web UI as a product surface. No summary model is CodeMesh-qualified. | -| Planned | Unambiguous checkout binding, fail-closed package freshness, a provider-free live runtime probe, reviewable installation, configured/onboarded evaluation, configurable ingestion allow/deny policy, shared-service identity controls, and broader parser and SDLC coverage. | +| Planned | Complete release prerequisites and obtain fresh exact-candidate evidence before any separately authorized product campaign. The explicit `capped-codex` runner is implemented; its first authorized campaign stopped for unavailable credits and is consumed. The default uncapped runner remains blocked before model calls. A measured repository-scoping correction improves provider-free search under controlled unrelated-node growth; it does not establish agent benefit. Keep expansion to additional one\|nine instances separately authorized. Shared-service identity controls and broader parser, UI, summary, memory, portability, and SDLC work remain deferred. | Passing tests establish the covered local behavior. They do not establish hosted CI success, generalized agent benefit, release readiness, deployment @@ -97,19 +97,27 @@ a live index. - Source-scoped MCP context packages with freshness and redaction diagnostics. - Deterministic fixture, live retrieval, and opt-in agent/model evaluation layers with explicit evidence boundaries. +- Provider-neutral summary qualification orchestration with clean-source, + corpus/profile, blinded-review, retrieval, and comparison gates; no model is + qualified by the runner's existence. - Local-first processing by default; generated summaries and external model providers are opt-in and unqualified. ## Verification -Use the risk-scaled commands in [Testing](docs/evaluation/testing.md). Current status and -retained product evidence are summarized in +Use the supported invocations in [`COMMANDS.md`](COMMANDS.md) and select them +with the risk-scaled rules in [Testing](docs/evaluation/testing.md). Current +status and retained product evidence are summarized in [Project Status](docs/current/project-status.md); dated results apply only to the commits they identify. ## Repository map - `src/CodeMesh.Parser.CSharp`: Roslyn parser worker. +- `src/CodeMesh.Parser.Python` and `src/CodeMesh.Parser.Rust`: bounded local + source-structural parsers used by the activated one|nine pilot. +- `src/CodeMesh.Parser.Markdown` and `src/CodeMesh.Parser.Deployment`: bounded + documentation and runtime-artifact parsers. - `src/CodeMesh.Ingestion`: ingestion, cleanup, redaction, embeddings, and optional summary generation. - `src/CodeMesh.Storage`: Neo4j, MongoDB, Qdrant, and in-memory adapters. @@ -117,6 +125,7 @@ commits they identify. operator commands. - `agent-access`: Python REST, MCP, CLI, ranking, and evaluation surfaces. - `tests/CodeMesh.Tests` and `agent-access/tests`: .NET and Python verification. +- `COMMANDS.md`: supported contributor and coding-agent command index. Start with [Agent Quickstart](docs/guides/agent-quickstart.md) when working in the repository. The [Documentation Hub](docs/README.md) organizes current diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/agent-access/codemesh_agent_access/binding.py b/agent-access/codemesh_agent_access/binding.py new file mode 100644 index 0000000..0212cc4 --- /dev/null +++ b/agent-access/codemesh_agent_access/binding.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +from typing import Any + + +class RepositoryBindingError(RuntimeError): + """Raised when a repository-scoped operation cannot honor its binding.""" + + +@dataclass(frozen=True) +class RepositoryBinding: + project_id: str + checkout_id: str + repository_root: str + source_view_hash: str | None = None + + def __post_init__(self) -> None: + project_id = self.project_id.strip() + checkout_id = self.checkout_id.strip() + repository_root = self.repository_root.strip() + source_view_hash = (self.source_view_hash or "").strip() or None + if not project_id: + raise ValueError("CodeMesh repository binding requires a project id.") + if not checkout_id: + raise ValueError("CodeMesh repository binding requires a checkout id.") + if not repository_root: + raise ValueError("CodeMesh repository binding requires a repository root.") + + root = Path(repository_root).expanduser() + if not root.is_absolute(): + raise ValueError( + "CodeMesh repository binding root must be an absolute path." + ) + + object.__setattr__(self, "project_id", project_id) + object.__setattr__(self, "checkout_id", checkout_id) + object.__setattr__(self, "repository_root", str(root.resolve(strict=False))) + object.__setattr__(self, "source_view_hash", source_view_hash) + + def accepts(self, repository_id: str | None) -> bool: + if repository_id is None or not repository_id.strip(): + return True + return repository_id.strip() in {self.project_id, self.checkout_id} + + def as_dict(self) -> dict[str, str | None]: + return { + "project_id": self.project_id, + "checkout_id": self.checkout_id, + "repository_root": self.repository_root, + "source_view_hash": self.source_view_hash, + } + + +def binding_from_values( + project_id: str | None = None, + checkout_id: str | None = None, + repository_root: str | None = None, + source_view_hash: str | None = None, +) -> RepositoryBinding | None: + values = { + "project_id": project_id or os.getenv("CODEMESH_PROJECT_ID"), + "checkout_id": checkout_id or os.getenv("CODEMESH_CHECKOUT_ID"), + "repository_root": repository_root or os.getenv("CODEMESH_REPOSITORY_ROOT"), + "source_view_hash": source_view_hash or os.getenv("CODEMESH_SOURCE_VIEW_HASH"), + } + required = ["project_id", "checkout_id", "repository_root"] + present = [name for name in required if (values[name] or "").strip()] + if not present: + if (values["source_view_hash"] or "").strip(): + raise ValueError( + "CODEMESH_SOURCE_VIEW_HASH cannot be used without a complete " + "project, checkout, and repository-root binding." + ) + return None + if len(present) != len(required): + missing = ", ".join( + name.replace("_", "-") for name in required if name not in present + ) + raise ValueError(f"Incomplete CodeMesh repository binding; missing: {missing}.") + + return RepositoryBinding( + project_id=str(values["project_id"]), + checkout_id=str(values["checkout_id"]), + repository_root=str(values["repository_root"]), + source_view_hash=( + str(values["source_view_hash"]) if values["source_view_hash"] else None + ), + ) + + +def bound_repository_reference( + binding: RepositoryBinding | None, + repository_id: str | None, + *, + required_when_unbound: bool, +) -> str | None: + if binding is not None: + if not binding.accepts(repository_id): + raise RepositoryBindingError( + f"Repository reference '{repository_id}' does not match the configured " + f"project '{binding.project_id}' and checkout '{binding.checkout_id}'." + ) + return binding.project_id + + if required_when_unbound and not (repository_id or "").strip(): + raise RepositoryBindingError( + "Repository selection is required because this CodeMesh server is unbound." + ) + return repository_id + + +def assess_binding( + binding: RepositoryBinding, + repository: Any | None, + freshness: dict[str, Any], +) -> dict[str, Any]: + issues: list[str] = [] + if repository is None: + issues.append("bound_repository_missing") + else: + if repository.project_id != binding.project_id: + issues.append("project_mismatch") + if repository.checkout_id != binding.checkout_id: + issues.append("checkout_mismatch") + if not _same_path(repository.root_path, binding.repository_root): + issues.append("repository_root_mismatch") + if ( + binding.source_view_hash + and repository.source_view_hash != binding.source_view_hash + ): + issues.append("source_view_hash_mismatch") + + if freshness.get("status") != "fresh" or freshness.get("is_stale") is not False: + issues.append("snapshot_not_fresh") + if freshness.get("provenance_status") != "known": + issues.append("snapshot_provenance_unknown") + + return { + "status": "accepted" if not issues else "rejected", + "issues": list(dict.fromkeys(issues)), + "project_id": binding.project_id, + "checkout_id": binding.checkout_id, + "repository_root": binding.repository_root, + "source_view_hash": binding.source_view_hash, + } + + +def require_accepted_binding(assessment: dict[str, Any]) -> None: + if assessment.get("status") == "accepted": + return + issues = ", ".join(str(value) for value in assessment.get("issues", [])) + raise RepositoryBindingError( + "CodeMesh refused repository context because the configured binding was " + f"not accepted: {issues or 'unknown_binding_failure'}." + ) + + +def _same_path(left: str, right: str) -> bool: + if not left or not right: + return False + return os.path.normcase(os.path.realpath(left)) == os.path.normcase( + os.path.realpath(right) + ) diff --git a/agent-access/codemesh_agent_access/cli.py b/agent-access/codemesh_agent_access/cli.py index 6033106..42bda02 100644 --- a/agent-access/codemesh_agent_access/cli.py +++ b/agent-access/codemesh_agent_access/cli.py @@ -2,19 +2,66 @@ import argparse import asyncio +from datetime import datetime import json import sys from pathlib import Path import uvicorn +from .binding import binding_from_values from .config import Settings -from . import evaluation, mcp, tools +from . import __version__, evaluation, feedback, feedback_session, mcp, tools from .formatting import format_context_package_for_agent +from .installer import ( + apply_installation_plan, + create_installation_plan, + load_installation_plan, + write_installation_plan, +) +from .probe import ( + RuntimeProbeError, + run_feedback_runtime_probe, + run_runtime_probe, + run_runtime_rejection_probe, +) + + +def _add_binding_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--project-id", + help="Stable CodeMesh project id (or CODEMESH_PROJECT_ID).", + ) + parser.add_argument( + "--checkout-id", + help="Stable CodeMesh checkout id (or CODEMESH_CHECKOUT_ID).", + ) + parser.add_argument( + "--repository-root", + help="Absolute bound checkout root (or CODEMESH_REPOSITORY_ROOT).", + ) + parser.add_argument( + "--source-view-hash", + help="Optional exact indexed source-view hash (or CODEMESH_SOURCE_VIEW_HASH).", + ) + + +def _add_feedback_session_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--feedback-session", + help="Absolute active development-feedback session manifest path.", + ) + parser.add_argument( + "--feedback-session-sha256", + help="Expected SHA-256 of the exact active session manifest file.", + ) def main() -> None: parser = argparse.ArgumentParser(prog="codemesh-agent-access") + parser.add_argument( + "--version", action="version", version=f"%(prog)s {__version__}" + ) subcommands = parser.add_subparsers(dest="command", required=True) subcommands.add_parser("rest", help="Run the REST API.") mcp_parser = subcommands.add_parser("mcp", help="Run the MCP server over stdio.") @@ -24,6 +71,8 @@ def main() -> None: default=mcp.DEFAULT_TOOL_PROFILE, help="Select the MCP tool surface (default: normal).", ) + _add_binding_arguments(mcp_parser) + _add_feedback_session_arguments(mcp_parser) manifest_parser = subcommands.add_parser( "mcp-manifest", help="Print the MCP tool contract manifest JSON." ) @@ -33,6 +82,196 @@ def main() -> None: default=mcp.DEFAULT_TOOL_PROFILE, help="Select the MCP tool surface (default: normal).", ) + _add_binding_arguments(manifest_parser) + _add_feedback_session_arguments(manifest_parser) + install_parser = subcommands.add_parser( + "install", + help="Plan or apply a reviewable checkout-bound MCP installation.", + ) + install_actions = install_parser.add_subparsers( + dest="install_action", + required=True, + ) + install_plan_parser = install_actions.add_parser( + "plan", + help="Print a reviewable installation plan without modifying the target.", + ) + install_plan_parser.add_argument("--target-root", required=True) + install_plan_parser.add_argument("--codemesh-root", required=True) + install_plan_parser.add_argument("--project-id", required=True) + install_plan_parser.add_argument("--checkout-id", required=True) + install_plan_parser.add_argument("--source-view-hash") + install_plan_parser.add_argument("--server-name", default="codemesh") + install_plan_parser.add_argument( + "--profile", + choices=mcp.tool_profile_names(), + default=mcp.DEFAULT_TOOL_PROFILE, + ) + _add_feedback_session_arguments(install_plan_parser) + install_plan_parser.add_argument( + "--configuration-only", + action="store_true", + help="Plan only MCP configuration when repository guidance is managed elsewhere.", + ) + install_plan_parser.add_argument("--output") + install_apply_parser = install_actions.add_parser( + "apply", + help="Apply an unchanged reviewed plan with its explicit hash.", + ) + install_apply_parser.add_argument("plan") + install_apply_parser.add_argument("--approve-plan-hash", required=True) + probe_parser = subcommands.add_parser( + "mcp-probe", + help="Launch and verify the exact MCP command in an applied installation plan.", + ) + probe_parser.add_argument("--plan", required=True) + probe_parser.add_argument("--timeout-seconds", type=int, default=30) + probe_parser.add_argument( + "--query", default="repository implementation entry points" + ) + probe_parser.add_argument("--expected-path", action="append", default=[]) + rejection_probe_parser = subcommands.add_parser( + "mcp-rejection-probe", + help="Prove that an unsafe normal-profile binding fails closed.", + ) + rejection_probe_parser.add_argument("--plan", required=True) + rejection_probe_parser.add_argument("--timeout-seconds", type=int, default=30) + rejection_probe_parser.add_argument("--project-id") + rejection_probe_parser.add_argument("--checkout-id") + rejection_probe_parser.add_argument("--repository-root") + rejection_probe_parser.add_argument("--source-view-hash") + rejection_probe_parser.add_argument( + "--expected-substring", action="append", default=[] + ) + feedback_parser = subcommands.add_parser( + "feedback", + help="Record, validate, or summarize sanitized CodeMesh feedback.", + ) + feedback_actions = feedback_parser.add_subparsers( + dest="feedback_action", + required=True, + ) + feedback_session_parser = feedback_actions.add_parser( + "session", + help="Plan, activate, inspect, or revoke a development feedback session.", + ) + feedback_session_actions = feedback_session_parser.add_subparsers( + dest="feedback_session_action", + required=True, + ) + feedback_session_plan = feedback_session_actions.add_parser( + "plan", + help="Create a reviewable immutable-session activation plan.", + ) + feedback_session_plan.add_argument("--manifest-path", required=True) + feedback_session_plan.add_argument("--codemesh-root", required=True) + feedback_session_plan.add_argument("--codemesh-project-id", required=True) + feedback_session_plan.add_argument("--codemesh-checkout-id", required=True) + feedback_session_plan.add_argument( + "--client-json", + action="append", + required=True, + help=( + "JSON object with project_id, checkout_id, repository_root, and " + "reporter_role; repeat for each client." + ), + ) + feedback_session_plan.add_argument("--expires-at", required=True) + feedback_session_plan.add_argument("--max-packets", type=int, default=100) + feedback_session_plan.add_argument( + "--max-requests-per-minute", type=int, default=20 + ) + feedback_session_plan.add_argument("--max-text-characters", type=int, default=500) + feedback_session_plan.add_argument("--max-paths", type=int, default=32) + feedback_session_plan.add_argument("--output") + feedback_session_activate = feedback_session_actions.add_parser( + "activate", + help="Create the exact reviewed session manifest exclusively.", + ) + feedback_session_activate.add_argument("plan") + feedback_session_activate.add_argument("--approve-plan-hash", required=True) + feedback_session_inspect = feedback_session_actions.add_parser( + "inspect", + help="Inspect a session without exposing client roots.", + ) + feedback_session_inspect.add_argument("--manifest", required=True) + feedback_session_inspect.add_argument("--manifest-sha256", required=True) + feedback_session_revoke = feedback_session_actions.add_parser( + "revoke", + help="Close a session by creating an immutable revocation record.", + ) + feedback_session_revoke.add_argument("--manifest", required=True) + feedback_session_revoke.add_argument("--manifest-sha256", required=True) + feedback_session_revoke.add_argument("--reason", required=True) + feedback_record_parser = feedback_actions.add_parser( + "record", + help="Write one versioned packet to an ignored checkout outbox.", + ) + feedback_record_parser.add_argument("--repository-root", required=True) + feedback_record_parser.add_argument("--codemesh-root") + feedback_record_parser.add_argument("--role", required=True) + feedback_record_parser.add_argument( + "--classification", + choices=["diagnostic", "controlled-evaluation"], + required=True, + ) + feedback_record_parser.add_argument( + "--confidence", choices=["low", "medium", "high"], required=True + ) + feedback_record_parser.add_argument("--project-id", required=True) + feedback_record_parser.add_argument("--checkout-id", required=True) + feedback_record_parser.add_argument("--snapshot-id", required=True) + feedback_record_parser.add_argument( + "--language", action="append", required=True, dest="languages" + ) + feedback_record_parser.add_argument("--parser-profile", required=True) + feedback_record_parser.add_argument("--task-family", required=True) + feedback_record_parser.add_argument( + "--issue-category", + choices=[ + "setup", + "binding", + "freshness", + "missing-language", + "missing-relationship", + "ranking", + "budget", + "validation", + "other", + ], + required=True, + ) + feedback_record_parser.add_argument( + "--validation-outcome", + choices=["passed", "failed", "not-run", "inconclusive"], + required=True, + ) + feedback_record_parser.add_argument("--tool-attempted", action="append", default=[]) + feedback_record_parser.add_argument("--tool-called", action="append", default=[]) + feedback_record_parser.add_argument("--helpful-path", action="append", default=[]) + feedback_record_parser.add_argument("--incorrect-path", action="append", default=[]) + feedback_record_parser.add_argument("--missed-path", action="append", default=[]) + feedback_record_parser.add_argument("--stale-path", action="append", default=[]) + feedback_record_parser.add_argument("--ambiguous-path", action="append", default=[]) + feedback_record_parser.add_argument("--fallback-reason") + feedback_record_parser.add_argument("--minimal-reproduction") + feedback_record_parser.add_argument( + "--expected-target", action="append", default=[] + ) + feedback_record_parser.add_argument("--proposed-correction") + feedback_record_parser.add_argument( + "--supersedes-feedback-id", action="append", default=[] + ) + feedback_validate_parser = feedback_actions.add_parser( + "validate", help="Validate packet schema, content id, and provenance shape." + ) + feedback_validate_parser.add_argument("packets", nargs="+") + feedback_summarize_parser = feedback_actions.add_parser( + "summarize", + help="Build a prioritized review packet from managed checkout outboxes.", + ) + feedback_summarize_parser.add_argument("roots", nargs="+") + feedback_summarize_parser.add_argument("--output") eval_parser = subcommands.add_parser( "eval", help="Run deterministic, live, or agent effectiveness evaluations." ) @@ -48,23 +287,72 @@ def main() -> None: live_eval_parser.add_argument("--warmup", type=int, default=1) live_eval_parser.add_argument("--allow-stale", action="store_true") live_eval_parser.add_argument("--keep-raw-traces", action="store_true") + live_eval_parser.add_argument( + "--capture-context-package-timings", + action="store_true", + help="Capture sanitized sideband context-package stage timings.", + ) live_eval_parser.add_argument("--output") agent_eval_parser = eval_modes.add_parser( "agent", help="Run paired agent tasks with and without CodeMesh." ) agent_eval_parser.add_argument("--suite", default="codemesh-agent") - agent_eval_parser.add_argument("--model", required=True) + agent_eval_parser.add_argument("--model") agent_eval_parser.add_argument("--local-provider", choices=["lmstudio", "ollama"]) agent_eval_parser.add_argument("--codex-executable", default="codex") + agent_eval_parser.add_argument( + "--runner", + choices=["codex", "capped-codex"], + default="codex", + help="Agent runner; capped-codex enforces complete-invocation accounting.", + ) agent_eval_parser.add_argument("--reasoning-effort", default="medium") agent_eval_parser.add_argument("--repetitions", type=int, default=3) + agent_eval_parser.add_argument( + "--max-reported-tokens", + type=int, + help=( + "Required hard campaign cap using input + output + reasoning tokens; " + "cached input is already included in input tokens." + ), + ) agent_eval_parser.add_argument("--seed", type=int, default=42) agent_eval_parser.add_argument("--repository-root") agent_eval_parser.add_argument("--repository-id") agent_eval_parser.add_argument("--output-dir") agent_eval_parser.add_argument("--keep-raw-traces", action="store_true") agent_eval_parser.add_argument("--keep-workspaces", action="store_true") + agent_eval_parser.add_argument( + "--configured-plan", + help="Reviewed and applied installation plan for a configured suite.", + ) + agent_eval_parser.add_argument( + "--configured-guidance-path", + action="append", + default=[], + help="Committed repository guidance file to identify (default: AGENTS.md).", + ) + agent_eval_parser.add_argument( + "--configured-probe-query", + default="repository implementation entry points", + ) + agent_eval_parser.add_argument( + "--configured-expected-path", + action="append", + default=[], + ) + agent_eval_parser.add_argument( + "--configured-probe-timeout-seconds", + type=int, + default=30, + ) + agent_eval_parser.add_argument( + "--preflight-only", + action="store_true", + help="Verify configured identity and positive/rejection probes without a model.", + ) + agent_eval_parser.add_argument("--preflight-output") model_eval_parser = eval_modes.add_parser( "model", help="Benchmark a model on CodeMesh query and change scenarios." @@ -228,11 +516,285 @@ def main() -> None: return if args.command == "mcp": - mcp.run(profile=args.profile) + try: + binding = binding_from_values( + args.project_id, + args.checkout_id, + args.repository_root, + args.source_view_hash, + ) + runtime = _feedback_runtime_from_args(args, binding) + if runtime is None: + mcp.run(profile=args.profile, binding=binding) + else: + mcp.run( + profile=args.profile, + binding=binding, + feedback_runtime=runtime, + ) + except ValueError as exc: + parser.error(str(exc)) return if args.command == "mcp-manifest": - print(json.dumps(mcp.tool_manifest(profile=args.profile), indent=2)) + try: + binding = binding_from_values( + args.project_id, + args.checkout_id, + args.repository_root, + args.source_view_hash, + ) + _feedback_runtime_from_args(args, binding) + except ValueError as exc: + parser.error(str(exc)) + print( + json.dumps( + mcp.tool_manifest(profile=args.profile, binding=binding), + indent=2, + ) + ) + return + + if args.command == "install": + try: + if args.install_action == "plan": + binding = binding_from_values( + args.project_id, + args.checkout_id, + args.target_root, + args.source_view_hash, + ) + assert binding is not None + plan = create_installation_plan( + args.target_root, + args.codemesh_root, + binding, + server_name=args.server_name, + profile=args.profile, + include_onboarding=not args.configuration_only, + feedback_session_path=args.feedback_session, + feedback_session_sha256=args.feedback_session_sha256, + ) + if args.output: + write_installation_plan(plan, args.output) + print(json.dumps(plan.to_dict(), indent=2, sort_keys=True)) + return + + plan = load_installation_plan(args.plan) + changed = apply_installation_plan(plan, args.approve_plan_hash) + print( + json.dumps( + { + "applied": True, + "plan_hash": plan.plan_hash, + "changed_files": changed, + }, + indent=2, + ) + ) + except (OSError, ValueError) as exc: + print(json.dumps({"applied": False, "error": str(exc)}), file=sys.stderr) + raise SystemExit(1) from exc + return + + if args.command == "mcp-probe": + try: + plan = load_installation_plan(args.plan) + if plan.profile in feedback_session.FEEDBACK_PROFILES: + report = asyncio.run( + run_feedback_runtime_probe( + plan, + timeout_seconds=args.timeout_seconds, + ) + ) + else: + report = asyncio.run( + run_runtime_probe( + plan, + timeout_seconds=args.timeout_seconds, + query=args.query, + expected_paths=args.expected_path, + ) + ) + print(json.dumps(report, indent=2, sort_keys=True)) + except (OSError, ValueError, RuntimeProbeError) as exc: + print(json.dumps({"passed": False, "error": str(exc)}), file=sys.stderr) + raise SystemExit(1) from exc + return + + if args.command == "mcp-rejection-probe": + try: + plan = load_installation_plan(args.plan) + report = asyncio.run( + run_runtime_rejection_probe( + plan, + timeout_seconds=args.timeout_seconds, + project_id=args.project_id, + checkout_id=args.checkout_id, + repository_root=args.repository_root, + source_view_hash=args.source_view_hash, + expected_substrings=args.expected_substring, + ) + ) + print(json.dumps(report, indent=2, sort_keys=True)) + except (OSError, ValueError, RuntimeProbeError) as exc: + print(json.dumps({"passed": False, "error": str(exc)}), file=sys.stderr) + raise SystemExit(1) from exc + return + + if args.command == "feedback": + try: + if args.feedback_action == "session": + if args.feedback_session_action == "plan": + clients = [ + feedback_session.SessionParticipant.model_validate_json(value) + for value in args.client_json + ] + plan = feedback_session.create_session_plan( + manifest_path=args.manifest_path, + codemesh=feedback_session.SessionParticipant( + project_id=args.codemesh_project_id, + checkout_id=args.codemesh_checkout_id, + repository_root=args.codemesh_root, + reporter_role="maintainer", + ), + clients=clients, + expires_at=_parse_datetime(args.expires_at), + bounds=feedback_session.SessionBounds( + max_packets=args.max_packets, + max_requests_per_minute=args.max_requests_per_minute, + max_text_characters=args.max_text_characters, + max_paths=args.max_paths, + ), + ) + if args.output: + feedback_session.write_session_plan(plan, args.output) + print( + json.dumps( + plan.model_dump(mode="json"), indent=2, sort_keys=True + ) + ) + return + + if args.feedback_session_action == "activate": + plan = feedback_session.load_session_plan(args.plan) + output = feedback_session.activate_session( + plan, + args.approve_plan_hash, + ) + print( + json.dumps( + { + "activated": True, + "session_id": plan.manifest.payload.session_id, + "manifest_path": str(output), + "manifest_file_sha256": plan.manifest_file_sha256, + }, + indent=2, + sort_keys=True, + ) + ) + return + + if args.feedback_session_action == "inspect": + print( + json.dumps( + feedback_session.inspect_session( + args.manifest, + args.manifest_sha256, + ), + indent=2, + sort_keys=True, + ) + ) + return + + output = feedback_session.revoke_session( + args.manifest, + args.manifest_sha256, + reason=args.reason, + ) + print( + json.dumps( + {"revoked": True, "revocation_path": str(output)}, + indent=2, + sort_keys=True, + ) + ) + return + + if args.feedback_action == "record": + packet, output = feedback.record_feedback( + repository_root=args.repository_root, + codemesh_root=args.codemesh_root, + role=args.role, + classification=args.classification, + confidence=args.confidence, + project_id=args.project_id, + checkout_id=args.checkout_id, + snapshot_id=args.snapshot_id, + languages=args.languages, + parser_profile=args.parser_profile, + task_family=args.task_family, + issue_category=args.issue_category, + validation_outcome=args.validation_outcome, + tools_attempted=args.tool_attempted, + tools_called=args.tool_called, + helpful_paths=args.helpful_path, + incorrect_paths=args.incorrect_path, + missed_paths=args.missed_path, + stale_paths=args.stale_path, + ambiguous_paths=args.ambiguous_path, + fallback_reason=args.fallback_reason, + minimal_reproduction=args.minimal_reproduction, + expected_targets=args.expected_target, + proposed_correction=args.proposed_correction, + supersedes_feedback_ids=args.supersedes_feedback_id, + ) + print( + json.dumps( + { + "recorded": True, + "feedback_id": packet.feedback_id, + "path": str(output), + }, + indent=2, + ) + ) + return + + if args.feedback_action == "validate": + packets = [ + feedback.validate_feedback(path).model_dump(mode="json") + for path in args.packets + ] + print( + json.dumps( + { + "valid": True, + "packet_count": len(packets), + "feedback_ids": [ + packet["feedback_id"] for packet in packets + ], + }, + indent=2, + ) + ) + return + + report = feedback.summarize_feedback(args.roots) + payload = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(payload, encoding="utf-8") + print(payload, end="") + except (OSError, ValueError) as exc: + print( + json.dumps({"valid": False, "error": str(exc)}, indent=2), + file=sys.stderr, + ) + raise SystemExit(1) from exc return if args.command == "eval": @@ -272,6 +834,9 @@ def main() -> None: warmup=args.warmup, allow_stale=args.allow_stale, keep_raw_traces=args.keep_raw_traces, + capture_context_package_timings=( + args.capture_context_package_timings + ), ) ) payload = json.dumps(report.model_dump(mode="json"), indent=2) @@ -284,6 +849,52 @@ def main() -> None: if args.eval_mode == "agent": suite, suite_path = evaluation.load_suite(args.suite, "agent") + if args.max_reported_tokens is None: + raise ValueError( + "eval agent requires --max-reported-tokens so campaign " + "budgeting can fail closed." + ) + configured_plan = ( + load_installation_plan(args.configured_plan) + if args.configured_plan + else None + ) + configured_options = { + "configured_plan": configured_plan, + "configured_guidance_paths": args.configured_guidance_path or None, + "configured_probe_query": args.configured_probe_query, + "configured_expected_paths": args.configured_expected_path, + "configured_probe_timeout_seconds": ( + args.configured_probe_timeout_seconds + ), + } + selected_runner = ( + evaluation.CappedCodexRunner(args.codex_executable) + if args.runner == "capped-codex" + else evaluation.CodexRunner(args.codex_executable) + ) + if args.preflight_only: + report = asyncio.run( + evaluation.run_agent_preflight( + suite, + max_reported_tokens=args.max_reported_tokens, + runner=selected_runner, + repository_root=args.repository_root, + repository_id=args.repository_id, + **configured_options, + ) + ) + payload = json.dumps(report, indent=2) + if args.preflight_output: + output_path = Path(args.preflight_output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(payload + "\n", encoding="utf-8") + print(payload) + return + if not args.model: + raise ValueError( + "eval agent requires --model unless --preflight-only is used." + ) report = asyncio.run( evaluation.run_agent_evaluation( suite, @@ -292,6 +903,7 @@ def main() -> None: local_provider=args.local_provider, codex_executable=args.codex_executable, reasoning_effort=args.reasoning_effort, + max_reported_tokens=args.max_reported_tokens, repetitions=args.repetitions, seed=args.seed, repository_root=args.repository_root, @@ -299,6 +911,8 @@ def main() -> None: output_dir=args.output_dir, keep_raw_traces=args.keep_raw_traces, keep_workspaces=args.keep_workspaces, + runner=selected_runner, + **configured_options, ) ) print(json.dumps(report.model_dump(mode="json"), indent=2)) @@ -517,3 +1131,37 @@ def _parse_filters(values: list[str]) -> dict[str, str]: filters[key] = item return filters + + +def _parse_datetime(value: str) -> datetime: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError(f"Invalid ISO-8601 timestamp: {value}") from exc + if parsed.tzinfo is None: + raise ValueError("Session timestamp must include a timezone.") + return parsed + + +def _feedback_runtime_from_args( + args: argparse.Namespace, + binding, +): + manifest = getattr(args, "feedback_session", None) + expected_hash = getattr(args, "feedback_session_sha256", None) + profile = args.profile + if profile in feedback_session.FEEDBACK_PROFILES: + if binding is None or not manifest or not expected_hash: + raise ValueError( + "Feedback MCP profiles require --feedback-session, " + "--feedback-session-sha256, and a complete repository binding." + ) + return feedback_session.create_session_runtime( + manifest, + expected_hash, + profile=profile, + binding=binding, + ) + if manifest or expected_hash: + raise ValueError("Feedback session options require a feedback MCP profile.") + return None diff --git a/agent-access/codemesh_agent_access/content_store.py b/agent-access/codemesh_agent_access/content_store.py index 2b6ae5d..03621f4 100644 --- a/agent-access/codemesh_agent_access/content_store.py +++ b/agent-access/codemesh_agent_access/content_store.py @@ -107,6 +107,81 @@ async def get_repository(self, repository_id: str) -> RepositorySummary | None: ) return normalize_repository(document) if document else None + async def get_bound_repository( + self, project_id: str, checkout_id: str + ) -> RepositorySummary | None: + """Resolve one checkout-current slot without using compatibility metadata.""" + if MongoClient is None: + return None + + def find_bound() -> dict[str, Any] | None: + database = self._mongo()[self._settings.mongo_database] + repositories = database[self._settings.mongo_repositories_collection] + base = find_repository_document(repositories, project_id) + resolved_project_id = str( + (base or {}).get("_id") + or (base or {}).get("repositoryId") + or project_id + ) + checkout = database["snapshot_checkouts"].find_one({"_id": checkout_id}) + if ( + checkout is None + or str(checkout.get("projectId") or "") != resolved_project_id + ): + return None + + project = database["snapshot_projects"].find_one( + {"_id": resolved_project_id} + ) + slots = [ + dict(slot) + for slot in (project or {}).get("slots", []) + if str(slot.get("kind") or "") == "CheckoutCurrent" + and str(slot.get("key") or "") == checkout_id + ] + if len(slots) > 1: + raise RuntimeError( + f"Checkout '{checkout_id}' has multiple checkout-current slots." + ) + if not slots: + return None + + slot = slots[0] + snapshot_id = str(slot.get("snapshotId") or "") + snapshot = database["snapshots"].find_one({"_id": snapshot_id}) + if ( + snapshot is None + or str(snapshot.get("projectId") or "") != resolved_project_id + ): + return None + + observations = list( + database["snapshot_observations"] + .find( + { + "projectId": resolved_project_id, + "checkoutId": checkout_id, + "snapshotId": snapshot_id, + } + ) + .sort("observedAtUtc", -1) + .limit(1) + ) + observation = observations[0] if observations else None + return bound_repository_document( + base, + dict(checkout), + slot, + dict(snapshot), + dict(observation) if observation else None, + ) + + try: + document = await asyncio.to_thread(find_bound) + except PyMongoError: + return None + return normalize_repository(document) if document else None + async def delete_repository(self, repository_id: str) -> None: if not repository_id or MongoClient is None: return @@ -327,6 +402,51 @@ def normalize_repository(document: dict[str, Any]) -> RepositorySummary: ) +def bound_repository_document( + base: dict[str, Any] | None, + checkout: dict[str, Any], + slot: dict[str, Any], + snapshot: dict[str, Any], + observation: dict[str, Any] | None, +) -> dict[str, Any]: + project_id = str(snapshot.get("projectId") or checkout.get("projectId") or "") + checkout_id = str(checkout.get("_id") or "") + snapshot_id = str(snapshot.get("_id") or slot.get("snapshotId") or "") + metadata = string_dict((base or {}).get("metadata")) + metadata.update( + { + "projectId": project_id, + "checkoutId": checkout_id, + "snapshotId": snapshot_id, + "observedSnapshotId": snapshot_id, + "slotKind": str(slot.get("kind") or "CheckoutCurrent"), + "sourceViewHash": str(snapshot.get("sourceViewHash") or ""), + } + ) + if observation is not None: + metadata["workingTreeDirty"] = str( + bool(observation.get("workingTreeDirty")) + ).lower() + + document = dict(base or {}) + document.update( + { + "_id": project_id, + "rootPath": str(checkout.get("rootPath") or ""), + "branch": (observation or {}).get("branch"), + "commit": (observation or {}).get("commit"), + "lastSeenAtUtc": checkout.get("lastSeenAtUtc") + or (base or {}).get("lastSeenAtUtc"), + "nodeCount": snapshot.get("nodeCount", 0), + "relationshipCount": snapshot.get("relationshipCount", 0), + "contentCount": snapshot.get("contentCount", 0), + "embeddingCount": snapshot.get("embeddingCount"), + "metadata": metadata, + } + ) + return document + + def normalize_ingestion_run(document: dict[str, Any]) -> IngestionRunSummary: return IngestionRunSummary( run_id=str(document.get("_id") or document.get("runId") or ""), diff --git a/agent-access/codemesh_agent_access/evaluation/__init__.py b/agent-access/codemesh_agent_access/evaluation/__init__.py index 81d247e..d8b4f8b 100644 --- a/agent-access/codemesh_agent_access/evaluation/__init__.py +++ b/agent-access/codemesh_agent_access/evaluation/__init__.py @@ -1,10 +1,20 @@ -from .agent import run_agent_evaluation +from .agent import run_agent_evaluation, run_agent_preflight from .benchmark import ( compare_model_reports, finalize_model_report, run_model_benchmark, ) -from .codex import AgentRunner, AgentRunRequest, AgentRunResult, CodexRunner +from .capped_codex import CappedCodexRunner +from .codex import ( + REPORTED_TOKEN_ACCOUNTING_DESCRIPTION, + REPORTED_TOKEN_ACCOUNTING_ID, + AgentRunner, + AgentRunnerCapabilities, + AgentRunRequest, + AgentRunResult, + CodexRunner, + count_reported_tokens, +) from .fixture import EvaluationCaseResult, EvaluationReport, run_mcp_evaluations from .live import EvaluationInfrastructureError, run_live_evaluation from .models import ( @@ -18,18 +28,24 @@ __all__ = [ "AgentSuite", "AgentRunner", + "AgentRunnerCapabilities", "AgentRunRequest", "AgentRunResult", "CodexRunner", + "CappedCodexRunner", "EvaluationCaseResult", "EvaluationReport", "EvaluationInfrastructureError", "LiveSuite", "ModelBenchmarkSuite", + "REPORTED_TOKEN_ACCOUNTING_DESCRIPTION", + "REPORTED_TOKEN_ACCOUNTING_ID", "compare_model_reports", + "count_reported_tokens", "finalize_model_report", "load_suite", "run_agent_evaluation", + "run_agent_preflight", "run_live_evaluation", "run_model_benchmark", "run_mcp_evaluations", diff --git a/agent-access/codemesh_agent_access/evaluation/agent.py b/agent-access/codemesh_agent_access/evaluation/agent.py index addfaae..d5c9794 100644 --- a/agent-access/codemesh_agent_access/evaluation/agent.py +++ b/agent-access/codemesh_agent_access/evaluation/agent.py @@ -2,18 +2,37 @@ import asyncio import fnmatch +import hashlib import json import random import shutil import subprocess import tempfile +import tomllib +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from statistics import median from time import perf_counter from typing import Any, Literal -from .codex import AgentRunRequest, AgentRunner, CodexRunner, sanitize_text +from ..installer import ( + InstallationPlan, + codex_mcp_overrides, + verify_plan_applied, +) +from ..mcp import tool_manifest +from ..probe import RuntimeProbeError, run_runtime_probe, run_runtime_rejection_probe +from .codex import ( + REPORTED_TOKEN_ACCOUNTING_DESCRIPTION, + REPORTED_TOKEN_ACCOUNTING_ID, + AgentRunRequest, + AgentRunner, + AgentRunnerCapabilities, + CodexRunner, + count_reported_tokens, + sanitize_text, +) from .live import EvaluationInfrastructureError, get_repository_preflight from .metrics import target_matches from .models import ( @@ -22,7 +41,9 @@ AgentSuite, CandidateRecord, ValidationCommand, + validate_suite_artifacts, ) +from .paths import managed_artifact_path, resolve_setup_patch, validate_artifact_id _ANSWER_SCHEMA = { @@ -49,6 +70,99 @@ } +@dataclass(frozen=True) +class AgentEvaluationContext: + root: Path + base_commit: str + preflight: dict[str, Any] + freshness: dict[str, Any] + mcp_cwd: Path + codemesh_root: Path + codemesh_source_commit: str + codemesh_source_branch: str + integration_mode: str + control_mcp_config: tuple[str, ...] | None + treatment_mcp_config: tuple[str, ...] | None + configured_integration: dict[str, Any] | None + configured_guidance_files: tuple[ConfiguredGuidanceFile, ...] = () + + +@dataclass(frozen=True) +class ConfiguredGuidanceFile: + path: str + content: bytes + sha256: str + + +@dataclass(frozen=True) +class ReportedTokenBudget: + ceiling: int + runner_capabilities: AgentRunnerCapabilities + + +def require_reported_token_hard_cap( + runner: AgentRunner, requested_tokens: int +) -> ReportedTokenBudget: + if ( + isinstance(requested_tokens, bool) + or not isinstance(requested_tokens, int) + or requested_tokens < 1 + ): + raise ValueError("Reported-token cap must be a positive integer.") + capabilities = getattr(runner, "capabilities", None) + if not isinstance(capabilities, AgentRunnerCapabilities): + raise EvaluationInfrastructureError( + "Selected agent runner does not declare the required hard reported-token " + "cap capability; preflight refused before any model call." + ) + if not capabilities.hard_reported_token_cap: + raise EvaluationInfrastructureError( + "Selected agent runner cannot guarantee the requested hard reported-token " + "cap; preflight refused before any model call." + ) + if capabilities.reported_token_accounting != REPORTED_TOKEN_ACCOUNTING_ID: + raise EvaluationInfrastructureError( + "Selected agent runner does not guarantee the required reported-token " + f"accounting {REPORTED_TOKEN_ACCOUNTING_ID!r}; preflight refused before " + "any model call." + ) + return ReportedTokenBudget( + ceiling=requested_tokens, + runner_capabilities=capabilities, + ) + + +async def run_agent_preflight( + suite: AgentSuite, + *, + max_reported_tokens: int, + runner: AgentRunner, + repository_root: str | Path | None = None, + repository_id: str | None = None, + configured_plan: InstallationPlan | None = None, + configured_guidance_paths: list[str] | None = None, + configured_probe_query: str = "repository implementation entry points", + configured_expected_paths: list[str] | None = None, + configured_probe_timeout_seconds: int = 30, +) -> dict[str, Any]: + budget = require_reported_token_hard_cap(runner, max_reported_tokens) + if suite.resolved_integration_mode != "configured": + raise ValueError( + "Provider-free agent preflight is available only for configured suites." + ) + context = await _prepare_agent_context( + suite, + repository_root=repository_root, + repository_id=repository_id, + configured_plan=configured_plan, + configured_guidance_paths=configured_guidance_paths, + configured_probe_query=configured_probe_query, + configured_expected_paths=configured_expected_paths, + configured_probe_timeout_seconds=configured_probe_timeout_seconds, + ) + return _preflight_report(suite, context, budget) + + async def run_agent_evaluation( suite: AgentSuite, suite_path: Path, @@ -57,6 +171,7 @@ async def run_agent_evaluation( local_provider: Literal["lmstudio", "ollama"] | None = None, codex_executable: str = "codex", reasoning_effort: str = "medium", + max_reported_tokens: int, repetitions: int = 3, seed: int = 42, repository_root: str | Path | None = None, @@ -65,43 +180,31 @@ async def run_agent_evaluation( keep_raw_traces: bool = False, keep_workspaces: bool = False, runner: AgentRunner | None = None, + configured_plan: InstallationPlan | None = None, + configured_guidance_paths: list[str] | None = None, + configured_probe_query: str = "repository implementation entry points", + configured_expected_paths: list[str] | None = None, + configured_probe_timeout_seconds: int = 30, ) -> AgentReport: if repetitions < 1: raise ValueError("Agent evaluation requires repetitions >= 1.") started = perf_counter() - root = _git_root(Path(repository_root) if repository_root else Path.cwd()) - if _git(root, "status", "--porcelain"): - raise EvaluationInfrastructureError( - "Agent evaluation requires a clean source checkout so both conditions use the same commit." - ) - base_commit = _git(root, "rev-parse", "HEAD") - resolved_repository = repository_id or suite.repository_id - preflight = await get_repository_preflight(resolved_repository) - freshness = preflight.get("freshness") or {} - if ( - freshness.get("indexed_commit") != base_commit - or freshness.get("current_commit") != base_commit - ): - raise EvaluationInfrastructureError( - "The CodeMesh indexed commit, repository checkout, and benchmark base commit must match." - ) - if suite.repository_commit and base_commit != suite.repository_commit: - raise EvaluationInfrastructureError( - f"Benchmark commit {base_commit!r} does not match the suite's pinned " - f"commit {suite.repository_commit!r}." - ) - codex_runner = runner or CodexRunner(codex_executable) + budget = require_reported_token_hard_cap(codex_runner, max_reported_tokens) + validate_suite_artifacts(suite, suite_path) + context = await _prepare_agent_context( + suite, + repository_root=repository_root, + repository_id=repository_id, + configured_plan=configured_plan, + configured_guidance_paths=configured_guidance_paths, + configured_probe_query=configured_probe_query, + configured_expected_paths=configured_expected_paths, + configured_probe_timeout_seconds=configured_probe_timeout_seconds, + ) + root = context.root + base_commit = context.base_commit rng = random.Random(seed) - mcp_cwd = Path(__file__).resolve().parents[2] - codemesh_root = _git_root(mcp_cwd) - if _git(codemesh_root, "status", "--porcelain"): - raise EvaluationInfrastructureError( - "Agent evaluation requires a clean CodeMesh tool-source checkout so " - "the treatment surface has a frozen commit identity." - ) - codemesh_source_commit = _git(codemesh_root, "rev-parse", "HEAD") - codemesh_source_branch = _git(codemesh_root, "rev-parse", "--abbrev-ref", "HEAD") destination = ( Path(output_dir) if output_dir @@ -109,13 +212,29 @@ async def run_agent_evaluation( ) destination.mkdir(parents=True, exist_ok=False) runs: list[AgentExecutionReport] = [] + aggregate_reported_tokens = 0 + campaign_failures: list[str] = [] + stopped_for_reported_token_cap = False for task in suite.tasks: + if campaign_failures: + break for repetition in range(1, repetitions + 1): + if campaign_failures: + break conditions = ["control", "treatment"] if rng.random() < 0.5: conditions.reverse() for condition in conditions: + remaining_reported_tokens = budget.ceiling - aggregate_reported_tokens + if remaining_reported_tokens == 0: + stopped_for_reported_token_cap = True + campaign_failures.append( + "Campaign stopped at the hard reported-token cap before the " + f"next scheduled run ({task.id}:{repetition}:{condition}); " + "no retry was attempted." + ) + break report = await _run_task_condition( suite_path=suite_path, task=task, @@ -126,16 +245,51 @@ async def run_agent_evaluation( model=model, local_provider=local_provider, reasoning_effort=reasoning_effort, - mcp_cwd=mcp_cwd, + mcp_cwd=context.mcp_cwd, keep_raw_traces=keep_raw_traces, keep_workspaces=keep_workspaces, runner=codex_runner, artifact_dir=destination, + mcp_config=( + context.treatment_mcp_config + if condition == "treatment" + else context.control_mcp_config + ), treatment_guidance=suite.treatment_guidance, + configured_guidance_files=context.configured_guidance_files, + max_reported_tokens=remaining_reported_tokens, ) + used_reported_tokens = _execution_reported_tokens(report) + if used_reported_tokens > remaining_reported_tokens: + raise EvaluationInfrastructureError( + "Agent runner violated its declared hard reported-token cap: " + f"run reported {used_reported_tokens} tokens against a " + f"{remaining_reported_tokens}-token limit." + ) runs.append(report) + aggregate_reported_tokens += used_reported_tokens + if not report.completed: + campaign_failures.append( + "Campaign stopped after an incomplete run " + f"({task.id}:{repetition}:{condition}); no retry was attempted." + ) + break verdict, summary = classify_agent_results(runs, suite) + if campaign_failures: + verdict = "insufficient" + summary.update( + { + "reported_token_accounting": REPORTED_TOKEN_ACCOUNTING_ID, + "reported_token_ceiling": budget.ceiling, + "aggregate_reported_tokens": aggregate_reported_tokens, + "remaining_reported_tokens": (budget.ceiling - aggregate_reported_tokens), + "reported_token_ceiling_met": (aggregate_reported_tokens <= budget.ceiling), + "stopped_for_reported_token_cap": stopped_for_reported_token_cap, + "automatic_retries": False, + "runner": budget.runner_capabilities.runner_id, + } + ) report = AgentReport( suite=suite.name, model=model, @@ -145,28 +299,44 @@ async def run_agent_evaluation( generated_at=datetime.now(timezone.utc).isoformat(), duration_ms=round((perf_counter() - started) * 1000, 3), verdict=verdict, - passed=verdict != "regressed", + passed=verdict != "regressed" and not campaign_failures, metadata={ - "repository_id": preflight.get("repository_id") or resolved_repository, + "repository_id": context.preflight.get("repository_id") + or repository_id + or suite.repository_id, "base_commit": base_commit, "model_provider": local_provider or "default", "codex_executable": codex_executable, - "current_branch": freshness.get("current_branch"), + "current_branch": context.freshness.get("current_branch"), "suite_repository_url": suite.repository_url, "suite_repository_commit": suite.repository_commit, "raw_traces_retained": keep_raw_traces, "source_checkout_modified": False, - "codemesh_source_commit": codemesh_source_commit, - "codemesh_source_branch": codemesh_source_branch, + "codemesh_source_commit": context.codemesh_source_commit, + "codemesh_source_branch": context.codemesh_source_branch, "codemesh_source_checkout_modified": False, "classification_requires_codemesh_adoption": True, - "adoption_mode": "assisted" if suite.treatment_guidance else "spontaneous", - "prompt_parity": suite.treatment_guidance is None, + "adoption_mode": context.integration_mode, + "integration_mode": context.integration_mode, + "prompt_parity": context.integration_mode != "assisted", "treatment_guidance": suite.treatment_guidance, + "configured_integration": context.configured_integration, "artifact_directory": str(destination.resolve()), + "reported_token_budget": { + "accounting": REPORTED_TOKEN_ACCOUNTING_ID, + "accounting_description": REPORTED_TOKEN_ACCOUNTING_DESCRIPTION, + "ceiling": budget.ceiling, + "hard_cap_enforced": True, + "runner_hard_cap_capability": ( + budget.runner_capabilities.hard_reported_token_cap + ), + "automatic_retries": False, + "runner": budget.runner_capabilities.runner_id, + }, }, summary=summary, runs=runs, + failures=campaign_failures, ) (destination / "report.json").write_text( json.dumps(report.model_dump(mode="json"), indent=2), encoding="utf-8" @@ -174,6 +344,462 @@ async def run_agent_evaluation( return report +async def _prepare_agent_context( + suite: AgentSuite, + *, + repository_root: str | Path | None, + repository_id: str | None, + configured_plan: InstallationPlan | None, + configured_guidance_paths: list[str] | None, + configured_probe_query: str, + configured_expected_paths: list[str] | None, + configured_probe_timeout_seconds: int, +) -> AgentEvaluationContext: + root = _git_root(Path(repository_root) if repository_root else Path.cwd()) + if _git(root, "status", "--porcelain"): + raise EvaluationInfrastructureError( + "Agent evaluation requires a clean source checkout so both conditions " + "use the same commit." + ) + base_commit = _git(root, "rev-parse", "HEAD") + if suite.repository_commit and base_commit != suite.repository_commit: + raise EvaluationInfrastructureError( + f"Benchmark commit {base_commit!r} does not match the suite's pinned " + f"commit {suite.repository_commit!r}." + ) + + resolved_repository = repository_id or suite.repository_id + preflight = await get_repository_preflight(resolved_repository) + freshness = preflight.get("freshness") or {} + if ( + freshness.get("indexed_commit") != base_commit + or freshness.get("current_commit") != base_commit + or freshness.get("working_tree_dirty") is True + ): + raise EvaluationInfrastructureError( + "The CodeMesh indexed commit, clean repository checkout, and benchmark " + "base commit must match." + ) + + mcp_cwd = Path(__file__).resolve().parents[2] + codemesh_root = _git_root(mcp_cwd) + if _git(codemesh_root, "status", "--porcelain"): + raise EvaluationInfrastructureError( + "Agent evaluation requires a clean CodeMesh tool-source checkout so " + "the treatment surface has a frozen commit identity." + ) + codemesh_source_commit = _git(codemesh_root, "rev-parse", "HEAD") + codemesh_source_branch = _git(codemesh_root, "rev-parse", "--abbrev-ref", "HEAD") + integration_mode = suite.resolved_integration_mode + configured_integration: dict[str, Any] | None = None + configured_guidance_files: tuple[ConfiguredGuidanceFile, ...] = () + control_mcp_config: tuple[str, ...] | None = None + treatment_mcp_config: tuple[str, ...] | None = None + if integration_mode == "configured": + if not suite.repository_commit: + raise EvaluationInfrastructureError( + "Configured agent evaluation requires a pinned repository commit." + ) + if configured_plan is None: + raise EvaluationInfrastructureError( + "Configured agent evaluation requires a reviewed installation plan." + ) + ( + configured_integration, + control_mcp_config, + treatment_mcp_config, + configured_guidance_files, + ) = await _prepare_configured_integration( + configured_plan, + root=root, + base_commit=base_commit, + codemesh_root=codemesh_root, + preflight=preflight, + guidance_paths=configured_guidance_paths, + probe_query=configured_probe_query, + expected_paths=configured_expected_paths, + timeout_seconds=configured_probe_timeout_seconds, + baseline_mcp_servers=suite.baseline_mcp_servers, + ) + elif configured_plan is not None: + raise EvaluationInfrastructureError( + "An installation plan can be used only by a configured agent suite." + ) + + return AgentEvaluationContext( + root=root, + base_commit=base_commit, + preflight=preflight, + freshness=freshness, + mcp_cwd=mcp_cwd, + codemesh_root=codemesh_root, + codemesh_source_commit=codemesh_source_commit, + codemesh_source_branch=codemesh_source_branch, + integration_mode=integration_mode, + control_mcp_config=control_mcp_config, + treatment_mcp_config=treatment_mcp_config, + configured_integration=configured_integration, + configured_guidance_files=configured_guidance_files, + ) + + +async def _prepare_configured_integration( + plan: InstallationPlan, + *, + root: Path, + base_commit: str, + codemesh_root: Path, + preflight: dict[str, Any], + guidance_paths: list[str] | None, + probe_query: str, + expected_paths: list[str] | None, + timeout_seconds: int, + baseline_mcp_servers: list[str], +) -> tuple[ + dict[str, Any], + tuple[str, ...], + tuple[str, ...], + tuple[ConfiguredGuidanceFile, ...], +]: + if timeout_seconds < 1: + raise ValueError("Configured probe timeout must be positive.") + if plan.profile != "normal": + raise EvaluationInfrastructureError( + "Configured agent evaluation requires the shipped normal MCP profile." + ) + try: + target_root = Path(plan.target_root).expanduser().resolve(strict=True) + plan_codemesh_root = Path(plan.codemesh_root).expanduser().resolve(strict=True) + verify_plan_applied(plan) + codemesh_config = codex_mcp_overrides(plan, approve_tools=True) + except (OSError, ValueError) as exc: + raise EvaluationInfrastructureError( + f"Configured installation plan is not applied unchanged: {exc}" + ) from exc + if target_root != root: + raise EvaluationInfrastructureError( + "Configured installation target does not match the benchmark checkout." + ) + if plan_codemesh_root != codemesh_root: + raise EvaluationInfrastructureError( + "Configured installation does not use the current CodeMesh tool source." + ) + + binding = plan.binding + source_view_hash = str(binding.get("source_view_hash") or "") + if not source_view_hash: + raise EvaluationInfrastructureError( + "Configured agent evaluation requires an exact source-view binding." + ) + repository = preflight.get("repository") or {} + expected_project = repository.get("project_id") or preflight.get("repository_id") + checks = { + "project": binding.get("project_id") == expected_project, + "checkout": binding.get("checkout_id") == repository.get("checkout_id"), + "root": _same_path(str(binding.get("repository_root") or ""), root), + "registered_root": _same_path(str(repository.get("root_path") or ""), root), + "source_view": source_view_hash == repository.get("source_view_hash"), + } + failed = sorted(name for name, passed in checks.items() if not passed) + if failed: + raise EvaluationInfrastructureError( + "Configured installation binding does not match repository preflight: " + + ", ".join(failed) + + "." + ) + + expected_tools = sorted( + item["name"] for item in tool_manifest(profile="normal")["tools"] + ) + if ( + sorted(str(value) for value in plan.launch.get("expected_tools", [])) + != expected_tools + ): + raise EvaluationInfrastructureError( + "Configured installation tool surface is not the shipped normal profile." + ) + guidance, guidance_files = _guidance_identities( + root, guidance_paths or ["AGENTS.md"] + ) + baseline_config, baseline_identities = _baseline_mcp_overrides( + plan, baseline_mcp_servers + ) + treatment_config = (*baseline_config, *codemesh_config) + + wrong_checkout = f"{binding['checkout_id']}:codemesh-eval-rejection" + try: + positive = await run_runtime_probe( + plan, + timeout_seconds=timeout_seconds, + query=probe_query, + expected_paths=expected_paths or [], + ) + negative = await run_runtime_rejection_probe( + plan, + timeout_seconds=timeout_seconds, + checkout_id=wrong_checkout, + expected_substrings=["bound_repository_missing"], + ) + except (OSError, ValueError, RuntimeProbeError) as exc: + raise EvaluationInfrastructureError( + f"Configured installation probe failed: {exc}" + ) from exc + if ( + positive.get("indexed_commit") != base_commit + or positive.get("current_commit") != base_commit + or positive.get("source_view_hash") != source_view_hash + or positive.get("binding_status") != "accepted" + ): + raise EvaluationInfrastructureError( + "Configured runtime probe identity does not match the benchmark candidate." + ) + if negative.get("passed") is not True or negative.get("rejected") is not True: + raise EvaluationInfrastructureError( + "Configured rejection probe did not prove fail-closed checkout binding." + ) + + installation_files = [ + {"path": item.path, "sha256": item.new_sha256} for item in plan.files + ] + metadata = { + "schema_version": "codemesh-configured-agent-integration-v1", + "plan_hash": plan.plan_hash, + "plan_schema_version": plan.schema_version, + "server_name": plan.server_name, + "profile": plan.profile, + "binding": dict(binding), + "launch_sha256": _identity_hash(plan.launch), + "installation_files": installation_files, + "guidance": guidance, + "baseline_mcp_servers": baseline_identities, + "positive_probe": _probe_identity(positive), + "rejection_probe": _probe_identity(negative), + } + return metadata, baseline_config, treatment_config, guidance_files + + +def _preflight_report( + suite: AgentSuite, + context: AgentEvaluationContext, + budget: ReportedTokenBudget, +) -> dict[str, Any]: + return { + "schema_version": "codemesh-agent-preflight-v1", + "passed": True, + "suite": suite.name, + "integration_mode": context.integration_mode, + "prompt_parity": context.integration_mode != "assisted", + "repository_id": context.preflight.get("repository_id") or suite.repository_id, + "base_commit": context.base_commit, + "current_branch": context.freshness.get("current_branch"), + "codemesh_source_commit": context.codemesh_source_commit, + "codemesh_source_branch": context.codemesh_source_branch, + "configured_integration": context.configured_integration, + "reported_token_budget": { + "accounting": REPORTED_TOKEN_ACCOUNTING_ID, + "accounting_description": REPORTED_TOKEN_ACCOUNTING_DESCRIPTION, + "ceiling": budget.ceiling, + "hard_cap_capability_verified": True, + "runner_hard_cap_capability": ( + budget.runner_capabilities.hard_reported_token_cap + ), + "automatic_retries": False, + "runner": budget.runner_capabilities.runner_id, + }, + } + + +def _guidance_identities( + root: Path, paths: list[str] +) -> tuple[list[dict[str, str]], tuple[ConfiguredGuidanceFile, ...]]: + identities: list[dict[str, str]] = [] + guidance_files: list[ConfiguredGuidanceFile] = [] + for value in paths: + relative = Path(value) + if relative.is_absolute() or ".." in relative.parts: + raise EvaluationInfrastructureError( + f"Configured guidance path must be repository-relative: {value}." + ) + path = (root / relative).resolve(strict=False) + if root != path and root not in path.parents: + raise EvaluationInfrastructureError( + f"Configured guidance path escapes the repository: {value}." + ) + if not path.is_file(): + raise EvaluationInfrastructureError( + f"Configured guidance file does not exist: {relative.as_posix()}." + ) + tracked = _run_command( + [ + "git", + "-C", + str(root), + "ls-files", + "--error-unmatch", + "--", + relative.as_posix(), + ], + timeout=30, + ) + source = "tracked" + if tracked.returncode != 0: + ignored = _run_command( + [ + "git", + "-C", + str(root), + "check-ignore", + "--quiet", + "--no-index", + "--", + relative.as_posix(), + ], + timeout=30, + ) + if ignored.returncode != 0: + raise EvaluationInfrastructureError( + "Configured guidance must be committed or an explicitly " + f"ignored installed file: {relative.as_posix()}." + ) + source = "installed-ignored" + content = path.read_bytes() + sha256 = hashlib.sha256(content).hexdigest() + identities.append( + { + "path": relative.as_posix(), + "sha256": sha256, + "source": source, + } + ) + guidance_files.append( + ConfiguredGuidanceFile( + path=relative.as_posix(), + content=content, + sha256=sha256, + ) + ) + if not identities: + raise EvaluationInfrastructureError( + "Configured agent evaluation requires at least one guidance file." + ) + return identities, tuple(guidance_files) + + +def _baseline_mcp_overrides( + plan: InstallationPlan, server_names: list[str] +) -> tuple[tuple[str, ...], list[dict[str, Any]]]: + overrides = ["mcp_servers={}"] + if not server_names: + return tuple(overrides), [] + config_file = next( + (item for item in plan.files if item.path == ".codex/config.toml"), None + ) + if config_file is None: + raise EvaluationInfrastructureError( + "Configured installation plan does not manage .codex/config.toml." + ) + try: + payload = tomllib.loads(config_file.content) + except tomllib.TOMLDecodeError as exc: + raise EvaluationInfrastructureError( + f"Configured installation contains invalid Codex configuration: {exc}" + ) from exc + servers = payload.get("mcp_servers") + if not isinstance(servers, dict): + raise EvaluationInfrastructureError( + "Configured installation contains no MCP server table." + ) + + identities: list[dict[str, Any]] = [] + allowed_fields = { + "command", + "args", + "cwd", + "required", + "startup_timeout_sec", + "tool_timeout_sec", + "enabled_tools", + "disabled_tools", + "env_vars", + } + for name in server_names: + if name == plan.server_name: + raise EvaluationInfrastructureError( + "The configured CodeMesh server cannot also be a control baseline." + ) + config = servers.get(name) + if not isinstance(config, dict): + raise EvaluationInfrastructureError( + f"Configured baseline MCP server is missing: {name}." + ) + unsupported = sorted(set(config) - allowed_fields - {"env"}) + if unsupported: + raise EvaluationInfrastructureError( + f"Baseline MCP server '{name}' uses unsupported field(s): " + + ", ".join(unsupported) + + "." + ) + if config.get("env"): + raise EvaluationInfrastructureError( + f"Baseline MCP server '{name}' embeds environment values; use " + "env_vars names instead." + ) + if not isinstance(config.get("command"), str) or not config["command"]: + raise EvaluationInfrastructureError( + f"Baseline MCP server '{name}' has no launch command." + ) + prefix = f"mcp_servers.{name}" + for key in sorted(allowed_fields): + if key not in config: + continue + overrides.append(f"{prefix}.{key}={_toml_override(config[key])}") + overrides.append(f'{prefix}.default_tools_approval_mode="approve"') + identities.append( + { + "name": name, + "configuration_sha256": _identity_hash(config), + "enabled_tools": list(config.get("enabled_tools") or []), + "env_vars": list(config.get("env_vars") or []), + } + ) + return tuple(overrides), identities + + +def _toml_override(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (str, int, float, list, dict)): + return json.dumps(value, separators=(",", ":")) + raise EvaluationInfrastructureError( + f"Unsupported MCP configuration value type: {type(value).__name__}." + ) + + +def _probe_identity(report: dict[str, Any]) -> dict[str, Any]: + stable = { + key: value + for key, value in report.items() + if key not in {"diagnostic", "duration_ms"} + } + return {**stable, "report_sha256": _identity_hash(stable)} + + +def _identity_hash(value: Any) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _same_path(value: str, expected: Path) -> bool: + if not value: + return False + return Path(value).expanduser().resolve(strict=False) == expected + + async def _run_task_condition( *, suite_path: Path, @@ -192,7 +818,16 @@ async def _run_task_condition( artifact_dir: Path, mcp_config: tuple[str, ...] | None = None, treatment_guidance: str | None = None, + configured_guidance_files: tuple[ConfiguredGuidanceFile, ...] = (), + max_reported_tokens: int | None = None, ) -> AgentExecutionReport: + validate_artifact_id(task.id) + patch_path = ( + resolve_setup_patch(suite_path, task.setup_patch) if task.setup_patch else None + ) + run_name = f"{task.id}-{repetition}-{condition}" + run_dir = managed_artifact_path(artifact_dir, run_name) + retained = managed_artifact_path(artifact_dir, "workspaces", run_name) temp_parent = Path(tempfile.mkdtemp(prefix="codemesh-eval-")) workspace = temp_parent / "workspace" try: @@ -218,12 +853,8 @@ async def _run_task_condition( base_commit, ] ) - if task.setup_patch: - patch_path = (suite_path.parent / task.setup_patch).resolve() - if not patch_path.is_file(): - raise EvaluationInfrastructureError( - f"Setup patch not found: {patch_path}" - ) + _materialize_configured_guidance(workspace, configured_guidance_files) + if patch_path is not None: _run_checked( ["git", "-C", str(workspace), "apply", "--check", str(patch_path)] ) @@ -288,9 +919,17 @@ async def _run_task_condition( output_schema=schema_path if task.task_type == "answer" else None, keep_raw_traces=keep_raw_traces, mcp_config=mcp_config, + max_reported_tokens=max_reported_tokens, ) ) + try: + reported_tokens = count_reported_tokens(result.usage) + except ValueError as exc: + raise EvaluationInfrastructureError( + f"Agent runner returned invalid reported-token usage: {exc}" + ) from exc + failures = list(result.failures) validation: list[dict[str, Any]] = [] changed_paths = _changed_paths(workspace) @@ -335,6 +974,8 @@ async def _run_task_condition( cached_input_tokens=result.usage.get("cached_input_tokens", 0), output_tokens=result.usage.get("output_tokens", 0), reasoning_output_tokens=result.usage.get("reasoning_output_tokens", 0), + reported_tokens=reported_tokens, + reported_token_limit=max_reported_tokens, mcp_calls=result.mcp_calls, mcp_call_attempts=result.mcp_call_attempts, mcp_failed_calls=result.mcp_failed_calls, @@ -349,10 +990,10 @@ async def _run_task_condition( sanitized_diff=sanitized_diff, raw_events=result.raw_events if keep_raw_traces else None, final_message=result.final_message if keep_raw_traces else None, + hard_cap_ledger=result.hard_cap_ledger, ) if keep_raw_traces: - run_dir = artifact_dir / f"{task.id}-{repetition}-{condition}" - run_dir.mkdir(parents=True, exist_ok=True) + run_dir.mkdir(parents=True, exist_ok=False) (run_dir / "events.jsonl").write_text( "\n".join(json.dumps(event) for event in result.raw_events), encoding="utf-8", @@ -362,9 +1003,6 @@ async def _run_task_condition( ) (run_dir / "changes.diff").write_text(diff, encoding="utf-8") if keep_workspaces: - retained = ( - artifact_dir / "workspaces" / f"{task.id}-{repetition}-{condition}" - ) retained.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(workspace), retained) return run_report @@ -373,6 +1011,24 @@ async def _run_task_condition( shutil.rmtree(temp_parent) +def _materialize_configured_guidance( + workspace: Path, guidance_files: tuple[ConfiguredGuidanceFile, ...] +) -> None: + for item in guidance_files: + relative = Path(item.path) + target = (workspace / relative).resolve(strict=False) + if workspace != target and workspace not in target.parents: + raise EvaluationInfrastructureError( + f"Configured guidance path escapes evaluation workspace: {item.path}." + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(item.content) + if hashlib.sha256(target.read_bytes()).hexdigest() != item.sha256: + raise EvaluationInfrastructureError( + f"Configured guidance identity changed while preparing: {item.path}." + ) + + def classify_agent_results( runs: list[AgentExecutionReport], suite: AgentSuite ) -> tuple[str, dict[str, Any]]: @@ -694,7 +1350,23 @@ def _run_command( def _tokens(run: AgentExecutionReport) -> float: - return float(run.input_tokens + run.output_tokens + run.reasoning_output_tokens) + return float(_execution_reported_tokens(run)) + + +def _execution_reported_tokens(run: AgentExecutionReport) -> int: + counted = count_reported_tokens( + { + "input_tokens": run.input_tokens, + "cached_input_tokens": run.cached_input_tokens, + "output_tokens": run.output_tokens, + "reasoning_output_tokens": run.reasoning_output_tokens, + } + ) + if run.reported_tokens not in {0, counted}: + raise EvaluationInfrastructureError( + "Agent execution report contains inconsistent reported-token accounting." + ) + return counted def _median(values: list[float]) -> float: diff --git a/agent-access/codemesh_agent_access/evaluation/benchmark.py b/agent-access/codemesh_agent_access/evaluation/benchmark.py index 203fc1a..e9bc637 100644 --- a/agent-access/codemesh_agent_access/evaluation/benchmark.py +++ b/agent-access/codemesh_agent_access/evaluation/benchmark.py @@ -40,7 +40,9 @@ ReviewFile, ToolPolicy, load_suite, + validate_suite_artifacts, ) +from .paths import managed_artifact_path, validate_artifact_id _CRITERIA = ( @@ -87,6 +89,7 @@ async def run_model_benchmark( f"Model benchmark requires at least {suite.thresholds.minimum_repetitions} repetitions." ) started = perf_counter() + validate_suite_artifacts(suite, suite_path) root = _git_root(Path(repository_root) if repository_root else Path.cwd()) if _git(root, "status", "--porcelain"): raise EvaluationInfrastructureError( @@ -248,6 +251,13 @@ async def _run_query( keep_workspace: bool, weights: CriterionWeights, ) -> BenchmarkRunReport: + validate_artifact_id(question.id) + run_dir = managed_artifact_path( + artifact_dir, f"{question.id}-{repetition}-treatment" + ) + retained = managed_artifact_path( + artifact_dir, "workspaces", f"{question.id}-{repetition}" + ) temp_parent = Path(tempfile.mkdtemp(prefix="codemesh-model-query-")) workspace = temp_parent / "workspace" response_path = temp_parent / "response.json" @@ -288,8 +298,7 @@ async def _run_query( if not keep_raw_traces: report.raw_events = None if keep_raw_traces: - run_dir = artifact_dir / f"{question.id}-{repetition}-treatment" - run_dir.mkdir(parents=True, exist_ok=True) + run_dir.mkdir(parents=True, exist_ok=False) (run_dir / "events.jsonl").write_text( "\n".join(json.dumps(event) for event in result.raw_events), encoding="utf-8", @@ -298,7 +307,6 @@ async def _run_query( json.dumps(payload, indent=2), encoding="utf-8" ) if keep_workspace: - retained = artifact_dir / "workspaces" / f"{question.id}-{repetition}" retained.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(workspace), retained) return report @@ -324,8 +332,9 @@ async def _run_change( keep_workspaces: bool, weights: CriterionWeights, ) -> BenchmarkRunReport: - audit_path = ( - artifact_dir / ".runtime" / f"{scenario.id}-{repetition}-{condition}.json" + validate_artifact_id(scenario.id) + audit_path = managed_artifact_path( + artifact_dir, ".runtime", f"{scenario.id}-{repetition}-{condition}.json" ) policy = scenario.tool_policy mcp_config = ( diff --git a/agent-access/codemesh_agent_access/evaluation/capped_codex.py b/agent-access/codemesh_agent_access/evaluation/capped_codex.py new file mode 100644 index 0000000..a946825 --- /dev/null +++ b/agent-access/codemesh_agent_access/evaluation/capped_codex.py @@ -0,0 +1,692 @@ +from __future__ import annotations + +import asyncio +import hashlib +import hmac +import json +import os +import secrets +import socket +import subprocess +import tempfile +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, AsyncIterator, Mapping + +import httpx +import uvicorn +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, Response, StreamingResponse +from starlette.routing import Route + +from .codex import ( + REPORTED_TOKEN_ACCOUNTING_ID, + AgentRunRequest, + AgentRunResult, + AgentRunnerCapabilities, + CodexRunner, + count_reported_tokens, +) + + +_INPUT_TOKEN_FIELDS = { + "conversation", + "input", + "instructions", + "model", + "parallel_tool_calls", + "personality", + "previous_response_id", + "reasoning", + "text", + "tool_choice", + "tools", + "truncation", +} +_ALLOWED_RESPONSE_FIELDS = _INPUT_TOKEN_FIELDS | { + "background", + "client_metadata", + "context_management", + "include", + "max_output_tokens", + "max_tool_calls", + "metadata", + "prompt_cache_key", + "prompt_cache_options", + "prompt_cache_retention", + "safety_identifier", + "service_tier", + "store", + "stream", + "stream_options", + "temperature", + "top_logprobs", + "top_p", + "user", +} +_ALLOWED_LOCAL_TOOL_TYPES = {"custom", "function"} +_MAX_GENERATED_TOKENS_PER_REQUEST = 128_000 +_USAGE_FIELDS = ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_output_tokens", +) + + +class CappedProxyError(RuntimeError): + pass + + +@dataclass(frozen=True) +class _Reservation: + sequence: int + input_tokens: int + reserved_tokens: int + + +class HardCapLedger: + def __init__(self, ceiling: int) -> None: + if isinstance(ceiling, bool) or not isinstance(ceiling, int) or ceiling < 1: + raise ValueError("Hard-cap ceiling must be a positive integer.") + self.ceiling = ceiling + self.committed_tokens = 0 + self.reported_usage = {field: 0 for field in _USAGE_FIELDS} + self.request_count = 0 + self.completed_request_count = 0 + self.failures: list[str] = [] + + def reserve( + self, input_tokens: int, requested_output_tokens: int | None + ) -> tuple[_Reservation, int]: + if self.failures: + raise CappedProxyError("The hard-cap ledger is already failed closed.") + if isinstance(input_tokens, bool) or input_tokens < 0: + raise CappedProxyError("Input-token counting returned an invalid value.") + if requested_output_tokens is not None and ( + isinstance(requested_output_tokens, bool) + or not isinstance(requested_output_tokens, int) + or requested_output_tokens < 1 + ): + raise CappedProxyError( + "The Responses request supplied an invalid max_output_tokens value." + ) + remaining = self.ceiling - self.committed_tokens + output_allowance = min( + (remaining - input_tokens) // 2, + _MAX_GENERATED_TOKENS_PER_REQUEST, + ) + if output_allowance < 1: + raise CappedProxyError( + "Insufficient remaining hard-cap allowance for the counted input and " + "one generated token under output-plus-reasoning accounting." + ) + if requested_output_tokens is not None: + output_allowance = min(output_allowance, requested_output_tokens) + reservation = _Reservation( + sequence=self.request_count + 1, + input_tokens=input_tokens, + reserved_tokens=input_tokens + (2 * output_allowance), + ) + self.request_count += 1 + self.committed_tokens += reservation.reserved_tokens + return reservation, output_allowance + + def settle(self, reservation: _Reservation, usage: Mapping[str, Any]) -> None: + normalized = normalize_response_usage(usage) + if normalized["input_tokens"] != reservation.input_tokens: + raise CappedProxyError( + "Response usage disagreed with the preflight input-token count." + ) + reported_tokens = count_reported_tokens(normalized) + if reported_tokens > reservation.reserved_tokens: + raise CappedProxyError("Response usage exceeded its hard-cap reservation.") + self.committed_tokens -= reservation.reserved_tokens - reported_tokens + for field in _USAGE_FIELDS: + self.reported_usage[field] += normalized[field] + self.completed_request_count += 1 + + def fail(self, message: str) -> None: + if message not in self.failures: + self.failures.append(message) + + def snapshot(self) -> dict[str, Any]: + reported_tokens = count_reported_tokens(self.reported_usage) + return { + "accounting": REPORTED_TOKEN_ACCOUNTING_ID, + "ceiling": self.ceiling, + "committed_tokens": self.committed_tokens, + "reported_tokens": reported_tokens, + "retained_reservation_tokens": self.committed_tokens - reported_tokens, + "remaining_tokens": self.ceiling - self.committed_tokens, + "request_count": self.request_count, + "completed_request_count": self.completed_request_count, + "usage": dict(self.reported_usage), + "request_max_retries": 0, + "stream_max_retries": 0, + "failures": list(self.failures), + "complete": ( + self.request_count > 0 + and self.request_count == self.completed_request_count + and not self.failures + ), + } + + +def normalize_response_usage(usage: Mapping[str, Any]) -> dict[str, int]: + if not isinstance(usage, Mapping): + raise CappedProxyError("A completed Responses call omitted usage accounting.") + input_tokens = _non_negative_integer(usage.get("input_tokens"), "input_tokens") + combined_output_tokens = _non_negative_integer( + usage.get("output_tokens"), "output_tokens" + ) + input_details = usage.get("input_tokens_details") or {} + output_details = usage.get("output_tokens_details") or {} + if not isinstance(input_details, Mapping) or not isinstance( + output_details, Mapping + ): + raise CappedProxyError("Responses usage details had an invalid shape.") + cached_input_tokens = _optional_non_negative_integer( + input_details.get("cached_tokens"), "cached_tokens" + ) + reasoning_output_tokens = _optional_non_negative_integer( + output_details.get("reasoning_tokens"), "reasoning_tokens" + ) + if cached_input_tokens > input_tokens: + raise CappedProxyError("Cached input usage exceeded total input usage.") + if reasoning_output_tokens > combined_output_tokens: + raise CappedProxyError("Reasoning usage exceeded total generated usage.") + return { + "input_tokens": input_tokens, + "cached_input_tokens": cached_input_tokens, + "output_tokens": combined_output_tokens, + "reasoning_output_tokens": reasoning_output_tokens, + } + + +class _SseUsageObserver: + def __init__(self) -> None: + self._buffer = b"" + self.terminal_usage: Mapping[str, Any] | None = None + + def feed(self, chunk: bytes) -> None: + self._buffer += chunk + normalized = self._buffer.replace(b"\r\n", b"\n") + frames = normalized.split(b"\n\n") + self._buffer = frames.pop() + for frame in frames: + data = b"\n".join( + line[5:].lstrip() + for line in frame.splitlines() + if line.startswith(b"data:") + ) + if not data or data == b"[DONE]": + continue + try: + event = json.loads(data) + except json.JSONDecodeError: + continue + if not isinstance(event, dict) or event.get("type") != "response.completed": + continue + response = event.get("response") + if isinstance(response, dict): + usage = response.get("usage") + if isinstance(usage, Mapping): + self.terminal_usage = usage + + +class CappedResponsesProxy: + def __init__( + self, + *, + ceiling: int, + local_token: str, + upstream_api_key: str, + upstream_base_url: str = "https://api.openai.com/v1", + upstream_client: httpx.AsyncClient | None = None, + ) -> None: + self.ledger = HardCapLedger(ceiling) + self._local_token = local_token + self._upstream_api_key = upstream_api_key + self._upstream_base_url = upstream_base_url.rstrip("/") + self._request_lock = asyncio.Lock() + self._seen_request_digests: set[str] = set() + self._owns_client = upstream_client is None + self._client = upstream_client or httpx.AsyncClient( + timeout=None, + transport=httpx.AsyncHTTPTransport(retries=0), + ) + self.app = Starlette( + routes=[ + Route("/v1/responses", self._responses, methods=["POST"]), + Route("/v1/models", self._models, methods=["GET"]), + Route( + "/{path:path}", + self._unsupported, + methods=["GET", "POST", "PUT", "PATCH", "DELETE"], + ), + ] + ) + + async def close(self) -> None: + if self._owns_client: + await self._client.aclose() + + @asynccontextmanager + async def serve(self) -> AsyncIterator[str]: + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(128) + port = int(listener.getsockname()[1]) + config = uvicorn.Config( + self.app, + host="127.0.0.1", + port=port, + access_log=False, + log_config=None, + lifespan="off", + server_header=False, + ) + server = uvicorn.Server(config) + task = asyncio.create_task(server.serve(sockets=[listener])) + try: + await asyncio.wait_for(_wait_for_server(server, task), timeout=5) + yield f"http://127.0.0.1:{port}/v1" + finally: + server.should_exit = True + try: + await asyncio.wait_for(task, timeout=5) + except TimeoutError: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + listener.close() + await self.close() + + async def _responses(self, request: Request) -> Response: + if not _authorized(request, self._local_token): + return self._reject("Unauthorized loopback proxy request.", 401) + retry_count = request.headers.get("x-stainless-retry-count", "0") + if retry_count not in {"", "0"}: + return self._reject("Unrecognized provider retry activity was rejected.") + try: + body = await request.json() + except (json.JSONDecodeError, UnicodeDecodeError): + return self._reject("Responses request body was not valid JSON.") + if not isinstance(body, dict): + return self._reject("Responses request body must be a JSON object.") + invalid = _validate_response_request(body) + if invalid: + return self._reject(invalid) + digest = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + async with self._request_lock: + if digest in self._seen_request_digests: + return self._reject( + "A duplicate Responses request was rejected as retry activity." + ) + self._seen_request_digests.add(digest) + try: + input_tokens = await self._count_input_tokens(body) + reservation, output_allowance = self.ledger.reserve( + input_tokens, body.get("max_output_tokens") + ) + except CappedProxyError as exc: + return self._reject(str(exc)) + + forwarded = dict(body) + forwarded["max_output_tokens"] = output_allowance + try: + upstream = await self._client.send( + self._client.build_request( + "POST", + f"{self._upstream_base_url}/responses", + headers=self._upstream_headers(), + json=forwarded, + ), + stream=True, + ) + except httpx.HTTPError: + self.ledger.fail( + "The upstream Responses request failed after reservation; the " + "reservation was retained." + ) + return JSONResponse({"error": {"message": "Capped upstream failure."}}, 502) + + if upstream.status_code >= 400: + await upstream.aread() + await upstream.aclose() + self.ledger.fail( + "The upstream Responses request was rejected after reservation; the " + "reservation was retained." + ) + return JSONResponse( + {"error": {"message": "Capped upstream request was rejected."}}, + status_code=upstream.status_code, + ) + + if body.get("stream") is True: + return StreamingResponse( + self._stream_response(upstream, reservation), + status_code=upstream.status_code, + media_type="text/event-stream", + ) + + content = await upstream.aread() + await upstream.aclose() + try: + payload = json.loads(content) + if not isinstance(payload, dict) or payload.get("status") != "completed": + raise CappedProxyError( + "The non-streaming Responses call did not complete normally." + ) + self.ledger.settle(reservation, payload.get("usage")) + except (json.JSONDecodeError, CappedProxyError) as exc: + self.ledger.fail(f"{exc} The reservation was retained.") + return Response( + content=content, + status_code=upstream.status_code, + media_type=upstream.headers.get("content-type", "application/json"), + ) + + async def _stream_response( + self, upstream: httpx.Response, reservation: _Reservation + ) -> AsyncIterator[bytes]: + observer = _SseUsageObserver() + settled = False + try: + async for chunk in upstream.aiter_raw(): + observer.feed(chunk) + if observer.terminal_usage is not None and not settled: + self.ledger.settle(reservation, observer.terminal_usage) + settled = True + yield chunk + if not settled: + raise CappedProxyError( + "The Responses stream ended without completed usage." + ) + except BaseException as exc: + if isinstance(exc, (KeyboardInterrupt, SystemExit)): + raise + message = ( + str(exc) + if isinstance(exc, CappedProxyError) + else "The Responses stream was interrupted." + ) + self.ledger.fail(f"{message} The reservation was retained.") + if not isinstance(exc, CappedProxyError): + raise + finally: + await upstream.aclose() + if ( + not settled + and observer.terminal_usage is None + and not self.ledger.failures + ): + self.ledger.fail( + "The Responses stream ended without completed usage; the " + "reservation was retained." + ) + + async def _count_input_tokens(self, body: Mapping[str, Any]) -> int: + count_body = {key: body[key] for key in _INPUT_TOKEN_FIELDS if key in body} + try: + response = await self._client.post( + f"{self._upstream_base_url}/responses/input_tokens", + headers=self._upstream_headers(), + json=count_body, + ) + except httpx.HTTPError as exc: + raise CappedProxyError("Input-token counting failed closed.") from exc + if response.status_code >= 400: + raise CappedProxyError("Input-token counting was rejected upstream.") + try: + payload = response.json() + except json.JSONDecodeError as exc: + raise CappedProxyError( + "Input-token counting returned invalid JSON." + ) from exc + if not isinstance(payload, dict): + raise CappedProxyError("Input-token counting returned an invalid payload.") + return _non_negative_integer(payload.get("input_tokens"), "input_tokens") + + def _upstream_headers(self) -> dict[str, str]: + headers = { + "authorization": f"Bearer {self._upstream_api_key}", + "content-type": "application/json", + } + organization = os.getenv("OPENAI_ORGANIZATION") + project = os.getenv("OPENAI_PROJECT") + if organization: + headers["openai-organization"] = organization + if project: + headers["openai-project"] = project + return headers + + async def _unsupported(self, request: Request) -> Response: + del request + return self._reject( + "Unsupported or cost-bearing provider endpoint was rejected.", 404 + ) + + async def _models(self, request: Request) -> Response: + if not _authorized(request, self._local_token): + return self._reject("Unauthorized loopback proxy request.", 401) + return JSONResponse({"models": []}) + + def _reject(self, message: str, status_code: int = 400) -> JSONResponse: + self.ledger.fail(message) + return JSONResponse({"error": {"message": message}}, status_code=status_code) + + +class CappedCodexRunner: + def __init__( + self, + executable: str = "codex", + *, + supports_ignore_user_config: bool | None = None, + upstream_base_url: str = "https://api.openai.com/v1", + api_key_env: str = "OPENAI_API_KEY", + ) -> None: + self.executable = executable + self._supports_ignore_user_config = supports_ignore_user_config + self._upstream_base_url = upstream_base_url + self._api_key_env = api_key_env + + @property + def capabilities(self) -> AgentRunnerCapabilities: + return AgentRunnerCapabilities( + hard_reported_token_cap=True, + reported_token_accounting=REPORTED_TOKEN_ACCOUNTING_ID, + runner_id="capped-codex", + ) + + async def run(self, request: AgentRunRequest) -> AgentRunResult: + if request.local_provider is not None: + raise ValueError("capped-codex does not support local model providers.") + if request.max_reported_tokens is None: + raise ValueError( + "capped-codex requires a per-execution reported-token cap." + ) + upstream_api_key = os.getenv(self._api_key_env) + if not upstream_api_key: + raise RuntimeError( + f"capped-codex requires the {self._api_key_env} environment variable." + ) + local_token = secrets.token_urlsafe(32) + proxy = CappedResponsesProxy( + ceiling=request.max_reported_tokens, + local_token=local_token, + upstream_api_key=upstream_api_key, + upstream_base_url=self._upstream_base_url, + ) + bundled_model = await asyncio.to_thread(self._load_bundled_model, request.model) + with tempfile.TemporaryDirectory(prefix="codemesh-capped-model-") as temp_dir: + catalog_path = Path(temp_dir) / "models.json" + catalog_path.write_text( + json.dumps({"models": [bundled_model]}), encoding="utf-8" + ) + async with proxy.serve() as base_url: + runner = CodexRunner( + self.executable, + supports_ignore_user_config=self._supports_ignore_user_config, + config_overrides=_capped_provider_config(base_url, catalog_path), + environment_overrides={"CODEMESH_CAPPED_PROXY_TOKEN": local_token}, + ) + result = await runner.run(request) + + if proxy.ledger.request_count == 0: + proxy.ledger.fail("Codex completed without a proxied Responses request.") + elif ( + proxy.ledger.request_count != proxy.ledger.completed_request_count + and not proxy.ledger.failures + ): + proxy.ledger.fail( + "A Responses reservation remained unsettled after Codex exited." + ) + for field in _USAGE_FIELDS: + result.usage.setdefault(field, proxy.ledger.reported_usage[field]) + ledger = proxy.ledger.snapshot() + if result.completed and ledger["complete"]: + try: + codex_reported_tokens = count_reported_tokens(result.usage) + except ValueError as exc: + proxy.ledger.fail(f"Codex final usage was invalid: {exc}") + else: + if codex_reported_tokens != ledger["reported_tokens"] or any( + result.usage.get(field) != ledger["usage"][field] + for field in _USAGE_FIELDS + ): + proxy.ledger.fail( + "Codex final usage disagreed with the capped proxy ledger." + ) + elif result.completed: + proxy.ledger.fail( + "Codex declared completion without a complete capped proxy ledger." + ) + + ledger = proxy.ledger.snapshot() + ledger["runner"] = "capped-codex" + ledger["model_catalog_sha256"] = hashlib.sha256( + json.dumps(bundled_model, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + ).hexdigest() + if not ledger["complete"]: + result.completed = False + result.failures.extend( + failure + for failure in ledger["failures"] + if failure not in result.failures + ) + result.hard_cap_ledger = ledger + return result + + def _load_bundled_model(self, model: str) -> dict[str, Any]: + completed = subprocess.run( + [self.executable, "debug", "models", "--bundled"], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + "capped-codex could not load the Codex bundled model catalog." + ) + try: + catalog = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError( + "capped-codex received an invalid Codex bundled model catalog." + ) from exc + models = catalog.get("models") if isinstance(catalog, dict) else None + if not isinstance(models, list): + raise RuntimeError( + "capped-codex received an invalid Codex bundled model catalog." + ) + selected = next( + ( + candidate + for candidate in models + if isinstance(candidate, dict) and candidate.get("slug") == model + ), + None, + ) + if selected is None: + raise RuntimeError( + f"capped-codex cannot use model {model!r} because it is absent from " + "the selected Codex executable's bundled catalog." + ) + return selected + + +def _capped_provider_config(base_url: str, catalog_path: Path) -> tuple[str, ...]: + prefix = "model_providers.codemesh_capped" + return ( + 'model_provider="codemesh_capped"', + f"model_catalog_json={json.dumps(str(catalog_path))}", + f'{prefix}.name="CodeMesh capped OpenAI"', + f"{prefix}.base_url={json.dumps(base_url)}", + f'{prefix}.env_key="CODEMESH_CAPPED_PROXY_TOKEN"', + f'{prefix}.wire_api="responses"', + f"{prefix}.requires_openai_auth=false", + f"{prefix}.request_max_retries=0", + f"{prefix}.stream_max_retries=0", + f"{prefix}.supports_websockets=false", + ) + + +def _validate_response_request(body: Mapping[str, Any]) -> str | None: + if set(body) - _ALLOWED_RESPONSE_FIELDS: + return ( + "A Responses request parameter outside the capped runner's reviewed " + "counting contract was rejected." + ) + background = body.get("background") + if background is not None and background is not False: + return "Background Responses calls are unsupported by the capped runner." + if body.get("context_management"): + return "Automatic context compaction is unsupported by the capped runner." + tools = body.get("tools") or [] + if not isinstance(tools, list): + return "Responses tools must be an array." + for tool in tools: + if ( + not isinstance(tool, Mapping) + or tool.get("type") not in _ALLOWED_LOCAL_TOOL_TYPES + ): + return "Unsupported cost-bearing Responses tool was rejected." + return None + + +def _authorized(request: Request, expected_token: str) -> bool: + authorization = request.headers.get("authorization", "") + prefix = "Bearer " + return authorization.startswith(prefix) and hmac.compare_digest( + authorization[len(prefix) :], expected_token + ) + + +async def _wait_for_server(server: uvicorn.Server, task: asyncio.Task[None]) -> None: + while not server.started: + if task.done(): + await task + raise RuntimeError("The capped Responses proxy stopped during startup.") + await asyncio.sleep(0.01) + + +def _non_negative_integer(value: Any, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise CappedProxyError(f"{field_name} must be a non-negative integer.") + return value + + +def _optional_non_negative_integer(value: Any, field_name: str) -> int: + if value is None: + return 0 + return _non_negative_integer(value, field_name) diff --git a/agent-access/codemesh_agent_access/evaluation/codex.py b/agent-access/codemesh_agent_access/evaluation/codex.py index 3b371aa..e1a8195 100644 --- a/agent-access/codemesh_agent_access/evaluation/codex.py +++ b/agent-access/codemesh_agent_access/evaluation/codex.py @@ -10,7 +10,14 @@ from dataclasses import dataclass, field from pathlib import Path from time import perf_counter -from typing import Any, Literal, Protocol +from typing import Any, Literal, Mapping, Protocol + + +REPORTED_TOKEN_ACCOUNTING_ID = "input-output-reasoning-v1" +REPORTED_TOKEN_ACCOUNTING_DESCRIPTION = ( + "input_tokens + output_tokens + reasoning_output_tokens; " + "cached_input_tokens is a subset of input_tokens and is not added again" +) _SECRET_PATTERNS = [ @@ -35,6 +42,7 @@ class AgentRunRequest: output_schema: Path | None = None keep_raw_traces: bool = False mcp_config: tuple[str, ...] | None = None + max_reported_tokens: int | None = None @dataclass @@ -52,9 +60,20 @@ class AgentRunResult: file_change_count: int = 0 failures: list[str] = field(default_factory=list) raw_events: list[dict[str, Any]] = field(default_factory=list) + hard_cap_ledger: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class AgentRunnerCapabilities: + hard_reported_token_cap: bool + reported_token_accounting: str | None = None + runner_id: str | None = None class AgentRunner(Protocol): + @property + def capabilities(self) -> AgentRunnerCapabilities: ... + async def run(self, request: AgentRunRequest) -> AgentRunResult: ... @@ -64,9 +83,24 @@ def __init__( executable: str = "codex", *, supports_ignore_user_config: bool | None = None, + config_overrides: tuple[str, ...] = (), + environment_overrides: Mapping[str, str] | None = None, ) -> None: self.executable = executable self._supports_ignore_user_config = supports_ignore_user_config + self._config_overrides = config_overrides + self._environment_overrides = dict(environment_overrides or {}) + + @property + def capabilities(self) -> AgentRunnerCapabilities: + # Codex JSONL reports usage only after a completed turn. The executable + # currently exposes no per-invocation hard limit using this accounting, + # so post-run observation must not be presented as enforcement. + return AgentRunnerCapabilities( + hard_reported_token_cap=False, + reported_token_accounting=REPORTED_TOKEN_ACCOUNTING_ID, + runner_id="codex", + ) async def run(self, request: AgentRunRequest) -> AgentRunResult: supports_ignore_user_config = self._detect_ignore_user_config() @@ -75,6 +109,9 @@ async def run(self, request: AgentRunRequest) -> AgentRunResult: ) isolated_home: tempfile.TemporaryDirectory[str] | None = None environment: dict[str, str] | None = None + if self._environment_overrides: + environment = os.environ.copy() + environment.update(self._environment_overrides) if not supports_ignore_user_config: if request.local_provider is None: raise RuntimeError( @@ -82,7 +119,7 @@ async def run(self, request: AgentRunRequest) -> AgentRunResult: "legacy isolation is supported only with a local provider." ) isolated_home = tempfile.TemporaryDirectory(prefix="codemesh-codex-home-") - environment = os.environ.copy() + environment = environment or os.environ.copy() environment["CODEX_HOME"] = isolated_home.name started = perf_counter() try: @@ -157,6 +194,8 @@ def _arguments( arguments.insert(3, "--ignore-user-config") for override in mcp_config: arguments.extend(["-c", override]) + for override in self._config_overrides: + arguments.extend(["-c", override]) if request.local_provider is not None: arguments.extend(["--oss", "--local-provider", request.local_provider]) if request.output_schema is not None: @@ -262,6 +301,31 @@ def sanitize_text(value: str) -> str: return sanitized +def count_reported_tokens(usage: Mapping[str, Any]) -> int: + """Count runner-reported tokens without double-counting cached input.""" + required_fields = ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_output_tokens", + ) + missing = [key for key in required_fields if key not in usage] + if missing: + raise ValueError( + "Reported usage is missing required token fields: " + ", ".join(missing) + ) + counts = {key: _reported_token_integer(usage[key], key) for key in required_fields} + if counts["cached_input_tokens"] > counts["input_tokens"]: + raise ValueError( + "cached_input_tokens cannot exceed input_tokens in reported usage." + ) + return ( + counts["input_tokens"] + + counts["output_tokens"] + + counts["reasoning_output_tokens"] + ) + + def _codemesh_mcp_config(cwd: Path) -> list[str]: enabled_tools = [ "codemesh_status", @@ -307,3 +371,11 @@ def _integer(value: Any) -> int: return int(value) except (TypeError, ValueError): return 0 + + +def _reported_token_integer(value: Any, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError( + f"{field_name} must be a non-negative integer in reported usage." + ) + return value diff --git a/agent-access/codemesh_agent_access/evaluation/live.py b/agent-access/codemesh_agent_access/evaluation/live.py index b2af40d..99b0763 100644 --- a/agent-access/codemesh_agent_access/evaluation/live.py +++ b/agent-access/codemesh_agent_access/evaluation/live.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import subprocess import sys import tempfile @@ -36,8 +37,13 @@ def __init__( args: list[str] | None = None, cwd: str | Path | None = None, timeout_seconds: int = 60, + capture_context_package_timings: bool = False, ) -> None: package_root = Path(__file__).resolve().parents[2] + environment = None + if capture_context_package_timings: + environment = os.environ.copy() + environment["CODEMESH_CONTEXT_PACKAGE_TIMINGS"] = "1" self.parameters = StdioServerParameters( command=command or sys.executable, args=args @@ -49,11 +55,15 @@ def __init__( "diagnostic", ], cwd=Path(cwd) if cwd else package_root, + env=environment, ) self.timeout = timedelta(seconds=timeout_seconds) self.session: ClientSession | None = None self.startup_ms = 0.0 self.tool_names: list[str] = [] + self.capture_context_package_timings = capture_context_package_timings + self.context_package_timing_labels: list[str | None] = [] + self.context_package_timings: list[dict[str, Any]] = [] @asynccontextmanager async def connect(self) -> AsyncIterator["McpEvaluationClient"]: @@ -86,16 +96,30 @@ async def connect(self) -> AsyncIterator["McpEvaluationClient"]: f"Could not start or communicate with CodeMesh MCP: {type(exc).__name__}: {exc}" ) from exc finally: + if self.capture_context_package_timings: + errlog.seek(0) + self.context_package_timings = _parse_context_package_timings( + errlog.read() + ) self.session = None async def call( - self, tool: str, arguments: dict[str, Any] + self, + tool: str, + arguments: dict[str, Any], + *, + timing_label: str | None = None, ) -> tuple[dict[str, Any], float]: if self.session is None: raise EvaluationInfrastructureError( "The MCP evaluation client is not connected." ) started = perf_counter() + if ( + self.capture_context_package_timings + and tool == "codemesh_get_context_package" + ): + self.context_package_timing_labels.append(timing_label) try: result = await self.session.call_tool( tool, @@ -125,12 +149,18 @@ async def run_live_evaluation( command: str | None = None, command_args: list[str] | None = None, command_cwd: str | Path | None = None, + capture_context_package_timings: bool = False, ) -> LiveReport: if repetitions < 1 or warmup < 0: raise ValueError("Live evaluation requires repetitions >= 1 and warmup >= 0.") started = perf_counter() resolved_repository = repository_id or suite.repository_id - client = McpEvaluationClient(command, command_args, command_cwd) + client = McpEvaluationClient( + command, + command_args, + command_cwd, + capture_context_package_timings=capture_context_package_timings, + ) tool_source = get_tool_source_identity(Path(client.parameters.cwd)) tool_source_comparable = bool(tool_source.get("available")) and not bool( tool_source.get("working_tree_dirty") @@ -229,6 +259,17 @@ async def run_live_evaluation( ) ) + timing_summary: dict[str, Any] | None = None + if capture_context_package_timings: + timing_summary = _context_package_timing_summary( + client.context_package_timings, + client.context_package_timing_labels, + ) + for case_report in case_reports: + case_timing = timing_summary["cases"].get(case_report.id) + if case_timing is not None: + case_report.diagnostics["context_package_timing"] = case_timing + retrieval_cases = [ case for case, source in zip(case_reports, suite.cases, strict=True) @@ -314,6 +355,11 @@ async def run_live_evaluation( [latency for case in case_reports for latency in case.latencies_ms], 0.95, ), + **( + {"context_package_timing": timing_summary} + if timing_summary is not None + else {} + ), }, cases=case_reports, failures=failures, @@ -332,10 +378,24 @@ async def get_repository_preflight( data = _data(payload) freshness = data.get("freshness") or {} if freshness.get("status") != "fresh": - raise EvaluationInfrastructureError( + identity = "; ".join( + f"{key}={freshness.get(key)!r}" + for key in ( + "status", + "indexed_commit", + "current_commit", + "working_tree_dirty", + "indexed_source_view_hash", + ) + if key in freshness + ) + detail = ( freshness.get("detail") or "Agent evaluation requires a fresh CodeMesh index." ) + raise EvaluationInfrastructureError( + f"{detail} Candidate identity: {identity}." if identity else detail + ) return data @@ -383,7 +443,11 @@ async def _run_live_case( failures = [] for _ in range(repetitions): try: - payload, latency = await client.call(case.tool, arguments) + payload, latency = await client.call( + case.tool, + arguments, + timing_label=case.id, + ) except EvaluationInfrastructureError as exc: failures.append(str(exc)) continue @@ -448,6 +512,91 @@ async def _run_live_case( ) +def _parse_context_package_timings(stderr: str) -> list[dict[str, Any]]: + prefix = "CODEMESH_CONTEXT_PACKAGE_TIMING " + records = [] + for line in stderr.splitlines(): + if not line.startswith(prefix): + continue + try: + payload = json.loads(line[len(prefix) :]) + except json.JSONDecodeError as exc: + raise EvaluationInfrastructureError( + "CodeMesh emitted a malformed context-package timing record." + ) from exc + if ( + not isinstance(payload, dict) + or payload.get("event") != "context_package_timing" + ): + raise EvaluationInfrastructureError( + "CodeMesh emitted an invalid context-package timing record." + ) + records.append(payload) + return records + + +def _context_package_timing_summary( + records: list[dict[str, Any]], labels: list[str | None] +) -> dict[str, Any]: + if len(records) != len(labels): + raise EvaluationInfrastructureError( + "Context-package timing record count did not match measured MCP calls." + ) + measured = [ + (label, record) + for label, record in zip(labels, records, strict=True) + if label is not None + ] + grouped: dict[str, list[dict[str, Any]]] = {} + for label, record in measured: + grouped.setdefault(label, []).append(record) + return { + "captured": True, + "record_count": len(measured), + "cases": { + label: _summarize_timing_records(case_records) + for label, case_records in sorted(grouped.items()) + }, + } + + +def _summarize_timing_records(records: list[dict[str, Any]]) -> dict[str, Any]: + stage_names = sorted( + {str(name) for record in records for name in (record.get("stages_ms") or {})} + ) + operation_names = sorted( + { + str(name) + for record in records + for name in (record.get("operations_ms") or {}) + } + ) + return { + "calls": len(records), + "successful_calls": sum(record.get("outcome") == "ok" for record in records), + "p50_total_ms": p50([float(record.get("total_ms") or 0) for record in records]), + "p50_stages_ms": { + name: p50( + [ + float((record.get("stages_ms") or {}).get(name, 0)) + for record in records + ] + ) + for name in stage_names + }, + "p50_operations_ms": { + name: p50( + [ + float((record.get("operations_ms") or {}).get(name, 0)) + for record in records + ] + ) + for name in operation_names + }, + "counts": records[0].get("counts", {}) if records else {}, + } + + def _tool_payload(result: Any) -> dict[str, Any]: structured = getattr(result, "structuredContent", None) if isinstance(structured, dict): diff --git a/agent-access/codemesh_agent_access/evaluation/models.py b/agent-access/codemesh_agent_access/evaluation/models.py index db83b9c..d6865cb 100644 --- a/agent-access/codemesh_agent_access/evaluation/models.py +++ b/agent-access/codemesh_agent_access/evaluation/models.py @@ -1,16 +1,29 @@ from __future__ import annotations import json +import re from pathlib import Path -from typing import Any, Literal +from typing import Annotated, Any, Literal -from pydantic import BaseModel, Field, model_validator +from pydantic import AfterValidator, BaseModel, Field, model_validator + +from .paths import ( + ARTIFACT_ID_PATTERN, + resolve_setup_patch, + validate_artifact_id, + validate_setup_patch_name, +) SCHEMA_VERSION = "1.0" MODEL_BENCHMARK_SCHEMA_VERSION = "1.0" Difficulty = Literal["easy", "medium", "hard"] +AgentIntegrationMode = Literal["spontaneous", "assisted", "configured"] +ArtifactId = Annotated[ + str, Field(pattern=ARTIFACT_ID_PATTERN), AfterValidator(validate_artifact_id) +] +SetupPatchPath = Annotated[str, AfterValidator(validate_setup_patch_name)] class RetrievalTarget(BaseModel): @@ -95,12 +108,12 @@ class CommandPolicy(BaseModel): class AgentTask(BaseModel): - id: str + id: ArtifactId description: str task_type: Literal["answer", "change"] prompt: str targets: list[RetrievalTarget] = Field(default_factory=list) - setup_patch: str | None = None + setup_patch: SetupPatchPath | None = None validation: list[ValidationCommand] = Field(default_factory=list) required_changed_paths: list[str] = Field(default_factory=list) allowed_changed_paths: list[str] = Field(default_factory=list) @@ -127,15 +140,63 @@ class AgentThresholds(BaseModel): class AgentSuite(BaseModel): schema_version: Literal["1.0"] = SCHEMA_VERSION kind: Literal["agent"] = "agent" - name: str + name: ArtifactId description: str = "" repository_id: str = "code_mesh" repository_url: str | None = None repository_commit: str | None = None + integration_mode: AgentIntegrationMode | None = None + baseline_mcp_servers: list[str] = Field(default_factory=list) treatment_guidance: str | None = None tasks: list[AgentTask] thresholds: AgentThresholds = Field(default_factory=AgentThresholds) + @model_validator(mode="after") + def validate_integration_mode(self) -> "AgentSuite": + _require_unique_artifact_ids(self.tasks) + if self.integration_mode == "configured" and self.treatment_guidance: + raise ValueError( + "Configured agent suites must use shipped repository guidance, " + "not evaluator-only treatment guidance." + ) + if self.integration_mode == "configured" and not self.repository_commit: + raise ValueError( + "Configured agent suites require a pinned repository commit." + ) + if self.baseline_mcp_servers and self.integration_mode != "configured": + raise ValueError( + "Baseline MCP servers can be selected only by configured suites." + ) + normalized_servers = [value.strip() for value in self.baseline_mcp_servers] + if any(not value for value in normalized_servers) or len( + set(normalized_servers) + ) != len(normalized_servers): + raise ValueError("Baseline MCP server names must be non-empty and unique.") + if any( + re.fullmatch(r"[A-Za-z0-9_-]+", value) is None + for value in normalized_servers + ): + raise ValueError( + "Baseline MCP server names may contain only letters, digits, " + "underscores, and hyphens." + ) + self.baseline_mcp_servers = normalized_servers + if self.integration_mode == "assisted" and not self.treatment_guidance: + raise ValueError( + "Assisted agent suites require evaluator treatment guidance." + ) + if self.integration_mode == "spontaneous" and self.treatment_guidance: + raise ValueError( + "Spontaneous agent suites cannot include treatment guidance." + ) + return self + + @property + def resolved_integration_mode(self) -> AgentIntegrationMode: + if self.integration_mode is not None: + return self.integration_mode + return "assisted" if self.treatment_guidance else "spontaneous" + class CriterionWeights(BaseModel): accuracy: int = Field(default=35, ge=0, le=100) @@ -182,7 +243,7 @@ class HumanRubric(BaseModel): class BenchmarkQuestion(BaseModel): - id: str + id: ArtifactId family_id: str difficulty: Difficulty description: str @@ -196,12 +257,12 @@ class BenchmarkQuestion(BaseModel): class BenchmarkChangeScenario(BaseModel): - id: str + id: ArtifactId family_id: str difficulty: Difficulty description: str prompt: str - setup_patch: str + setup_patch: SetupPatchPath validation: list[ValidationCommand] = Field(min_length=1) required_changed_paths: list[str] = Field(min_length=1) allowed_changed_paths: list[str] = Field(min_length=1) @@ -228,7 +289,7 @@ class ModelBenchmarkThresholds(BaseModel): class ModelBenchmarkSuite(BaseModel): schema_version: Literal["1.0"] = MODEL_BENCHMARK_SCHEMA_VERSION kind: Literal["model-benchmark"] = "model-benchmark" - name: str + name: ArtifactId description: str = "" repository_id: str = "code_mesh" weights: CriterionWeights = Field(default_factory=CriterionWeights) @@ -240,6 +301,7 @@ class ModelBenchmarkSuite(BaseModel): @model_validator(mode="after") def validate_matrix(self) -> "ModelBenchmarkSuite": + _require_unique_artifact_ids([*self.questions, *self.change_scenarios]) _require_difficulty_matrix(self.questions, family_count=8, label="question") _require_difficulty_matrix( self.change_scenarios, family_count=3, label="change scenario" @@ -309,6 +371,8 @@ class AgentExecutionReport(BaseModel): cached_input_tokens: int = 0 output_tokens: int = 0 reasoning_output_tokens: int = 0 + reported_tokens: int = 0 + reported_token_limit: int | None = None mcp_calls: list[str] = Field(default_factory=list) mcp_call_attempts: list[str] = Field(default_factory=list) mcp_failed_calls: list[str] = Field(default_factory=list) @@ -321,6 +385,7 @@ class AgentExecutionReport(BaseModel): sanitized_diff: str | None = None raw_events: list[dict[str, Any]] | None = None final_message: str | None = None + hard_cap_ledger: dict[str, Any] | None = None class AgentReport(BaseModel): @@ -423,6 +488,7 @@ class ModelComparisonReport(BaseModel): def built_in_suite_path(name: str) -> Path: + validate_artifact_id(name) path = Path(__file__).with_name("suites") / f"{name}.json" if not path.is_file(): raise ValueError(f"Unknown built-in evaluation suite '{name}'.") @@ -445,10 +511,35 @@ def load_suite( if kind == "live": return LiveSuite.model_validate(payload), path if kind == "agent": - return AgentSuite.model_validate(payload), path - if kind == "model-benchmark": - return ModelBenchmarkSuite.model_validate(payload), path - raise ValueError(f"Evaluation suite '{path}' has unsupported kind '{kind}'.") + suite = AgentSuite.model_validate(payload) + elif kind == "model-benchmark": + suite = ModelBenchmarkSuite.model_validate(payload) + else: + raise ValueError(f"Evaluation suite '{path}' has unsupported kind '{kind}'.") + validate_suite_artifacts(suite, path) + return suite, path + + +def validate_suite_artifacts( + suite: AgentSuite | ModelBenchmarkSuite, suite_path: Path +) -> None: + validate_artifact_id(suite.name) + items = ( + suite.tasks + if isinstance(suite, AgentSuite) + else [*suite.questions, *suite.change_scenarios] + ) + _require_unique_artifact_ids(items) + for item in items: + validate_artifact_id(item.id) + if patch := getattr(item, "setup_patch", None): + resolve_setup_patch(suite_path, patch) + + +def _require_unique_artifact_ids(items: list[Any]) -> None: + identifiers = [item.id.casefold() for item in items] + if len(set(identifiers)) != len(identifiers): + raise ValueError("Evaluation identifiers must be unique ignoring case.") def suite_json_schema(kind: Literal["live", "agent", "model"]) -> dict[str, Any]: diff --git a/agent-access/codemesh_agent_access/evaluation/paths.py b/agent-access/codemesh_agent_access/evaluation/paths.py new file mode 100644 index 0000000..00d7d93 --- /dev/null +++ b/agent-access/codemesh_agent_access/evaluation/paths.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import re +from pathlib import Path, PurePosixPath, PureWindowsPath + + +ARTIFACT_ID_PATTERN = r"^[A-Za-z0-9_][A-Za-z0-9_.-]*$" +_RESERVED_NAMES = {"CON", "PRN", "AUX", "NUL"} | { + f"{prefix}{number}" for prefix in ("COM", "LPT") for number in range(1, 10) +} + + +def validate_artifact_id(value: str) -> str: + if ( + re.fullmatch(ARTIFACT_ID_PATTERN, value) is None + or value.endswith(".") + or value.split(".", 1)[0].upper() in _RESERVED_NAMES + ): + raise ValueError( + "Evaluation identifiers must be safe path segments: ASCII letters, " + "digits, underscores, hyphens, and internal dots; no path syntax, " + "trailing dot, or reserved device name." + ) + return value + + +def validate_setup_patch_name(value: str) -> str: + path = PurePosixPath(value) + if ( + not value + or "\\" in value + or ":" in value + or "\x00" in value + or path.is_absolute() + or PureWindowsPath(value).drive + or ".." in path.parts + ): + raise ValueError("Setup patches must use contained suite-relative paths.") + return value + + +def resolve_setup_patch(suite_path: Path, relative: str) -> Path: + validate_setup_patch_name(relative) + try: + root = suite_path.parent.resolve(strict=True) + path = (root / relative).resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ValueError( + "Setup patch must be an accessible suite-relative file." + ) from exc + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError("Setup patch must be a file within its suite directory.") + return path + + +def managed_artifact_path(root: Path, *parts: str) -> Path: + """Keep generated destinations inside the operator-selected artifact root.""" + resolved_root = root.resolve() + path = resolved_root.joinpath(*parts).resolve() + if path == resolved_root or not path.is_relative_to(resolved_root): + raise ValueError("Generated artifact path escapes its managed directory.") + return path diff --git a/agent-access/codemesh_agent_access/evaluation/suites/config-net-cache-configured-agent.json b/agent-access/codemesh_agent_access/evaluation/suites/config-net-cache-configured-agent.json new file mode 100644 index 0000000..7372573 --- /dev/null +++ b/agent-access/codemesh_agent_access/evaluation/suites/config-net-cache-configured-agent.json @@ -0,0 +1,45 @@ +{ + "schema_version": "1.0", + "kind": "agent", + "name": "config-net-cache-configured-agent", + "description": "Pinned configured/onboarded Config.Net C# cache-coherence impact task.", + "repository_id": "config_net", + "repository_url": "https://github.com/aloneguid/config.git", + "repository_commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "integration_mode": "configured", + "thresholds": { + "efficiency_delta": 0.10, + "minimum_completed_pairs": 3 + }, + "tasks": [ + { + "id": "cached-write-coherence-impact", + "description": "Distinguish required and generic impact when cached writes must become immediately visible.", + "task_type": "answer", + "prompt": "Plan the impact of making writes immediately visible through a cached Config.Net configuration proxy without waiting for the CacheFor TTL. Trace property and configurable-method setters through interception, path construction, value conversion, store fan-out, and the keyed cache. Identify the source and test changes that are actually required, then distinguish generic paths that should already adapt without edits. Preserve store precedence and no-cache behavior. Cite implementation and test files with exact line ranges. Do not implement the change.", + "targets": [ + { + "file_path": "src/Config.Net/Core/IoHandler.cs" + }, + { + "file_path": "src/Config.Net/Core/LazyVar.cs" + }, + { + "file_path": "src/Config.Net/Core/DynamicWriter.cs" + }, + { + "file_path": "src/Config.Net/Core/InterfaceInterceptor.cs" + }, + { + "file_path": "src/Config.Net/ConfigurationBuilder.cs" + }, + { + "file_path": "src/Config.Net.Tests/LogicTest.cs" + }, + { + "file_path": "src/Config.Net.Tests/ConfigurableMethodsTest.cs" + } + ] + } + ] +} diff --git a/agent-access/codemesh_agent_access/evaluation/suites/config-net-cache-focused-live.json b/agent-access/codemesh_agent_access/evaluation/suites/config-net-cache-focused-live.json new file mode 100644 index 0000000..4a2d672 --- /dev/null +++ b/agent-access/codemesh_agent_access/evaluation/suites/config-net-cache-focused-live.json @@ -0,0 +1,120 @@ +{ + "schema_version": "1.0", + "kind": "live", + "name": "config-net-cache-focused-live", + "description": "Provider-free focused-query coverage for the multi-part Config.Net cached-write impact task.", + "repository_id": "config_net", + "repository_url": "https://github.com/aloneguid/config.git", + "repository_commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "cases": [ + { + "id": "cached-write-core", + "description": "Find cache configuration, shared I/O, and existing cached-property tests.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "cached writes CacheFor TTL", + "repository_id": "config_net", + "limit": 8, + "max_characters": 12000 + }, + "targets": [ + { + "file_path": "src/Config.Net/Core/IoHandler.cs", + "primary": true + }, + { + "file_path": "src/Config.Net/ConfigurationBuilder.cs" + }, + { + "file_path": "src/Config.Net.Tests/LogicTest.cs" + } + ], + "min_hit_count": 3, + "top_k": 8 + }, + { + "id": "setter-routing", + "description": "Find property and configurable-method setter paths and tests.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "property configurable method setters", + "repository_id": "config_net", + "limit": 8, + "max_characters": 12000 + }, + "targets": [ + { + "file_path": "src/Config.Net/Core/DynamicWriter.cs", + "primary": true + }, + { + "file_path": "src/Config.Net.Tests/ConfigurableMethodsTest.cs" + } + ], + "min_hit_count": 2, + "top_k": 8 + }, + { + "id": "proxy-interception", + "description": "Find proxy interception and its writer handoff.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "proxy intercept writes", + "repository_id": "config_net", + "limit": 8, + "max_characters": 12000 + }, + "targets": [ + { + "file_path": "src/Config.Net/Core/InterfaceInterceptor.cs", + "primary": true + }, + { + "file_path": "src/Config.Net/Core/DynamicWriter.cs" + } + ], + "min_hit_count": 2, + "top_k": 8 + }, + { + "id": "cache-expiration-value", + "description": "Find the expiring keyed cache value and cache-duration configuration.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "TTL expiration TimeSpan cached value", + "repository_id": "config_net", + "limit": 8, + "max_characters": 12000 + }, + "targets": [ + { + "file_path": "src/Config.Net/Core/LazyVar.cs", + "primary": true + }, + { + "file_path": "src/Config.Net/ConfigurationBuilder.cs" + } + ], + "min_hit_count": 2, + "top_k": 8 + }, + { + "id": "freshness", + "description": "Require the pinned Config.Net checkout to remain freshly indexed.", + "tool": "codemesh_get_repository_status", + "arguments": { + "repository_id": "config_net" + }, + "expected_values": { + "freshness.status": "fresh" + } + } + ], + "thresholds": { + "min_tool_success_rate": 1.0, + "min_recall_at_k": 1.0, + "min_mrr": 0.25, + "max_primary_rank": 8, + "max_secret_leaks": 0 + } +} diff --git a/agent-access/codemesh_agent_access/evaluation/suites/onenine-native-replay-configured-agent.json b/agent-access/codemesh_agent_access/evaluation/suites/onenine-native-replay-configured-agent.json new file mode 100644 index 0000000..139c1c3 --- /dev/null +++ b/agent-access/codemesh_agent_access/evaluation/suites/onenine-native-replay-configured-agent.json @@ -0,0 +1,44 @@ +{ + "schema_version": "1.0", + "kind": "agent", + "name": "onenine-native-replay-configured-agent", + "description": "Pinned configured/onboarded one|nine Python-to-Rust native replay change-impact task.", + "repository_id": "onenine", + "repository_commit": "fe2f761583bf78601961ff17934185c4b6a632f9", + "integration_mode": "configured", + "thresholds": { + "efficiency_delta": 0.10, + "minimum_completed_pairs": 3 + }, + "tasks": [ + { + "id": "native-replay-four-worker-impact", + "description": "Distinguish the native replay contract changes required to allow four parallel workers from unrelated worker-count settings.", + "task_type": "answer", + "prompt": "Plan the likely impact of allowing four workers in native feed-hub replay parallel processing instead of the current maximum of three. Trace the Python adapter and session validation, the PyO3 boundary and Rust worker/queue pipeline, telemetry and evidence validation, benchmark measurement loops, and focused tests. Identify the source and test changes actually required, distinguish unrelated strategy-job worker_count=4 paths that already use a separate contract, and recommend the risk-appropriate validation commands. Cite implementation and test files with exact line ranges. Do not implement the change.", + "targets": [ + { + "file_path": "app/feed_hub/native_replay.py" + }, + { + "file_path": "native/feed_hub_replay/src/lib.rs" + }, + { + "file_path": "app/feed_hub/evidence.py" + }, + { + "file_path": "tests/test_feed_hub_native_replay_adapter.py" + }, + { + "file_path": "tests/test_feed_hub_native_replay_state.py" + }, + { + "file_path": "tests/test_feed_hub_replay_runners.py" + }, + { + "file_path": "tests/test_feed_hub_evidence.py" + } + ] + } + ] +} diff --git a/agent-access/codemesh_agent_access/evaluation/suites/youtube-downloader-impact-adaptive-live.json b/agent-access/codemesh_agent_access/evaluation/suites/youtube-downloader-impact-adaptive-live.json new file mode 100644 index 0000000..759341b --- /dev/null +++ b/agent-access/codemesh_agent_access/evaluation/suites/youtube-downloader-impact-adaptive-live.json @@ -0,0 +1,105 @@ +{ + "schema_version": "1.0", + "kind": "live", + "name": "youtube-downloader-impact-adaptive-live", + "description": "Provider-free task-adaptive query comparison for the frozen YoutubeDownloader output-container impact task.", + "repository_id": "youtube_downloader", + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "cases": [ + { + "id": "task-shaped-package", + "description": "Try one concise task-shaped package against every frozen impact target.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "Plan the likely impact of adding a new audio-only output container. Find hard-coded option-generation and batch-selector changes, then trace generic single-download selection, multi-download preferences, persisted settings, and dashboard execution.", + "repository_id": "youtube_downloader", + "limit": 12, + "max_characters": 12000, + "context_depth": 4, + "relationship_limit": 12 + }, + "targets": [ + { "file_path": "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs", "primary": true }, + { "file_path": "YoutubeDownloader.Core/Downloading/VideoDownloadPreference.cs" }, + { "file_path": "YoutubeDownloader/ViewModels/Dialogs/DownloadMultipleSetupViewModel.cs" }, + { "file_path": "YoutubeDownloader/ViewModels/Dialogs/DownloadSingleSetupViewModel.cs" }, + { "file_path": "YoutubeDownloader/Services/SettingsService.cs" }, + { "file_path": "YoutubeDownloader/ViewModels/Components/DashboardViewModel.cs" } + ], + "min_hit_count": 6, + "top_k": 12 + }, + { + "id": "focused-option-generation", + "description": "Find hard-coded audio-container option generation and relevant validation.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "audio-only output container option generation tests", + "repository_id": "youtube_downloader", + "limit": 8, + "max_characters": 12000, + "context_depth": 4, + "relationship_limit": 8 + }, + "targets": [ + { "file_path": "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs", "primary": true } + ], + "min_hit_count": 1, + "top_k": 8 + }, + { + "id": "focused-selection-persistence", + "description": "Trace selected-container flow through single, multiple, preference, and settings paths.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "selected output container single-download multi-download preferences persisted settings tests", + "repository_id": "youtube_downloader", + "limit": 8, + "max_characters": 12000, + "context_depth": 4, + "relationship_limit": 8 + }, + "targets": [ + { "file_path": "YoutubeDownloader.Core/Downloading/VideoDownloadPreference.cs" }, + { "file_path": "YoutubeDownloader/ViewModels/Dialogs/DownloadMultipleSetupViewModel.cs", "primary": true }, + { "file_path": "YoutubeDownloader/ViewModels/Dialogs/DownloadSingleSetupViewModel.cs" }, + { "file_path": "YoutubeDownloader/Services/SettingsService.cs" } + ], + "min_hit_count": 4, + "top_k": 8 + }, + { + "id": "focused-dashboard-execution", + "description": "Find the selected-container dashboard execution path.", + "tool": "codemesh_get_context_package", + "arguments": { + "query": "selected output container dashboard download execution tests", + "repository_id": "youtube_downloader", + "limit": 8, + "max_characters": 12000, + "context_depth": 4, + "relationship_limit": 8 + }, + "targets": [ + { "file_path": "YoutubeDownloader/ViewModels/Components/DashboardViewModel.cs", "primary": true } + ], + "min_hit_count": 1, + "top_k": 8 + }, + { + "id": "freshness", + "description": "Require the frozen YoutubeDownloader checkout to remain freshly indexed.", + "tool": "codemesh_get_repository_status", + "arguments": { "repository_id": "youtube_downloader" }, + "expected_values": { "freshness.status": "fresh" } + } + ], + "thresholds": { + "min_tool_success_rate": 1.0, + "min_recall_at_k": 0.85, + "min_mrr": 0.75, + "max_primary_rank": 3, + "max_secret_leaks": 0 + } +} diff --git a/agent-access/codemesh_agent_access/evaluation/suites/youtube-downloader-impact-configured-agent.json b/agent-access/codemesh_agent_access/evaluation/suites/youtube-downloader-impact-configured-agent.json new file mode 100644 index 0000000..a460d4b --- /dev/null +++ b/agent-access/codemesh_agent_access/evaluation/suites/youtube-downloader-impact-configured-agent.json @@ -0,0 +1,42 @@ +{ + "schema_version": "1.0", + "kind": "agent", + "name": "youtube-downloader-impact-configured-agent", + "description": "Pinned configured/onboarded C# impact task for independent product-evidence replication.", + "repository_id": "youtube_downloader", + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "integration_mode": "configured", + "thresholds": { + "efficiency_delta": 0.10, + "minimum_completed_pairs": 3 + }, + "tasks": [ + { + "id": "audio-container-impact", + "description": "Distinguish required and generic impact when adding an audio-only output container.", + "task_type": "answer", + "prompt": "Plan the likely impact of adding a new audio-only output container to YoutubeDownloader. Identify the hard-coded option-generation and batch-selector changes that are actually required, then trace how the selected container flows through single-download selection, multi-download preferences, persisted settings, and dashboard execution. Explicitly distinguish generic paths that should already adapt from places requiring edits. Cite implementation files and exact line ranges. Do not implement the change.", + "targets": [ + { + "file_path": "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs" + }, + { + "file_path": "YoutubeDownloader.Core/Downloading/VideoDownloadPreference.cs" + }, + { + "file_path": "YoutubeDownloader/ViewModels/Dialogs/DownloadMultipleSetupViewModel.cs" + }, + { + "file_path": "YoutubeDownloader/ViewModels/Dialogs/DownloadSingleSetupViewModel.cs" + }, + { + "file_path": "YoutubeDownloader/Services/SettingsService.cs" + }, + { + "file_path": "YoutubeDownloader/ViewModels/Components/DashboardViewModel.cs" + } + ] + } + ] +} diff --git a/agent-access/codemesh_agent_access/feedback.py b/agent-access/codemesh_agent_access/feedback.py new file mode 100644 index 0000000..91f84af --- /dev/null +++ b/agent-access/codemesh_agent_access/feedback.py @@ -0,0 +1,1184 @@ +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from collections import defaultdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal, TYPE_CHECKING + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +PACKET_SCHEMA_VERSION = "codemesh-feedback/v1" +PACKET_SCHEMA_VERSION_V2 = "codemesh-feedback/v2" +SUMMARY_SCHEMA_VERSION = "codemesh-feedback-summary/v1" +RESOLUTION_PLAN_SCHEMA_VERSION = "codemesh-feedback-resolution-plan/v1" +OUTBOX_NAME = ".codemesh-feedback" + +if TYPE_CHECKING: + from .feedback_session import SessionRuntime + +Classification = Literal["diagnostic", "controlled-evaluation"] +Confidence = Literal["low", "medium", "high"] +ValidationOutcome = Literal["passed", "failed", "not-run", "inconclusive"] +IssueCategory = Literal[ + "setup", + "binding", + "freshness", + "missing-language", + "missing-relationship", + "ranking", + "budget", + "validation", + "other", +] + +_SENSITIVE_TEXT = re.compile( + r"(?i)(authorization\s*:|bearer\s+|password\s*[:=]|passwd\s*[:=]|" + r"token\s*[:=]|api[_-]?key\s*[:=]|secret\s*[:=]|" + r"-----BEGIN [A-Z ]*PRIVATE KEY-----)" +) + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class RepositoryEvidence(_StrictModel): + root: str + branch: str + commit: str + dirty: bool + + +class CodeMeshEvidence(_StrictModel): + commit: str + dirty: bool | None = None + working_tree_state_hash: str | None = None + project_id: str + checkout_id: str + snapshot_id: str + languages: list[str] = Field(min_length=1, max_length=8) + parser_profile: str + + @field_validator("languages") + @classmethod + def canonical_languages(cls, value: list[str]) -> list[str]: + canonical = sorted({item.strip().lower() for item in value if item.strip()}) + if not canonical: + raise ValueError("at least one non-empty language is required") + return canonical + + +class TaskEvidence(_StrictModel): + family: str + tools_attempted: list[str] = Field(default_factory=list, max_length=32) + tools_called: list[str] = Field(default_factory=list, max_length=32) + + +class ResultEvidence(_StrictModel): + helpful_paths: list[str] = Field(default_factory=list, max_length=32) + incorrect_paths: list[str] = Field(default_factory=list, max_length=32) + missed_paths: list[str] = Field(default_factory=list, max_length=32) + stale_paths: list[str] = Field(default_factory=list, max_length=32) + ambiguous_paths: list[str] = Field(default_factory=list, max_length=32) + fallback_reason: str | None = None + validation_outcome: ValidationOutcome + + +class IssueEvidence(_StrictModel): + category: IssueCategory + minimal_reproduction: str | None = None + expected_targets: list[str] = Field(default_factory=list, max_length=32) + proposed_correction: str | None = None + supersedes_feedback_ids: list[str] = Field(default_factory=list, max_length=32) + + +class ReporterEvidence(_StrictModel): + role: str + confidence: Confidence + + +class FeedbackPacket(_StrictModel): + schema_version: Literal["codemesh-feedback/v1"] = PACKET_SCHEMA_VERSION + feedback_id: str + recorded_at: datetime + classification: Classification + reporter: ReporterEvidence + repository: RepositoryEvidence + codemesh: CodeMeshEvidence + task: TaskEvidence + result: ResultEvidence + issue: IssueEvidence + + +class SessionEvidence(_StrictModel): + session_id: str + profile: Literal["development-feedback"] = "development-feedback" + local_only: Literal[True] = True + + @field_validator("session_id") + @classmethod + def session_identifier(cls, value: str) -> str: + if not re.fullmatch(r"session-[0-9a-f]{20}", value): + raise ValueError("Feedback session id has an invalid shape.") + return value + + +class BindingEvidence(_StrictModel): + status: Literal["accepted", "unavailable"] + diagnostic_code: str | None = None + + @model_validator(mode="after") + def diagnostic_matches_status(self) -> BindingEvidence: + if self.status == "unavailable" and not self.diagnostic_code: + raise ValueError( + "Unavailable feedback provenance requires a diagnostic code." + ) + if self.status == "accepted" and self.diagnostic_code is not None: + raise ValueError("Accepted feedback provenance cannot have a failure code.") + return self + + +class CodeMeshEvidenceV2(_StrictModel): + commit: str + dirty: bool + working_tree_state_hash: str | None = None + project_id: str + checkout_id: str + snapshot_id: str | None = None + languages: list[str] = Field(default_factory=list, max_length=8) + parser_profile: str | None = None + binding: BindingEvidence + + @field_validator("languages") + @classmethod + def canonical_languages(cls, value: list[str]) -> list[str]: + return sorted({item.strip().lower() for item in value if item.strip()}) + + +class ToolTraceEvent(_StrictModel): + sequence: int = Field(ge=1) + tool: str + outcome: Literal["succeeded", "failed"] + diagnostic_code: str | None = None + + +class TaskEvidenceV2(_StrictModel): + family: str + recent_tool_events: list[ToolTraceEvent] = Field( + default_factory=list, max_length=32 + ) + association: Literal["same_server_recent_window"] = "same_server_recent_window" + + +class ResultEvidenceV2(_StrictModel): + helpful_paths: list[str] = Field(default_factory=list, max_length=64) + incorrect_paths: list[str] = Field(default_factory=list, max_length=64) + missed_paths: list[str] = Field(default_factory=list, max_length=64) + stale_paths: list[str] = Field(default_factory=list, max_length=64) + ambiguous_paths: list[str] = Field(default_factory=list, max_length=64) + fallback_reason: str | None = None + validation_outcome: ValidationOutcome + + +class IssueEvidenceV2(_StrictModel): + category: IssueCategory + minimal_reproduction: str | None = None + expected_targets: list[str] = Field(default_factory=list, max_length=64) + proposed_correction: str | None = None + supersedes_feedback_ids: list[str] = Field(default_factory=list, max_length=32) + + +class FeedbackPacketV2(_StrictModel): + schema_version: Literal["codemesh-feedback/v2"] = PACKET_SCHEMA_VERSION_V2 + feedback_id: str + recorded_at: datetime + classification: Literal["diagnostic"] = "diagnostic" + session: SessionEvidence + reporter: ReporterEvidence + repository: RepositoryEvidence + codemesh: CodeMeshEvidenceV2 + task: TaskEvidenceV2 + result: ResultEvidenceV2 + issue: IssueEvidenceV2 + + @model_validator(mode="after") + def category_provenance(self) -> FeedbackPacketV2: + if self.issue.category not in {"setup", "binding", "freshness"} and ( + self.codemesh.binding.status != "accepted" + or not self.codemesh.snapshot_id + or not self.codemesh.languages + or not self.codemesh.parser_profile + ): + raise ValueError( + f"{self.issue.category} feedback requires accepted snapshot and parser provenance." + ) + if ( + self.issue.supersedes_feedback_ids + and self.result.validation_outcome != "passed" + ): + raise ValueError("Only a passing recheck may supersede earlier feedback.") + return self + + +FeedbackPacketType = FeedbackPacket | FeedbackPacketV2 + + +def record_feedback( + *, + repository_root: str | Path, + role: str, + classification: Classification, + confidence: Confidence, + project_id: str, + checkout_id: str, + snapshot_id: str, + languages: list[str], + parser_profile: str, + task_family: str, + issue_category: IssueCategory, + validation_outcome: ValidationOutcome, + tools_attempted: list[str] | None = None, + tools_called: list[str] | None = None, + helpful_paths: list[str] | None = None, + incorrect_paths: list[str] | None = None, + missed_paths: list[str] | None = None, + stale_paths: list[str] | None = None, + ambiguous_paths: list[str] | None = None, + fallback_reason: str | None = None, + minimal_reproduction: str | None = None, + expected_targets: list[str] | None = None, + proposed_correction: str | None = None, + supersedes_feedback_ids: list[str] | None = None, + codemesh_root: str | Path | None = None, +) -> tuple[FeedbackPacket, Path]: + root = _exact_git_root(repository_root) + outbox = root / OUTBOX_NAME + if outbox.is_symlink() or outbox.is_junction() or outbox.resolve() != outbox: + raise ValueError("Feedback outbox must be a local directory in the checkout.") + _require_ignored_outbox(root) + codemesh = _exact_git_root(codemesh_root or Path(__file__).resolve().parents[2]) + codemesh_dirty, codemesh_state_hash = _working_tree_state(codemesh) + + packet_data = { + "schema_version": PACKET_SCHEMA_VERSION, + "recorded_at": datetime.now(UTC), + "classification": classification, + "reporter": { + "role": _safe_text(role, "role", 80), + "confidence": confidence, + }, + "repository": { + "root": str(root), + "branch": _git(root, "branch", "--show-current") or "DETACHED", + "commit": _git(root, "rev-parse", "HEAD"), + "dirty": bool( + _git(root, "status", "--porcelain", "--untracked-files=normal") + ), + }, + "codemesh": { + "commit": _git(codemesh, "rev-parse", "HEAD"), + "dirty": codemesh_dirty, + "working_tree_state_hash": codemesh_state_hash, + "project_id": _safe_text(project_id, "project_id", 160), + "checkout_id": _safe_text(checkout_id, "checkout_id", 160), + "snapshot_id": _safe_text(snapshot_id, "snapshot_id", 160), + "languages": languages, + "parser_profile": _safe_text(parser_profile, "parser_profile", 160), + }, + "task": { + "family": _safe_text(task_family, "task_family", 120), + "tools_attempted": _safe_names(tools_attempted or [], "tools_attempted"), + "tools_called": _safe_names(tools_called or [], "tools_called"), + }, + "result": { + "helpful_paths": _safe_paths(helpful_paths or [], root, "helpful_paths"), + "incorrect_paths": _safe_paths( + incorrect_paths or [], root, "incorrect_paths" + ), + "missed_paths": _safe_paths(missed_paths or [], root, "missed_paths"), + "stale_paths": _safe_paths(stale_paths or [], root, "stale_paths"), + "ambiguous_paths": _safe_paths( + ambiguous_paths or [], root, "ambiguous_paths" + ), + "fallback_reason": _safe_optional_text( + fallback_reason, "fallback_reason", 400 + ), + "validation_outcome": validation_outcome, + }, + "issue": { + "category": issue_category, + "minimal_reproduction": _safe_optional_text( + minimal_reproduction, "minimal_reproduction", 500 + ), + "expected_targets": _safe_paths( + expected_targets or [], root, "expected_targets" + ), + "proposed_correction": _safe_optional_text( + proposed_correction, "proposed_correction", 500 + ), + "supersedes_feedback_ids": _safe_feedback_ids( + supersedes_feedback_ids or [] + ), + }, + } + normalized = FeedbackPacket.model_validate( + {**packet_data, "feedback_id": "feedback-pending"} + ).model_dump(mode="json", exclude={"feedback_id"}) + feedback_id = _content_id(normalized) + packet = FeedbackPacket.model_validate({**normalized, "feedback_id": feedback_id}) + _validate_packet_safety(packet) + outbox.mkdir(exist_ok=True) + output = outbox / f"{feedback_id}.json" + with output.open("x", encoding="utf-8") as stream: + stream.write( + json.dumps(packet.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + ) + return packet, output + + +def record_development_feedback( + *, + runtime: SessionRuntime, + confidence: Confidence, + task_family: str, + issue_category: IssueCategory, + validation_outcome: ValidationOutcome, + provenance: dict[str, Any], + recent_tool_events: list[dict[str, Any]] | None = None, + helpful_paths: list[str] | None = None, + incorrect_paths: list[str] | None = None, + missed_paths: list[str] | None = None, + stale_paths: list[str] | None = None, + ambiguous_paths: list[str] | None = None, + fallback_reason: str | None = None, + minimal_reproduction: str | None = None, + expected_targets: list[str] | None = None, + proposed_correction: str | None = None, + supersedes_feedback_ids: list[str] | None = None, +) -> tuple[FeedbackPacketV2, Path]: + session = runtime.assert_active() + root = _exact_git_root(runtime.binding.repository_root) + participant = runtime.participant + if ( + participant.project_id != runtime.binding.project_id + or participant.checkout_id != runtime.binding.checkout_id + or Path(participant.repository_root).resolve() != root + ): + raise ValueError( + "Development feedback runtime binding changed after activation." + ) + outbox = root / OUTBOX_NAME + _validate_outbox(root, outbox) + existing_packets = ( + len(list(outbox.glob("feedback-*.json"))) if outbox.exists() else 0 + ) + if existing_packets >= session.bounds.max_packets: + raise ValueError("Development feedback session packet limit has been reached.") + + availability = str(provenance.get("status") or "unavailable") + if availability not in {"accepted", "unavailable"}: + raise ValueError("Feedback provenance status must be accepted or unavailable.") + snapshot_id = _safe_optional_text( + _optional_string(provenance.get("snapshot_id")), + "snapshot_id", + 160, + ) + parser_profile = _safe_optional_text( + _optional_string(provenance.get("parser_profile")), + "parser_profile", + 160, + ) + languages = [str(value) for value in provenance.get("languages") or []] + diagnostic_code = _safe_optional_name( + _optional_string(provenance.get("diagnostic_code")), + "diagnostic_code", + ) + provenance_required = issue_category not in {"setup", "binding", "freshness"} + if provenance_required and ( + availability != "accepted" + or not snapshot_id + or not languages + or not parser_profile + ): + raise ValueError( + f"{issue_category} feedback requires accepted snapshot and parser provenance." + ) + + codemesh_root = _exact_git_root(session.codemesh.repository_root) + codemesh_dirty, codemesh_state_hash = _working_tree_state(codemesh_root) + text_limit = session.bounds.max_text_characters + path_limit = session.bounds.max_paths + path_fields = { + "helpful_paths": helpful_paths or [], + "incorrect_paths": incorrect_paths or [], + "missed_paths": missed_paths or [], + "stale_paths": stale_paths or [], + "ambiguous_paths": ambiguous_paths or [], + "expected_targets": expected_targets or [], + } + for field, values in path_fields.items(): + if len(values) > path_limit: + raise ValueError(f"{field} exceeds the session path limit of {path_limit}.") + + packet_data = { + "schema_version": PACKET_SCHEMA_VERSION_V2, + "recorded_at": runtime.clock(), + "classification": "diagnostic", + "session": { + "session_id": session.session_id, + "profile": "development-feedback", + "local_only": True, + }, + "reporter": { + "role": participant.reporter_role, + "confidence": confidence, + }, + "repository": { + "root": str(root), + "branch": _git(root, "branch", "--show-current") or "DETACHED", + "commit": _git(root, "rev-parse", "HEAD"), + "dirty": bool( + _git(root, "status", "--porcelain", "--untracked-files=normal") + ), + }, + "codemesh": { + "commit": _git(codemesh_root, "rev-parse", "HEAD"), + "dirty": codemesh_dirty, + "working_tree_state_hash": codemesh_state_hash, + "project_id": participant.project_id, + "checkout_id": participant.checkout_id, + "snapshot_id": snapshot_id, + "languages": languages, + "parser_profile": parser_profile, + "binding": { + "status": availability, + "diagnostic_code": diagnostic_code, + }, + }, + "task": { + "family": _safe_text(task_family, "task_family", 120), + "recent_tool_events": recent_tool_events or [], + "association": "same_server_recent_window", + }, + "result": { + "helpful_paths": _safe_paths( + path_fields["helpful_paths"], root, "helpful_paths" + ), + "incorrect_paths": _safe_paths( + path_fields["incorrect_paths"], root, "incorrect_paths" + ), + "missed_paths": _safe_paths( + path_fields["missed_paths"], root, "missed_paths" + ), + "stale_paths": _safe_paths(path_fields["stale_paths"], root, "stale_paths"), + "ambiguous_paths": _safe_paths( + path_fields["ambiguous_paths"], root, "ambiguous_paths" + ), + "fallback_reason": _safe_optional_text( + fallback_reason, "fallback_reason", text_limit + ), + "validation_outcome": validation_outcome, + }, + "issue": { + "category": issue_category, + "minimal_reproduction": _safe_optional_text( + minimal_reproduction, "minimal_reproduction", text_limit + ), + "expected_targets": _safe_paths( + path_fields["expected_targets"], root, "expected_targets" + ), + "proposed_correction": _safe_optional_text( + proposed_correction, "proposed_correction", text_limit + ), + "supersedes_feedback_ids": _safe_feedback_ids( + supersedes_feedback_ids or [] + ), + }, + } + normalized = FeedbackPacketV2.model_validate( + {**packet_data, "feedback_id": "feedback-pending"} + ).model_dump(mode="json", exclude={"feedback_id"}) + feedback_id = _content_id(normalized) + packet = FeedbackPacketV2.model_validate({**normalized, "feedback_id": feedback_id}) + _validate_packet_safety(packet) + outbox.mkdir(exist_ok=True) + output = outbox / f"{feedback_id}.json" + with output.open("x", encoding="utf-8") as stream: + stream.write( + json.dumps(packet.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + ) + return packet, output + + +def validate_feedback(path: str | Path) -> FeedbackPacketType: + requested = Path(path) + if ( + requested.is_symlink() + or requested.is_junction() + or not requested.is_file() + or requested.stat().st_nlink != 1 + ): + raise ValueError("Feedback packet must be an unlinked regular file.") + source = requested.resolve(strict=True) + if source.stat().st_size > 65_536: + raise ValueError("Feedback packet exceeds the 65,536-byte validation limit.") + payload = json.loads(source.read_text(encoding="utf-8")) + schema_version = payload.get("schema_version") + if schema_version == PACKET_SCHEMA_VERSION: + packet: FeedbackPacketType = FeedbackPacket.model_validate(payload) + elif schema_version == PACKET_SCHEMA_VERSION_V2: + packet = FeedbackPacketV2.model_validate(payload) + else: + raise ValueError(f"Unsupported CodeMesh feedback schema: {schema_version}.") + expected_id = _content_id( + {key: value for key, value in payload.items() if key != "feedback_id"} + ) + if packet.feedback_id != expected_id: + raise ValueError( + f"Feedback id mismatch for {source}: expected {expected_id}, " + f"found {packet.feedback_id}." + ) + _validate_packet_safety(packet) + return packet + + +def summarize_feedback(roots: list[str | Path]) -> dict[str, object]: + if not roots: + raise ValueError("At least one managed checkout root is required.") + + packets: list[tuple[FeedbackPacketType, Path]] = [] + seen: set[str] = set() + for root_value in roots: + root = _exact_git_root(root_value) + outbox = root / OUTBOX_NAME + if not outbox.is_dir(): + continue + for path in sorted(outbox.glob("*.json")): + packet = validate_feedback(path) + if Path(packet.repository.root).resolve() != root: + raise ValueError( + f"Feedback packet {path} claims repository root " + f"{packet.repository.root}, expected {root}." + ) + if packet.feedback_id not in seen: + seen.add(packet.feedback_id) + packets.append((packet, path)) + + groups: dict[tuple[str, str, tuple[str, ...]], list[FeedbackPacketType]] = ( + defaultdict(list) + ) + for packet, _ in packets: + key = ( + packet.issue.category, + packet.task.family, + tuple(packet.codemesh.languages), + ) + groups[key].append(packet) + + ranked = [] + for (category, task_family, languages), members in groups.items(): + superseded = _superseded_feedback_ids(members) + unresolved_failures = [ + member.feedback_id + for member in members + if member.result.validation_outcome == "failed" + and member.feedback_id not in superseded + ] + all_passed = all( + member.result.validation_outcome == "passed" for member in members + ) + resolved = bool(superseded) and not unresolved_failures + verified = all_passed and not superseded + status = "resolved" if resolved else "verified" if verified else "open" + score = ( + 0 + if status in {"resolved", "verified"} + else _priority_score(category, members) + ) + ranked.append( + { + "status": status, + "priority": _priority_label(score), + "priority_score": score, + "issue_category": category, + "task_family": task_family, + "languages": list(languages), + "occurrences": len(members), + "validation_outcomes": _counts( + member.result.validation_outcome for member in members + ), + "roles": sorted({member.reporter.role for member in members}), + "proposed_corrections": sorted( + { + member.issue.proposed_correction + for member in members + if member.issue.proposed_correction + } + ), + "superseded_feedback_ids": sorted(superseded), + "unresolved_failure_ids": sorted(unresolved_failures), + "provenance": [ + { + "feedback_id": member.feedback_id, + "recorded_at": member.recorded_at.isoformat(), + "repository_commit": member.repository.commit, + "codemesh_commit": member.codemesh.commit, + "codemesh_dirty": member.codemesh.dirty, + "codemesh_working_tree_state_hash": ( + member.codemesh.working_tree_state_hash + ), + "project_id": member.codemesh.project_id, + "checkout_id": member.codemesh.checkout_id, + "snapshot_id": member.codemesh.snapshot_id, + } + for member in sorted(members, key=lambda item: item.feedback_id) + ], + } + ) + ranked.sort( + key=lambda item: ( + -int(item["priority_score"]), + str(item["issue_category"]), + str(item["task_family"]), + ) + ) + return { + "schema_version": SUMMARY_SCHEMA_VERSION, + "generated_at": datetime.now(UTC).isoformat(), + "packet_count": len(packets), + "invalid_packet_count": 0, + "source_outboxes": sorted({str(path.parent.resolve()) for _, path in packets}), + "priorities": ranked, + } + + +def list_session_feedback( + runtime: SessionRuntime, + *, + include_legacy: bool = False, + state: Literal["open", "resolved", "verified"] | None = None, + category: IssueCategory | None = None, + task_family: str | None = None, + language: str | None = None, + reporter_role: str | None = None, + feedback_id: str | None = None, + offset: int = 0, + limit: int = 50, +) -> dict[str, object]: + session = _require_maintainer_runtime(runtime) + if offset < 0 or limit < 1 or limit > 100: + raise ValueError("Feedback pagination requires offset >= 0 and limit 1..100.") + packets, invalid = _session_packets(runtime, include_legacy=include_legacy) + superseded = _superseded_feedback_ids([packet for packet, _path, _scope in packets]) + rows: list[dict[str, object]] = [] + for packet, path, scope in packets: + packet_state = _packet_state(packet, superseded) + languages = list(packet.codemesh.languages) + if state and packet_state != state: + continue + if category and packet.issue.category != category: + continue + if task_family and packet.task.family != task_family: + continue + if language and language.casefold() not in { + item.casefold() for item in languages + }: + continue + if reporter_role and packet.reporter.role != reporter_role: + continue + if feedback_id and packet.feedback_id != feedback_id: + continue + rows.append( + { + "feedback_id": packet.feedback_id, + "schema_version": packet.schema_version, + "session_scope": scope, + "state": packet_state, + "recorded_at": packet.recorded_at.isoformat(), + "category": packet.issue.category, + "task_family": packet.task.family, + "validation_outcome": packet.result.validation_outcome, + "confidence": packet.reporter.confidence, + "reporter_role": packet.reporter.role, + "languages": languages, + "checkout_id": packet.codemesh.checkout_id, + "packet_path": ( + f"{packet.codemesh.checkout_id}/{OUTBOX_NAME}/{path.name}" + ), + } + ) + rows.sort(key=lambda item: (str(item["recorded_at"]), str(item["feedback_id"]))) + selected = rows[offset : offset + limit] + return { + "schema_version": "codemesh-session-feedback-list/v1", + "session_id": session.session_id, + "total": len(rows), + "offset": offset, + "limit": limit, + "items": selected, + "invalid_packets": invalid[:100], + "invalid_packet_count": len(invalid), + "human_review_required": True, + } + + +def get_session_feedback( + runtime: SessionRuntime, + feedback_id: str, + *, + include_legacy: bool = False, +) -> dict[str, object]: + session = _require_maintainer_runtime(runtime) + identifier = _safe_feedback_ids([feedback_id])[0] + packets, invalid = _session_packets(runtime, include_legacy=include_legacy) + matches = [item for item in packets if item[0].feedback_id == identifier] + if len(matches) != 1: + if not matches: + raise ValueError( + f"Feedback id was not found in this session: {identifier}." + ) + raise ValueError( + f"Feedback id is ambiguous across session outboxes: {identifier}." + ) + packet, _path, scope = matches[0] + superseded = _superseded_feedback_ids( + [candidate for candidate, _candidate_path, _candidate_scope in packets] + ) + return { + "schema_version": "codemesh-session-feedback-detail/v1", + "session_id": session.session_id, + "session_scope": scope, + "state": _packet_state(packet, superseded), + "packet": packet.model_dump(mode="json"), + "invalid_packet_count": len(invalid), + "human_review_required": True, + } + + +def prepare_feedback_resolution( + runtime: SessionRuntime, + *, + feedback_ids: list[str], + reproduction_state: Literal["reproduced", "not-reproduced", "blocked"], + change_summary: str, + proposed_paths: list[str], + risks: list[str] | None = None, + required_tests: list[str] | None = None, + unresolved_questions: list[str] | None = None, + include_legacy: bool = False, +) -> dict[str, object]: + session = _require_maintainer_runtime(runtime) + identifiers = _safe_feedback_ids(feedback_ids) + if not identifiers: + raise ValueError("At least one feedback id is required.") + details = [ + get_session_feedback(runtime, identifier, include_legacy=include_legacy) + for identifier in identifiers + ] + codemesh_root = _exact_git_root(session.codemesh.repository_root) + paths = _safe_paths(proposed_paths, codemesh_root, "proposed_paths") + if not paths: + raise ValueError("At least one proposed CodeMesh-relative path is required.") + packet_provenance = [] + for detail in details: + packet = detail["packet"] + assert isinstance(packet, dict) + repository = packet["repository"] + codemesh = packet["codemesh"] + assert isinstance(repository, dict) and isinstance(codemesh, dict) + packet_provenance.append( + { + "feedback_id": packet["feedback_id"], + "session_scope": detail["session_scope"], + "repository_commit": repository["commit"], + "codemesh_commit": codemesh["commit"], + "snapshot_id": codemesh.get("snapshot_id"), + } + ) + unsigned = { + "schema_version": RESOLUTION_PLAN_SCHEMA_VERSION, + "session_id": session.session_id, + "feedback_ids": identifiers, + "feedback_provenance": sorted( + packet_provenance, + key=lambda item: str(item["feedback_id"]), + ), + "reproduction_state": reproduction_state, + "change_summary": _safe_text(change_summary, "change_summary", 1000), + "proposed_paths": paths, + "risks": _safe_text_list(risks or [], "risks", 500, 32), + "required_tests": _safe_text_list( + required_tests or [], "required_tests", 300, 32 + ), + "unresolved_questions": _safe_text_list( + unresolved_questions or [], "unresolved_questions", 500, 32 + ), + "evidence_boundary": ( + "This deterministic plan is diagnostic product input. It is not human " + "approval, implementation evidence, release authority, or product-benefit evidence." + ), + "human_review_required": True, + "human_approved": False, + } + return {**unsigned, "plan_hash": _hash_payload(unsigned)} + + +def _session_packets( + runtime: SessionRuntime, + *, + include_legacy: bool, +) -> tuple[list[tuple[FeedbackPacketType, Path, str]], list[dict[str, str]]]: + session = _require_maintainer_runtime(runtime) + packets: list[tuple[FeedbackPacketType, Path, str]] = [] + invalid: list[dict[str, str]] = [] + for participant in session.clients: + root = _exact_git_root(participant.repository_root) + outbox = root / OUTBOX_NAME + if outbox.is_symlink() or outbox.is_junction() or outbox.resolve() != outbox: + invalid.append( + {"checkout_id": participant.checkout_id, "code": "unsafe_outbox"} + ) + continue + if not outbox.is_dir(): + continue + for path in sorted(outbox.glob("*.json")): + try: + if ( + not path.is_file() + or path.is_symlink() + or path.is_junction() + or path.resolve().parent != outbox.resolve() + or path.stat().st_nlink != 1 + ): + raise ValueError("unsafe_packet_file") + packet = validate_feedback(path) + if Path(packet.repository.root).resolve() != root: + raise ValueError("claimed_root_mismatch") + if isinstance(packet, FeedbackPacketV2): + if packet.session.session_id != session.session_id: + continue + scope = "active_session" + elif include_legacy: + scope = "legacy_unscoped" + else: + continue + packets.append((packet, path, scope)) + except (OSError, ValueError, json.JSONDecodeError): + invalid.append( + { + "checkout_id": participant.checkout_id, + "packet": path.name, + "code": "invalid_packet", + } + ) + return packets, invalid + + +def _require_maintainer_runtime(runtime: SessionRuntime) -> Any: + if runtime.profile != "feedback-maintainer": + raise ValueError("Feedback intake requires the feedback-maintainer profile.") + return runtime.assert_active() + + +def _packet_state(packet: FeedbackPacketType, superseded: set[str]) -> str: + if packet.feedback_id in superseded: + return "resolved" + if packet.result.validation_outcome == "passed": + return "verified" + return "open" + + +def _superseded_feedback_ids(packets: list[FeedbackPacketType]) -> set[str]: + by_id = {packet.feedback_id: packet for packet in packets} + superseded: set[str] = set() + for recheck in packets: + if recheck.result.validation_outcome != "passed": + continue + for identifier in recheck.issue.supersedes_feedback_ids: + original = by_id.get(identifier) + if ( + original is not None + and original.result.validation_outcome != "passed" + and original.repository.root == recheck.repository.root + and original.issue.category == recheck.issue.category + and original.task.family == recheck.task.family + ): + superseded.add(identifier) + return superseded + + +def _exact_git_root(value: str | Path) -> Path: + candidate = Path(value).expanduser().resolve(strict=True) + if not candidate.is_dir(): + raise ValueError(f"Repository root is not a directory: {candidate}") + discovered = Path(_git(candidate, "rev-parse", "--show-toplevel")).resolve() + if discovered != candidate: + raise ValueError( + f"Repository root must be exact: supplied {candidate}, git reports {discovered}." + ) + return discovered + + +def _validate_packet_safety(packet: FeedbackPacketType) -> None: + root = Path(packet.repository.root).expanduser() + if not root.is_absolute(): + raise ValueError("Feedback repository root must be absolute.") + root = root.resolve(strict=False) + text_fields: list[tuple[str | None, str, int]] = [ + (packet.reporter.role, "reporter.role", 80), + (packet.repository.branch, "repository.branch", 200), + (packet.repository.commit, "repository.commit", 160), + (packet.codemesh.commit, "codemesh.commit", 160), + (packet.codemesh.project_id, "codemesh.project_id", 160), + (packet.codemesh.checkout_id, "codemesh.checkout_id", 160), + (packet.codemesh.snapshot_id, "codemesh.snapshot_id", 160), + (packet.codemesh.parser_profile, "codemesh.parser_profile", 160), + (packet.task.family, "task.family", 120), + (packet.result.fallback_reason, "result.fallback_reason", 2000), + (packet.issue.minimal_reproduction, "issue.minimal_reproduction", 2000), + (packet.issue.proposed_correction, "issue.proposed_correction", 2000), + ] + if isinstance(packet, FeedbackPacketV2): + text_fields.append( + ( + packet.codemesh.binding.diagnostic_code, + "codemesh.binding.diagnostic_code", + 120, + ) + ) + sequences = [event.sequence for event in packet.task.recent_tool_events] + if sequences != sorted(set(sequences)): + raise ValueError( + "Feedback tool-trace sequences must be unique and ordered." + ) + for event in packet.task.recent_tool_events: + _safe_text(event.tool, "task.recent_tool_events.tool", 120) + if event.diagnostic_code is not None: + _safe_optional_name( + event.diagnostic_code, + "task.recent_tool_events.diagnostic_code", + ) + else: + for name in [*packet.task.tools_attempted, *packet.task.tools_called]: + _safe_text(name, "task.tools", 120) + for value, field, maximum in text_fields: + if value is not None: + _safe_text(value, field, maximum) + languages = sorted( + { + _safe_text(value, "codemesh.languages", 40).lower() + for value in packet.codemesh.languages + } + ) + if languages != packet.codemesh.languages: + raise ValueError("codemesh.languages must be lowercase, sorted, and unique.") + path_fields = { + "helpful_paths": packet.result.helpful_paths, + "incorrect_paths": packet.result.incorrect_paths, + "missed_paths": packet.result.missed_paths, + "stale_paths": packet.result.stale_paths, + "ambiguous_paths": packet.result.ambiguous_paths, + "expected_targets": packet.issue.expected_targets, + } + for field, values in path_fields.items(): + if _safe_paths(values, root, field) != values: + raise ValueError(f"{field} must be sorted and unique.") + if _safe_feedback_ids(packet.issue.supersedes_feedback_ids) != ( + packet.issue.supersedes_feedback_ids + ): + raise ValueError("supersedes_feedback_ids must be sorted and unique.") + + +def _require_ignored_outbox(root: Path) -> None: + probe = f"{OUTBOX_NAME}/_codemesh-ignore-probe.json" + result = subprocess.run( + ["git", "check-ignore", "--no-index", "--quiet", probe], + cwd=root, + check=False, + ) + if result.returncode != 0: + raise ValueError( + f"{root / OUTBOX_NAME} is not ignored. Add '/{OUTBOX_NAME}/' to " + ".gitignore through the checkout's managed instance configuration first." + ) + + +def _validate_outbox(root: Path, outbox: Path) -> None: + if outbox.is_symlink() or outbox.is_junction() or outbox.resolve() != outbox: + raise ValueError("Feedback outbox must be a local directory in the checkout.") + _require_ignored_outbox(root) + + +def _git(root: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + detail = result.stderr.strip() or result.stdout.strip() or "unknown git error" + raise ValueError(f"git {' '.join(arguments)} failed in {root}: {detail}") + return result.stdout.strip() + + +def _safe_names(values: list[str], field: str) -> list[str]: + return sorted({_safe_text(value, field, 120) for value in values}) + + +def _safe_text_list( + values: list[str], field: str, maximum: int, maximum_items: int +) -> list[str]: + if len(values) > maximum_items: + raise ValueError(f"{field} exceeds {maximum_items} items.") + return sorted({_safe_text(value, field, maximum) for value in values}) + + +def _safe_optional_name(value: str | None, field: str) -> str | None: + if value is None: + return None + text = _safe_text(value, field, 120) + if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,119}", text): + raise ValueError(f"{field} contains unsupported characters.") + return text + + +def _safe_feedback_ids(values: list[str]) -> list[str]: + identifiers = _safe_names(values, "supersedes_feedback_ids") + invalid = [ + value + for value in identifiers + if not re.fullmatch(r"feedback-[0-9a-f]{20}", value) + ] + if invalid: + raise ValueError( + "supersedes_feedback_ids contains invalid feedback id(s): " + + ", ".join(invalid) + ) + return identifiers + + +def _safe_paths(values: list[str], root: Path, field: str) -> list[str]: + paths: set[str] = set() + for value in values: + text = _safe_text(value, field, 300).replace("\\", "/") + candidate = Path(text) + if candidate.is_absolute() or ".." in candidate.parts: + raise ValueError(f"{field} must contain repository-relative paths: {value}") + resolved = (root / candidate).resolve() + if not resolved.is_relative_to(root): + raise ValueError(f"{field} escapes repository root: {value}") + paths.add(candidate.as_posix()) + return sorted(paths) + + +def _safe_optional_text(value: str | None, field: str, maximum: int) -> str | None: + if value is None: + return None + return _safe_text(value, field, maximum) + + +def _safe_text(value: str, field: str, maximum: int) -> str: + text = value.strip() + if not text: + raise ValueError(f"{field} must not be empty.") + if len(text) > maximum: + raise ValueError(f"{field} exceeds {maximum} characters.") + if any(ord(character) < 32 for character in text): + raise ValueError(f"{field} contains control characters.") + if _SENSITIVE_TEXT.search(text): + raise ValueError(f"{field} appears to contain a credential or secret.") + return text + + +def _content_id(payload: dict[str, object]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return f"feedback-{hashlib.sha256(encoded.encode('utf-8')).hexdigest()[:20]}" + + +def _hash_payload(payload: dict[str, object]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def _optional_string(value: Any) -> str | None: + return str(value) if value is not None else None + + +def _working_tree_state(root: Path) -> tuple[bool, str | None]: + status = subprocess.run( + ["git", "status", "--porcelain=v1", "-z", "--untracked-files=normal"], + cwd=root, + check=True, + capture_output=True, + ).stdout + if not status: + return False, None + + digest = hashlib.sha256() + digest.update(status) + digest.update( + subprocess.run( + ["git", "diff", "--binary", "HEAD", "--"], + cwd=root, + check=True, + capture_output=True, + ).stdout + ) + untracked = subprocess.run( + ["git", "ls-files", "--others", "--exclude-standard", "-z"], + cwd=root, + check=True, + capture_output=True, + ).stdout.split(b"\0") + for raw_path in sorted(path for path in untracked if path): + path = root / raw_path.decode("utf-8", errors="surrogateescape") + digest.update(raw_path) + if path.is_file(): + digest.update(path.read_bytes()) + return True, digest.hexdigest() + + +def _counts(values: object) -> dict[str, int]: + counts: dict[str, int] = {} + for value in values: # type: ignore[union-attr] + counts[str(value)] = counts.get(str(value), 0) + 1 + return dict(sorted(counts.items())) + + +def _priority_score(category: str, members: list[FeedbackPacketType]) -> int: + category_weight = { + "binding": 5, + "freshness": 5, + "setup": 4, + "missing-language": 4, + "missing-relationship": 4, + "ranking": 3, + "validation": 3, + "budget": 2, + "other": 1, + }[category] + failure_weight = sum( + 2 if member.result.validation_outcome == "failed" else 1 + for member in members + if member.result.validation_outcome in {"failed", "inconclusive"} + ) + incorrect_weight = sum( + bool(member.result.incorrect_paths or member.result.stale_paths) + for member in members + ) + return category_weight + min(len(members), 5) + failure_weight + incorrect_weight + + +def _priority_label(score: int) -> str: + if score >= 8: + return "high" + if score >= 5: + return "medium" + return "low" diff --git a/agent-access/codemesh_agent_access/feedback_session.py b/agent-access/codemesh_agent_access/feedback_session.py new file mode 100644 index 0000000..8d1f858 --- /dev/null +++ b/agent-access/codemesh_agent_access/feedback_session.py @@ -0,0 +1,643 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +from typing import Callable, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from .binding import RepositoryBinding +from .feedback import OUTBOX_NAME + + +SESSION_PLAN_SCHEMA_VERSION = "codemesh-development-feedback-session-plan/v1" +SESSION_SCHEMA_VERSION = "codemesh-development-feedback-session/v1" +REVOCATION_SCHEMA_VERSION = "codemesh-development-feedback-revocation/v1" + +CLIENT_PROFILE = "development-feedback" +MAINTAINER_PROFILE = "feedback-maintainer" +FEEDBACK_PROFILES = (CLIENT_PROFILE, MAINTAINER_PROFILE) + +NORMAL_TOOLS = ( + "codemesh_get_context_package", + "codemesh_get_repository_status", + "codemesh_list_repositories", + "codemesh_get_node", +) +CLIENT_TOOLS = ( + *NORMAL_TOOLS, + "codemesh_get_feedback_session", + "codemesh_record_feedback", +) +MAINTAINER_TOOLS = ( + *NORMAL_TOOLS, + "codemesh_list_feedback", + "codemesh_get_feedback", + "codemesh_prepare_feedback_resolution", +) + +MAX_SESSION_DURATION = timedelta(days=7) +DEFAULT_MAX_PACKETS = 100 +DEFAULT_MAX_REQUESTS_PER_MINUTE = 20 +DEFAULT_MAX_TEXT_CHARACTERS = 500 +DEFAULT_MAX_PATHS = 32 + +_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9:._/-]{0,159}") + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class SessionParticipant(_StrictModel): + project_id: str + checkout_id: str + repository_root: str + reporter_role: str + + @field_validator("project_id", "checkout_id", "reporter_role") + @classmethod + def safe_identifier(cls, value: str) -> str: + text = value.strip() + if not _IDENTIFIER.fullmatch(text): + raise ValueError("Session identity contains unsupported characters.") + return text + + @field_validator("repository_root") + @classmethod + def absolute_root(cls, value: str) -> str: + path = Path(value).expanduser() + if not path.is_absolute(): + raise ValueError("Session repository roots must be absolute paths.") + return str(path.resolve(strict=False)) + + def binding(self) -> RepositoryBinding: + return RepositoryBinding( + self.project_id, + self.checkout_id, + self.repository_root, + ) + + +class SessionBounds(_StrictModel): + max_packets: int = Field(default=DEFAULT_MAX_PACKETS, ge=1, le=1000) + max_requests_per_minute: int = Field( + default=DEFAULT_MAX_REQUESTS_PER_MINUTE, + ge=1, + le=120, + ) + max_text_characters: int = Field( + default=DEFAULT_MAX_TEXT_CHARACTERS, + ge=80, + le=2000, + ) + max_paths: int = Field(default=DEFAULT_MAX_PATHS, ge=1, le=64) + + +class SessionPayload(_StrictModel): + session_id: str + created_at: datetime + expires_at: datetime + codemesh: SessionParticipant + clients: list[SessionParticipant] = Field(min_length=1, max_length=32) + profiles: dict[str, list[str]] + local_only: Literal[True] = True + provider_free: Literal[True] = True + bounds: SessionBounds = Field(default_factory=SessionBounds) + + @field_validator("session_id") + @classmethod + def session_identifier(cls, value: str) -> str: + if not re.fullmatch(r"session-[0-9a-f]{20}", value): + raise ValueError("Session id has an invalid shape.") + return value + + @field_validator("created_at", "expires_at") + @classmethod + def timezone_required(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("Session timestamps must include a timezone.") + return value.astimezone(UTC) + + @model_validator(mode="after") + def valid_contract(self) -> SessionPayload: + if self.expires_at <= self.created_at: + raise ValueError("Session expiry must be after creation.") + if self.expires_at - self.created_at > MAX_SESSION_DURATION: + raise ValueError("Session duration exceeds seven days.") + expected_profiles = { + CLIENT_PROFILE: list(CLIENT_TOOLS), + MAINTAINER_PROFILE: list(MAINTAINER_TOOLS), + } + if self.profiles != expected_profiles: + raise ValueError( + "Session tool profiles do not match the supported contract." + ) + identities = [(item.project_id, item.checkout_id) for item in self.clients] + roots = [ + os.path.normcase(os.path.realpath(item.repository_root)) + for item in self.clients + ] + if len(identities) != len(set(identities)) or len(roots) != len(set(roots)): + raise ValueError("Session client participants must be unique.") + codemesh_root = os.path.normcase( + os.path.realpath(self.codemesh.repository_root) + ) + if codemesh_root in roots: + raise ValueError("The CodeMesh checkout cannot also be a client checkout.") + return self + + +class SessionManifest(_StrictModel): + schema_version: Literal["codemesh-development-feedback-session/v1"] = ( + SESSION_SCHEMA_VERSION + ) + payload: SessionPayload + payload_sha256: str + + @field_validator("payload_sha256") + @classmethod + def sha256_shape(cls, value: str) -> str: + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError("Session payload hash has an invalid shape.") + return value + + +class SessionPlan(_StrictModel): + schema_version: Literal["codemesh-development-feedback-session-plan/v1"] = ( + SESSION_PLAN_SCHEMA_VERSION + ) + manifest_path: str + manifest: SessionManifest + manifest_file_sha256: str + plan_hash: str + + @field_validator("manifest_path") + @classmethod + def absolute_manifest_path(cls, value: str) -> str: + path = Path(value).expanduser() + if not path.is_absolute(): + raise ValueError("Session manifest path must be absolute.") + return str(path.resolve(strict=False)) + + @field_validator("manifest_file_sha256", "plan_hash") + @classmethod + def hash_shape(cls, value: str) -> str: + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError("Session plan hash has an invalid shape.") + return value + + +class SessionRevocation(_StrictModel): + schema_version: Literal["codemesh-development-feedback-revocation/v1"] = ( + REVOCATION_SCHEMA_VERSION + ) + session_id: str + manifest_file_sha256: str + revoked_at: datetime + reason: str = Field(min_length=1, max_length=200) + content_sha256: str + + @field_validator("session_id") + @classmethod + def session_identifier(cls, value: str) -> str: + if not re.fullmatch(r"session-[0-9a-f]{20}", value): + raise ValueError("Revocation session id has an invalid shape.") + return value + + @field_validator("manifest_file_sha256", "content_sha256") + @classmethod + def hash_shape(cls, value: str) -> str: + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError("Revocation hash has an invalid shape.") + return value + + @field_validator("revoked_at") + @classmethod + def timezone_required(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("Revocation timestamp must include a timezone.") + return value.astimezone(UTC) + + +@dataclass(frozen=True) +class SessionRuntime: + manifest_path: Path + manifest_file_sha256: str + profile: str + binding: RepositoryBinding + participant: SessionParticipant + clock: Callable[[], datetime] + + @property + def payload(self) -> SessionPayload: + return load_active_session( + self.manifest_path, + self.manifest_file_sha256, + profile=self.profile, + binding=self.binding, + now=self.clock(), + ).payload + + def assert_active(self) -> SessionPayload: + return self.payload + + +def create_session_plan( + *, + manifest_path: str | Path, + codemesh: SessionParticipant, + clients: list[SessionParticipant], + expires_at: datetime, + bounds: SessionBounds | None = None, + now: datetime | None = None, +) -> SessionPlan: + created_at = _utc(now or datetime.now(UTC)) + expires_at = _utc(expires_at) + codemesh = _validated_participant(codemesh, require_ignored_outbox=False) + clients = [ + _validated_participant(participant, require_ignored_outbox=True) + for participant in clients + ] + clients.sort( + key=lambda item: ( + item.project_id.casefold(), + item.checkout_id.casefold(), + os.path.normcase(item.repository_root), + ) + ) + selected_bounds = bounds or SessionBounds() + profiles = { + CLIENT_PROFILE: list(CLIENT_TOOLS), + MAINTAINER_PROFILE: list(MAINTAINER_TOOLS), + } + core = { + "created_at": created_at.isoformat(), + "expires_at": expires_at.isoformat(), + "codemesh": codemesh.model_dump(mode="json"), + "clients": [item.model_dump(mode="json") for item in clients], + "profiles": profiles, + "bounds": selected_bounds.model_dump(mode="json"), + } + session_id = f"session-{_hash_json(core)[:20]}" + payload = SessionPayload( + session_id=session_id, + created_at=created_at, + expires_at=expires_at, + codemesh=codemesh, + clients=clients, + profiles=profiles, + bounds=selected_bounds, + ) + payload_hash = _hash_json(payload.model_dump(mode="json")) + manifest = SessionManifest(payload=payload, payload_sha256=payload_hash) + manifest_hash = _hash_bytes(_manifest_bytes(manifest)) + requested_path = Path(manifest_path).expanduser() + if not requested_path.is_absolute(): + raise ValueError("Session manifest path must be absolute.") + path = requested_path.resolve(strict=False) + if not path.parent.is_dir(): + raise ValueError("Session manifest parent directory does not exist.") + unsigned = { + "schema_version": SESSION_PLAN_SCHEMA_VERSION, + "manifest_path": str(path), + "manifest": manifest.model_dump(mode="json"), + "manifest_file_sha256": manifest_hash, + } + return SessionPlan(**unsigned, plan_hash=_hash_json(unsigned)) + + +def write_session_plan(plan: SessionPlan, path: str | Path) -> None: + verify_session_plan(plan) + _exclusive_write(Path(path), _json_bytes(plan.model_dump(mode="json"))) + + +def load_session_plan(path: str | Path) -> SessionPlan: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + plan = SessionPlan.model_validate(payload) + verify_session_plan(plan) + return plan + + +def verify_session_plan(plan: SessionPlan) -> None: + payload = plan.model_dump(mode="json") + plan_hash = payload.pop("plan_hash") + if _hash_json(payload) != plan_hash: + raise ValueError("Session plan hash does not match its contents.") + _verify_manifest(plan.manifest) + if _hash_bytes(_manifest_bytes(plan.manifest)) != plan.manifest_file_sha256: + raise ValueError("Planned session manifest hash does not match its contents.") + + +def activate_session(plan: SessionPlan, approved_plan_hash: str) -> Path: + verify_session_plan(plan) + if approved_plan_hash != plan.plan_hash: + raise ValueError("Approved plan hash does not match the reviewed session plan.") + path = Path(plan.manifest_path) + if ( + not path.parent.is_dir() + or path.parent.is_symlink() + or path.parent.is_junction() + or path.parent.resolve() != path.parent + ): + raise ValueError("Session manifest parent changed after review.") + _exclusive_write(path, _manifest_bytes(plan.manifest)) + return path + + +def inspect_session( + manifest_path: str | Path, + expected_sha256: str, + *, + now: datetime | None = None, +) -> dict[str, object]: + manifest, path = _load_manifest(manifest_path, expected_sha256) + payload = manifest.payload + current = _utc(now or datetime.now(UTC)) + status = "expired" if current >= payload.expires_at else "active" + revocation = _load_revocation( + path, + expected_sha256, + payload.session_id, + required=False, + ) + if revocation is not None: + status = "revoked" + return { + "schema_version": "codemesh-development-feedback-session-status/v1", + "session_id": payload.session_id, + "status": status, + "created_at": payload.created_at.isoformat(), + "expires_at": payload.expires_at.isoformat(), + "local_only": payload.local_only, + "provider_free": payload.provider_free, + "client_count": len(payload.clients), + "profiles": sorted(payload.profiles), + "manifest_file_sha256": expected_sha256, + } + + +def load_active_session( + manifest_path: str | Path, + expected_sha256: str, + *, + profile: str, + binding: RepositoryBinding, + now: datetime | None = None, +) -> SessionManifest: + if profile not in FEEDBACK_PROFILES: + raise ValueError("A development feedback session requires a feedback profile.") + manifest, path = _load_manifest(manifest_path, expected_sha256) + payload = manifest.payload + current = _utc(now or datetime.now(UTC)) + if current >= payload.expires_at: + raise ValueError("Development feedback session has expired.") + if ( + _load_revocation( + path, + expected_sha256, + payload.session_id, + required=False, + ) + is not None + ): + raise ValueError("Development feedback session has been revoked.") + participants = payload.clients if profile == CLIENT_PROFILE else [payload.codemesh] + matching = [item for item in participants if _binding_matches(item, binding)] + if len(matching) != 1: + raise ValueError( + "Development feedback session does not authorize this binding." + ) + return manifest + + +def create_session_runtime( + manifest_path: str | Path, + expected_sha256: str, + *, + profile: str, + binding: RepositoryBinding, + clock: Callable[[], datetime] | None = None, +) -> SessionRuntime: + selected_clock = clock or (lambda: datetime.now(UTC)) + manifest = load_active_session( + manifest_path, + expected_sha256, + profile=profile, + binding=binding, + now=selected_clock(), + ) + participants = ( + manifest.payload.clients + if profile == CLIENT_PROFILE + else [manifest.payload.codemesh] + ) + participant = next(item for item in participants if _binding_matches(item, binding)) + return SessionRuntime( + manifest_path=Path(manifest_path).resolve(strict=True), + manifest_file_sha256=expected_sha256, + profile=profile, + binding=binding, + participant=participant, + clock=selected_clock, + ) + + +def revoke_session( + manifest_path: str | Path, + expected_sha256: str, + *, + reason: str, + now: datetime | None = None, +) -> Path: + manifest, path = _load_manifest(manifest_path, expected_sha256) + safe_reason = reason.strip() + if ( + not safe_reason + or len(safe_reason) > 200 + or any(ord(item) < 32 for item in safe_reason) + ): + raise ValueError("Revocation reason must contain 1 to 200 safe characters.") + draft = SessionRevocation( + session_id=manifest.payload.session_id, + manifest_file_sha256=expected_sha256, + revoked_at=_utc(now or datetime.now(UTC)), + reason=safe_reason, + content_sha256="0" * 64, + ) + unsigned = draft.model_dump(mode="json", exclude={"content_sha256"}) + revocation = draft.model_copy(update={"content_sha256": _hash_json(unsigned)}) + output = revocation_path(path) + _exclusive_write(output, _json_bytes(revocation.model_dump(mode="json"))) + return output + + +def revocation_path(manifest_path: str | Path) -> Path: + path = Path(manifest_path) + return path.with_name(f"{path.name}.revoked.json") + + +def _load_manifest( + manifest_path: str | Path, + expected_sha256: str, +) -> tuple[SessionManifest, Path]: + if not re.fullmatch(r"[0-9a-f]{64}", expected_sha256): + raise ValueError("Expected session manifest hash has an invalid shape.") + requested = Path(manifest_path).expanduser() + if ( + requested.is_symlink() + or requested.is_junction() + or not requested.is_file() + or requested.stat().st_nlink != 1 + ): + raise ValueError("Session manifest must be an unlinked local regular file.") + path = requested.resolve(strict=True) + if path.stat().st_size > 65_536: + raise ValueError("Session manifest exceeds the 65,536-byte limit.") + raw = path.read_bytes() + if _hash_bytes(raw) != expected_sha256: + raise ValueError("Session manifest hash does not match the reviewed value.") + manifest = SessionManifest.model_validate(json.loads(raw)) + _verify_manifest(manifest) + return manifest, path + + +def _verify_manifest(manifest: SessionManifest) -> None: + expected = _hash_json(manifest.payload.model_dump(mode="json")) + if expected != manifest.payload_sha256: + raise ValueError("Session payload hash does not match its contents.") + + +def _load_revocation( + manifest_path: Path, + expected_sha256: str, + expected_session_id: str, + *, + required: bool, +) -> SessionRevocation | None: + path = revocation_path(manifest_path) + if path.is_symlink() or path.is_junction(): + raise ValueError( + "Session revocation record must be an unlinked local regular file." + ) + if not path.exists(): + if required: + raise ValueError("Session revocation record does not exist.") + return None + if not path.is_file() or path.stat().st_nlink != 1: + raise ValueError( + "Session revocation record must be an unlinked local regular file." + ) + if path.stat().st_size > 4096: + raise ValueError("Session revocation record exceeds the 4,096-byte limit.") + payload = json.loads(path.read_text(encoding="utf-8")) + revocation = SessionRevocation.model_validate(payload) + unsigned = revocation.model_dump(mode="json") + content_hash = unsigned.pop("content_sha256") + if _hash_json(unsigned) != content_hash: + raise ValueError("Session revocation hash does not match its contents.") + if revocation.manifest_file_sha256 != expected_sha256: + raise ValueError("Session revocation does not match the reviewed manifest.") + if revocation.session_id != expected_session_id: + raise ValueError("Session revocation does not match the active session id.") + return revocation + + +def _validated_participant( + participant: SessionParticipant, + *, + require_ignored_outbox: bool, +) -> SessionParticipant: + root = _exact_git_root(participant.repository_root) + if require_ignored_outbox: + outbox = root / OUTBOX_NAME + if outbox.is_symlink() or outbox.is_junction() or outbox.resolve() != outbox: + raise ValueError( + "Feedback outbox must be a local directory in the checkout." + ) + probe = f"{OUTBOX_NAME}/_codemesh-ignore-probe.json" + result = subprocess.run( + ["git", "check-ignore", "--no-index", "--quiet", probe], + cwd=root, + check=False, + ) + if result.returncode != 0: + raise ValueError(f"{outbox} is not ignored.") + return participant.model_copy(update={"repository_root": str(root)}) + + +def _exact_git_root(value: str | Path) -> Path: + candidate = Path(value).expanduser().resolve(strict=True) + if not candidate.is_dir(): + raise ValueError(f"Repository root is not a directory: {candidate}") + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=candidate, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise ValueError(f"Session root is not a Git checkout: {candidate}") + discovered = Path(result.stdout.strip()).resolve() + if discovered != candidate: + raise ValueError( + f"Repository root must be exact: supplied {candidate}, git reports {discovered}." + ) + return discovered + + +def _binding_matches( + participant: SessionParticipant, binding: RepositoryBinding +) -> bool: + return ( + participant.project_id == binding.project_id + and participant.checkout_id == binding.checkout_id + and os.path.normcase(os.path.realpath(participant.repository_root)) + == os.path.normcase(os.path.realpath(binding.repository_root)) + ) + + +def _utc(value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("Session timestamps must include a timezone.") + return value.astimezone(UTC) + + +def _manifest_bytes(manifest: SessionManifest) -> bytes: + return _json_bytes(manifest.model_dump(mode="json")) + + +def _json_bytes(value: dict[str, object]) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8") + + +def _hash_json(value: dict[str, object]) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _hash_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _exclusive_write(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + path.unlink() + except FileNotFoundError: + pass + raise diff --git a/agent-access/codemesh_agent_access/graph_store.py b/agent-access/codemesh_agent_access/graph_store.py index 02f5d40..87b5d0d 100644 --- a/agent-access/codemesh_agent_access/graph_store.py +++ b/agent-access/codemesh_agent_access/graph_store.py @@ -348,7 +348,16 @@ async def search_context_nodes( if _load_async_graph_database() is None: return [] + normalized_search_text = search_text.casefold() search_terms = lexical_terms(search_text) + # Scope before lexical projection so other snapshots do not pay the + # cost of loading and matching every searchable property. Keep the + # administrative, unbound query and all values parameterized. + node_match = ( + "MATCH (n:CodeMeshNode {repositoryId: $repository_id})" + if repository_id is not None + else "MATCH (n:CodeMeshNode)" + ) try: driver = self._neo4j() @@ -356,19 +365,20 @@ async def search_context_nodes( database=self._settings.neo4j_database ) as session: result = await session.run( - """ - MATCH (n:CodeMeshNode) + f""" + {node_match} WITH n, [term IN $search_terms WHERE any(value IN [ toLower(coalesce(n.id, "")), toLower(coalesce(n.stableKey, "")), toLower(coalesce(n.name, "")), - toLower(coalesce(n.filePath, "")) + toLower(coalesce(n.filePath, "")), + toLower(coalesce(n.language, "")), + toLower(coalesce(n.metadataJson, "")) ] WHERE value CONTAINS term)] AS matched_terms WHERE (size($search_terms) = 0 OR size(matched_terms) > 0) AND (size($kinds) = 0 OR toLower(coalesce(n.kind, "")) IN $kinds) - AND ($repository_id IS NULL OR n.repositoryId = $repository_id) RETURN properties(n) AS node ORDER BY size(matched_terms) DESC, CASE @@ -381,7 +391,7 @@ async def search_context_nodes( toLower(coalesce(n.name, n.id, "")) LIMIT $limit """, - search_text=search_text, + search_text=normalized_search_text, search_terms=search_terms, kinds=kinds, repository_id=repository_id, @@ -433,10 +443,11 @@ def _neo4j(self) -> Any: def lexical_terms(search_text: str) -> list[str]: + expanded = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", search_text) return list( dict.fromkeys( term - for term in re.findall(r"[a-z0-9]+", search_text.casefold()) + for term in re.findall(r"[a-z0-9]+", expanded.casefold()) if term not in _LEXICAL_STOP_WORDS ) ) diff --git a/agent-access/codemesh_agent_access/installer.py b/agent-access/codemesh_agent_access/installer.py new file mode 100644 index 0000000..b3d7a64 --- /dev/null +++ b/agent-access/codemesh_agent_access/installer.py @@ -0,0 +1,530 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import tempfile +from typing import Any + +from .binding import RepositoryBinding +from .feedback_session import FEEDBACK_PROFILES, load_active_session +from .mcp import DEFAULT_TOOL_PROFILE, tool_manifest + + +SCHEMA_VERSION_V1 = "codemesh-installation-plan-v1" +SCHEMA_VERSION = "codemesh-installation-plan-v2" +CONFIG_START = "# codemesh:config:start" +CONFIG_END = "# codemesh:config:end" +ONBOARDING_START = "" +ONBOARDING_END = "" + +_FORWARDED_ENVIRONMENT_NAMES = [ + "CODEMESH_NEO4J_URI", + "CODEMESH_NEO4J_AUTH", + "CODEMESH_NEO4J_USER", + "CODEMESH_NEO4J_PASSWORD", + "CODEMESH_NEO4J_DATABASE", + "CODEMESH_MONGO_URI", + "CODEMESH_MONGO_DATABASE", + "CODEMESH_MONGO_CONTENT_COLLECTION", + "CODEMESH_MONGO_REPOSITORIES_COLLECTION", + "CODEMESH_MONGO_INGESTION_RUNS_COLLECTION", + "CODEMESH_MONGO_NODE_SUMMARIES_COLLECTION", + "CODEMESH_QDRANT_URL", + "CODEMESH_QDRANT_COLLECTION", +] + + +@dataclass(frozen=True) +class PlannedFile: + path: str + previous_sha256: str | None + new_sha256: str + content: str + diff: str + + +@dataclass(frozen=True) +class InstallationPlan: + schema_version: str + target_root: str + codemesh_root: str + server_name: str + profile: str + binding: dict[str, str | None] + session: dict[str, str] | None + launch: dict[str, Any] + files: tuple[PlannedFile, ...] + plan_hash: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def create_installation_plan( + target_root: str | Path, + codemesh_root: str | Path, + binding: RepositoryBinding, + *, + server_name: str = "codemesh", + profile: str = DEFAULT_TOOL_PROFILE, + include_onboarding: bool = True, + feedback_session_path: str | Path | None = None, + feedback_session_sha256: str | None = None, +) -> InstallationPlan: + target = Path(target_root).expanduser().resolve(strict=True) + codemesh = Path(codemesh_root).expanduser().resolve(strict=True) + agent_access = codemesh / "agent-access" + if not (agent_access / "pyproject.toml").is_file(): + raise ValueError(f"CodeMesh Agent Access was not found under '{agent_access}'.") + if Path(binding.repository_root) != target: + raise ValueError( + "Installation target must match the configured repository binding root." + ) + if not re.fullmatch(r"[A-Za-z0-9_-]+", server_name): + raise ValueError( + "MCP server name may contain only letters, numbers, underscores, and hyphens." + ) + + session: dict[str, str] | None = None + if profile in FEEDBACK_PROFILES: + if feedback_session_path is None or feedback_session_sha256 is None: + raise ValueError( + "Feedback installation profiles require a session manifest path " + "and expected SHA-256." + ) + session_path = Path(feedback_session_path).expanduser().resolve(strict=True) + load_active_session( + session_path, + feedback_session_sha256, + profile=profile, + binding=binding, + ) + session = { + "manifest_path": str(session_path), + "manifest_file_sha256": feedback_session_sha256, + } + elif feedback_session_path is not None or feedback_session_sha256 is not None: + raise ValueError("Session manifest options require a feedback MCP profile.") + + launch = { + "command": "uv", + "args": [ + "run", + "--directory", + str(agent_access), + "python", + "-m", + "codemesh_agent_access", + "mcp", + "--profile", + profile, + "--project-id", + binding.project_id, + "--checkout-id", + binding.checkout_id, + "--repository-root", + binding.repository_root, + *( + ["--source-view-hash", binding.source_view_hash] + if binding.source_view_hash + else [] + ), + *( + [ + "--feedback-session", + session["manifest_path"], + "--feedback-session-sha256", + session["manifest_file_sha256"], + ] + if session + else [] + ), + ], + "cwd": str(agent_access), + "profile": profile, + "expected_tools": [ + item["name"] for item in tool_manifest(profile=profile)["tools"] + ], + } + config_path = target / ".codex" / "config.toml" + agents_path = target / "AGENTS.md" + config_block = _config_block(server_name, launch) + onboarding_block = _onboarding_block(server_name, profile) + planned_files = [ + _plan_file( + config_path, + _updated_managed_file( + config_path, + CONFIG_START, + CONFIG_END, + config_block, + unmanaged_conflict=f"[mcp_servers.{server_name}]", + ), + target, + ) + ] + if include_onboarding: + planned_files.append( + _plan_file( + agents_path, + _updated_managed_file( + agents_path, + ONBOARDING_START, + ONBOARDING_END, + onboarding_block, + ), + target, + ) + ) + files = tuple(planned_files) + unsigned = { + "schema_version": SCHEMA_VERSION, + "target_root": str(target), + "codemesh_root": str(codemesh), + "server_name": server_name, + "profile": profile, + "binding": binding.as_dict(), + "session": session, + "launch": launch, + "files": [asdict(item) for item in files], + } + return InstallationPlan( + schema_version=SCHEMA_VERSION, + target_root=str(target), + codemesh_root=str(codemesh), + server_name=server_name, + profile=profile, + binding=binding.as_dict(), + session=session, + launch=launch, + files=files, + plan_hash=_hash_json(unsigned), + ) + + +def load_installation_plan(path: str | Path) -> InstallationPlan: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + schema_version = payload.get("schema_version") + if schema_version not in {SCHEMA_VERSION_V1, SCHEMA_VERSION}: + raise ValueError("Unsupported CodeMesh installation-plan schema.") + files = tuple(PlannedFile(**item) for item in payload.get("files", [])) + plan = InstallationPlan( + schema_version=str(payload["schema_version"]), + target_root=str(payload["target_root"]), + codemesh_root=str(payload["codemesh_root"]), + server_name=str(payload["server_name"]), + profile=str(payload["profile"]), + binding=dict(payload["binding"]), + session=(dict(payload["session"]) if payload.get("session") else None), + launch=dict(payload["launch"]), + files=files, + plan_hash=str(payload["plan_hash"]), + ) + verify_installation_plan(plan) + return plan + + +def write_installation_plan(plan: InstallationPlan, path: str | Path) -> None: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + _atomic_write( + output, + json.dumps(plan.to_dict(), indent=2, sort_keys=True) + "\n", + ) + + +def apply_installation_plan( + plan: InstallationPlan, + approved_plan_hash: str, +) -> list[str]: + verify_installation_plan(plan) + _verify_active_plan_session(plan) + if approved_plan_hash != plan.plan_hash: + raise ValueError( + "Approved plan hash does not match the reviewed installation plan." + ) + target = Path(plan.target_root).resolve(strict=True) + pending: list[tuple[Path, PlannedFile]] = [] + for item in plan.files: + path = (target / item.path).resolve(strict=False) + if target != path and target not in path.parents: + raise ValueError(f"Installation path escapes target root: {item.path}") + current = path.read_text(encoding="utf-8") if path.exists() else "" + current_hash = _hash_text(current) if path.exists() else None + if current_hash != item.previous_sha256: + raise ValueError(f"Installation target changed after review: {item.path}.") + if _hash_text(item.content) != item.new_sha256: + raise ValueError(f"Planned content hash is invalid: {item.path}.") + pending.append((path, item)) + + changed: list[str] = [] + for path, item in pending: + current = path.read_text(encoding="utf-8") if path.exists() else "" + if current == item.content: + continue + path.parent.mkdir(parents=True, exist_ok=True) + _atomic_write(path, item.content) + changed.append(item.path) + return changed + + +def verify_plan_applied(plan: InstallationPlan) -> None: + verify_installation_plan(plan) + target = Path(plan.target_root).resolve(strict=True) + for item in plan.files: + path = (target / item.path).resolve(strict=False) + current = path.read_text(encoding="utf-8") if path.exists() else "" + if _hash_text(current) != item.new_sha256: + raise ValueError( + f"Installed file does not match the reviewed plan: {item.path}." + ) + + +def codex_mcp_overrides( + plan: InstallationPlan, + *, + approve_tools: bool = False, +) -> tuple[str, ...]: + """Render the reviewed launch as isolated Codex CLI configuration overrides.""" + verify_installation_plan(plan) + _verify_active_plan_session(plan) + launch = plan.launch + if launch.get("profile") != plan.profile: + raise ValueError("Installation launch profile does not match the plan profile.") + expected_tools = launch.get("expected_tools") + if not isinstance(expected_tools, list) or not expected_tools: + raise ValueError("Installation plan has no expected MCP tool surface.") + command = launch.get("command") + args = launch.get("args") + cwd = launch.get("cwd") + if not isinstance(command, str) or not command: + raise ValueError("Installation plan has no MCP launch command.") + if not isinstance(args, list) or not all(isinstance(value, str) for value in args): + raise ValueError("Installation plan has invalid MCP launch arguments.") + if not isinstance(cwd, str) or not cwd: + raise ValueError("Installation plan has no MCP launch working directory.") + + prefix = f"mcp_servers.{plan.server_name}" + overrides = [ + f"{prefix}.command={json.dumps(command)}", + f"{prefix}.args={json.dumps(args)}", + f"{prefix}.cwd={json.dumps(cwd)}", + f"{prefix}.required=true", + f"{prefix}.startup_timeout_sec=30", + f"{prefix}.tool_timeout_sec=60", + f"{prefix}.enabled_tools={json.dumps(expected_tools)}", + f"{prefix}.env_vars={json.dumps(_FORWARDED_ENVIRONMENT_NAMES)}", + f'{prefix}.env={{ CODEMESH_MODEL_PROVIDER = "none" }}', + ] + if approve_tools: + overrides.append(f'{prefix}.default_tools_approval_mode="approve"') + return tuple(overrides) + + +def verify_installation_plan(plan: InstallationPlan) -> None: + if plan.plan_hash != _plan_hash(plan): + raise ValueError("Installation plan hash does not match its contents.") + if plan.profile in FEEDBACK_PROFILES: + if plan.session is None: + raise ValueError("Feedback installation plan has no pinned session.") + expected = { + "--feedback-session": plan.session.get("manifest_path"), + "--feedback-session-sha256": plan.session.get("manifest_file_sha256"), + } + for flag, value in expected.items(): + if not value or flag not in plan.launch.get("args", []): + raise ValueError( + "Feedback installation launch is missing session pins." + ) + index = plan.launch["args"].index(flag) + if plan.launch["args"][index + 1] != value: + raise ValueError("Feedback installation launch session pin changed.") + elif plan.session is not None: + raise ValueError("Non-feedback installation plan contains a session block.") + if plan.launch.get("profile") != plan.profile: + raise ValueError("Installation launch profile does not match the plan profile.") + expected_tools = sorted( + item["name"] for item in tool_manifest(profile=plan.profile)["tools"] + ) + if sorted(plan.launch.get("expected_tools") or []) != expected_tools: + raise ValueError("Installation launch tool surface does not match its profile.") + + +def _verify_active_plan_session(plan: InstallationPlan) -> None: + if plan.profile not in FEEDBACK_PROFILES: + return + assert plan.session is not None + load_active_session( + plan.session["manifest_path"], + plan.session["manifest_file_sha256"], + profile=plan.profile, + binding=RepositoryBinding(**plan.binding), + ) + + +def _config_block(server_name: str, launch: dict[str, Any]) -> str: + enabled_tools = launch["expected_tools"] + env_names = ", ".join(json.dumps(name) for name in _FORWARDED_ENVIRONMENT_NAMES) + return "\n".join( + [ + CONFIG_START, + f"[mcp_servers.{server_name}]", + f"command = {json.dumps(launch['command'])}", + f"args = {_toml_array(launch['args'])}", + f"cwd = {json.dumps(launch['cwd'])}", + "required = true", + "startup_timeout_sec = 30", + "tool_timeout_sec = 60", + f"enabled_tools = {_toml_array(enabled_tools)}", + f"env_vars = [{env_names}]", + 'env = { CODEMESH_MODEL_PROVIDER = "none" }', + CONFIG_END, + ] + ) + + +def _onboarding_block(server_name: str, profile: str) -> str: + lines = [ + ONBOARDING_START, + "## CodeMesh repository context", + "", + f"The `{server_name}` MCP server is bound to this exact checkout.", + "For implementation discovery, likely change-impact analysis, and", + "validation selection:", + "", + "1. Start with one `codemesh_get_context_package` call using a concise", + " task-shaped query and default limits, with", + ' `output_format = "agent"`; omit `repository_id` so the server uses', + " its checkout binding. If the package covers every required facet,", + " stop retrieving. Only issue one focused follow-up when a specific", + " implementation or test facet remains unresolved. Use at most three", + " packages for the task and avoid overlapping queries. The package", + " validates the exact", + " binding and freshness before search; it fails closed when they are not accepted.", + "2. If the package rejects the binding or freshness, stop using CodeMesh", + " and fall back to direct source inspection. Use", + " `codemesh_get_repository_status` only for explicit diagnostics.", + "3. Verify consequential findings against returned source spans with", + " targeted reads. Relationships are navigation evidence, not runtime proof.", + "4. Record missing, stale, incorrect, or unhelpful results through the", + " repository's CodeMesh feedback workflow when it is available.", + ] + if profile == "development-feedback": + lines.extend( + [ + "5. This checkout has an explicitly enabled local feedback session.", + " Inspect it with `codemesh_get_feedback_session`. After targeted", + " source verification, use `codemesh_record_feedback` for a", + " sanitized missing, stale, incorrect, or unhelpful result.", + " Never include raw prompts, MCP payloads, source excerpts, secrets,", + " credentials, or environment values. Human review remains required.", + ] + ) + elif profile == "feedback-maintainer": + lines.extend( + [ + "5. Start feedback review with `codemesh_list_feedback`, investigate", + " selected packets read-only, and prepare an exact proposal with", + " `codemesh_prepare_feedback_resolution`. Do not edit CodeMesh until", + " a human approves the exact feedback ids and returned plan hash.", + ] + ) + lines.append(ONBOARDING_END) + return "\n".join(lines) + + +def _updated_managed_file( + path: Path, + start: str, + end: str, + block: str, + *, + unmanaged_conflict: str | None = None, +) -> str: + current = path.read_text(encoding="utf-8") if path.exists() else "" + start_count = current.count(start) + end_count = current.count(end) + if start_count != end_count or start_count > 1: + raise ValueError(f"Managed CodeMesh markers are malformed in '{path}'.") + if start_count == 0: + if unmanaged_conflict and unmanaged_conflict in current: + raise ValueError( + f"Unmanaged CodeMesh MCP configuration already exists in '{path}'." + ) + prefix = current.rstrip() + return f"{prefix}\n\n{block}\n" if prefix else f"{block}\n" + + before, remainder = current.split(start, 1) + _old, after = remainder.split(end, 1) + return f"{before}{block}{after}".rstrip() + "\n" + + +def _plan_file(path: Path, content: str, target: Path) -> PlannedFile: + previous = path.read_text(encoding="utf-8") if path.exists() else "" + relative = path.relative_to(target).as_posix() + diff = "".join( + difflib.unified_diff( + previous.splitlines(keepends=True), + content.splitlines(keepends=True), + fromfile=f"a/{relative}", + tofile=f"b/{relative}", + ) + ) + return PlannedFile( + path=relative, + previous_sha256=_hash_text(previous) if path.exists() else None, + new_sha256=_hash_text(content), + content=content, + diff=diff, + ) + + +def _plan_hash(plan: InstallationPlan) -> str: + payload = plan.to_dict() + payload.pop("plan_hash", None) + if plan.schema_version == SCHEMA_VERSION_V1: + payload.pop("session", None) + return _hash_json(payload) + + +def _hash_json(value: dict[str, Any]) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _hash_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _toml_array(values: list[str]) -> str: + return "[" + ", ".join(json.dumps(value) for value in values) + "]" + + +def _atomic_write(path: Path, content: str) -> None: + handle, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + text=True, + ) + try: + with os.fdopen(handle, "w", encoding="utf-8", newline="\n") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_name, path) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise diff --git a/agent-access/codemesh_agent_access/mcp.py b/agent-access/codemesh_agent_access/mcp.py index 1868904..fea4676 100644 --- a/agent-access/codemesh_agent_access/mcp.py +++ b/agent-access/codemesh_agent_access/mcp.py @@ -1,13 +1,26 @@ from __future__ import annotations import asyncio +from collections import deque from copy import deepcopy +from functools import wraps import json +from pathlib import Path +from time import monotonic from typing import Any, Callable, Literal -from mcp.types import CallToolResult, TextContent +from mcp.types import CallToolResult, TextContent, ToolAnnotations -from . import tools +from . import feedback, tools +from .binding import RepositoryBinding, bound_repository_reference +from .feedback_session import ( + CLIENT_PROFILE, + CLIENT_TOOLS, + FEEDBACK_PROFILES, + MAINTAINER_PROFILE, + MAINTAINER_TOOLS, + SessionRuntime, +) from .formatting import format_context_package_for_agent @@ -35,9 +48,11 @@ "codemesh_get_node", ] -_TOOL_PROFILES = { +_TOOL_PROFILES: dict[str, list[str]] = { DEFAULT_TOOL_PROFILE: _NORMAL_TOOL_NAMES, - "diagnostic": _TOOL_NAMES, + "diagnostic": list(_TOOL_NAMES), + CLIENT_PROFILE: list(CLIENT_TOOLS), + MAINTAINER_PROFILE: list(MAINTAINER_TOOLS), } _MAX_MCP_AGENT_CHARACTERS = 12000 @@ -45,14 +60,56 @@ _SERVER_INSTRUCTIONS = ( "Use CodeMesh first for source-grounded implementation discovery, likely " "change-impact analysis, and validation selection in indexed repositories. " - "Start with codemesh_get_context_package using the full task, the repository " - "id or alias, output_format='agent', and default limits. Use the returned " - "ranked file spans for targeted, batched checkout reads needed to verify " - "citations instead of repeating broad repository exploration. If the " - "repository is unknown, call codemesh_list_repositories; before trusting " - "context, call codemesh_get_repository_status to confirm freshness." + "Start with one codemesh_get_context_package call using a concise task-shaped " + "query, the repository id or alias, output_format='agent', and default limits. " + "If that package covers every required facet, stop retrieving. Only when it " + "leaves a specific implementation or test facet unresolved, issue one focused " + "follow-up for that facet. Use at most three context packages for a task and " + "avoid overlapping queries. Use the returned ranked " + "file spans for targeted, batched checkout reads needed to verify citations " + "instead of repeating broad repository exploration. The package " + "checks the exact binding and freshness before search and fails closed when " + "they are not accepted. If the repository is unknown, call " + "codemesh_list_repositories. Use codemesh_get_repository_status only for " + "explicit ingestion, count, or freshness diagnostics." +) + +_CLIENT_FEEDBACK_INSTRUCTIONS = ( + " This is an explicitly enabled local development-feedback session. Use " + "codemesh_get_feedback_session to inspect its bounds. When targeted source " + "verification finds missing, stale, incorrect, or unhelpful CodeMesh output, " + "use codemesh_record_feedback with a sanitized observation. Do not submit raw " + "prompts, MCP payloads, source excerpts, credentials, secrets, or environment " + "values. Recording feedback does not approve or schedule a CodeMesh change." ) +_MAINTAINER_FEEDBACK_INSTRUCTIONS = ( + " This is an explicitly enabled local feedback-maintainer session. Start with " + "codemesh_list_feedback, investigate selected packets read-only, and use " + "codemesh_prepare_feedback_resolution to create a stable proposal hash. Do not " + "edit CodeMesh until a human approves the exact feedback ids and plan hash." +) + +_READ_ONLY_ANNOTATIONS = ToolAnnotations( + readOnlyHint=True, + destructiveHint=False, + idempotentHint=True, + openWorldHint=False, +) +_RECORD_ANNOTATIONS = ToolAnnotations( + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=False, +) +_STRICT_ARGUMENT_TOOLS = { + "codemesh_get_feedback_session", + "codemesh_record_feedback", + "codemesh_list_feedback", + "codemesh_get_feedback", + "codemesh_prepare_feedback_resolution", +} + _SEARCH_CONTEXT_ARGUMENTS: dict[str, dict[str, Any]] = { "query": {"type": "string", "required": True}, "repository_id": { @@ -154,9 +211,9 @@ _GET_REPOSITORY_STATUS_ARGUMENTS: dict[str, dict[str, Any]] = { "repository_id": { - "type": "string", - "required": True, - "description": "Repository hash id or alias.", + "type": "string|null", + "default": None, + "description": "Repository hash id or alias; omit when the server is checkout-bound.", }, } @@ -177,6 +234,55 @@ "limit": {"type": "integer", "default": 50}, } +_RECORD_FEEDBACK_ARGUMENTS: dict[str, dict[str, Any]] = { + "confidence": { + "type": "string", + "required": True, + "values": ["low", "medium", "high"], + }, + "task_family": {"type": "string", "required": True}, + "issue_category": {"type": "string", "required": True}, + "validation_outcome": {"type": "string", "required": True}, + "helpful_paths": {"type": "string[]|null", "default": None}, + "incorrect_paths": {"type": "string[]|null", "default": None}, + "missed_paths": {"type": "string[]|null", "default": None}, + "stale_paths": {"type": "string[]|null", "default": None}, + "ambiguous_paths": {"type": "string[]|null", "default": None}, + "fallback_reason": {"type": "string|null", "default": None}, + "minimal_reproduction": {"type": "string|null", "default": None}, + "expected_targets": {"type": "string[]|null", "default": None}, + "proposed_correction": {"type": "string|null", "default": None}, + "supersedes_feedback_ids": {"type": "string[]|null", "default": None}, +} + +_LIST_FEEDBACK_ARGUMENTS: dict[str, dict[str, Any]] = { + "include_legacy": {"type": "boolean", "default": False}, + "state": {"type": "string|null", "default": None}, + "category": {"type": "string|null", "default": None}, + "task_family": {"type": "string|null", "default": None}, + "language": {"type": "string|null", "default": None}, + "reporter_role": {"type": "string|null", "default": None}, + "feedback_id": {"type": "string|null", "default": None}, + "offset": {"type": "integer", "default": 0}, + "limit": {"type": "integer", "default": 50}, +} + +_GET_FEEDBACK_ARGUMENTS: dict[str, dict[str, Any]] = { + "feedback_id": {"type": "string", "required": True}, + "include_legacy": {"type": "boolean", "default": False}, +} + +_PREPARE_RESOLUTION_ARGUMENTS: dict[str, dict[str, Any]] = { + "feedback_ids": {"type": "string[]", "required": True}, + "reproduction_state": {"type": "string", "required": True}, + "change_summary": {"type": "string", "required": True}, + "proposed_paths": {"type": "string[]", "required": True}, + "risks": {"type": "string[]|null", "default": None}, + "required_tests": {"type": "string[]|null", "default": None}, + "unresolved_questions": {"type": "string[]|null", "default": None}, + "include_legacy": {"type": "boolean", "default": False}, +} + _RELATIONSHIP_GROUPS = [ "callers", "callees", @@ -226,7 +332,7 @@ }, }, "codemesh_get_context_package": { - "description": "Start here for implementation discovery, likely change-impact analysis, or validation selection in an indexed repository. Returns ranked snippets, graph relationships, freshness metadata, and repository-scoped validation recommendations. Agent format is capped at 12,000 characters; start with defaults, then verify returned file spans with targeted reads instead of duplicating broad repository exploration.", + "description": "Start here for implementation discovery, likely change-impact analysis, or validation selection in an indexed repository. Begin with one concise task-shaped query at default limits. If the package covers every required facet, stop retrieving; only issue one focused follow-up when a specific implementation or test facet remains unresolved. Use at most three context packages per task and avoid overlapping queries. Verifies the exact binding and freshness and fails closed before search. Returns ranked snippets, graph relationships, freshness metadata, and repository-scoped validation recommendations. Agent format is capped at 12,000 characters; verify returned file spans with targeted reads instead of duplicating broad repository exploration.", "arguments": _CONTEXT_PACKAGE_ARGUMENTS, "response": { "relationship_groups": list(_RELATIONSHIP_GROUPS), @@ -265,7 +371,7 @@ "arguments": _GET_REPOSITORY_ARGUMENTS, }, "codemesh_get_repository_status": { - "description": "Use before trusting context to check counts, latest ingestion run, summary coverage, and local git freshness.", + "description": "Use for explicit counts, latest-ingestion, summary-coverage, or local-freshness diagnostics; it is not a required first step before context-package retrieval. If called, follow codemesh_next_action in the result: a fresh accepted binding continues to codemesh_get_context_package; any other state stops CodeMesh use.", "arguments": _GET_REPOSITORY_STATUS_ARGUMENTS, "response": { "fields": [ @@ -276,6 +382,7 @@ "counts", "freshness", "refresh", + "codemesh_next_action", "generated_at", ], }, @@ -302,6 +409,26 @@ "description": "List CodeMesh ingestion runs, optionally scoped to a repository id or alias.", "arguments": _LIST_RUNS_ARGUMENTS, }, + "codemesh_get_feedback_session": { + "description": "Inspect the active local development-feedback session, bounds, expiry, and agent safety instructions.", + "arguments": {}, + }, + "codemesh_record_feedback": { + "description": "Record one sanitized, append-only local feedback packet with server-derived repository, snapshot, binding, session, and CodeMesh provenance. Human review remains required.", + "arguments": _RECORD_FEEDBACK_ARGUMENTS, + }, + "codemesh_list_feedback": { + "description": "List validated feedback summaries only from client outboxes allowlisted by the active local development session.", + "arguments": _LIST_FEEDBACK_ARGUMENTS, + }, + "codemesh_get_feedback": { + "description": "Get one validated feedback packet by exact id from an active-session allowlisted client outbox.", + "arguments": _GET_FEEDBACK_ARGUMENTS, + }, + "codemesh_prepare_feedback_resolution": { + "description": "Validate and hash a read-only proposed CodeMesh resolution for exact feedback ids; this never records approval or edits source.", + "arguments": _PREPARE_RESOLUTION_ARGUMENTS, + }, } @@ -319,6 +446,14 @@ def _tool_names_for_profile(profile: str) -> list[str]: ) from exc +def _tool_annotations(name: str) -> ToolAnnotations: + return ( + _RECORD_ANNOTATIONS + if name == "codemesh_record_feedback" + else _READ_ONLY_ANNOTATIONS + ) + + def _filter_tool_guidance( payload: dict[str, Any], tool_names: set[str] ) -> dict[str, Any]: @@ -340,25 +475,57 @@ def _filter_tool_guidance( return payload -def tool_manifest(profile: str = DEFAULT_TOOL_PROFILE) -> dict[str, Any]: +def _server_instructions( + binding: RepositoryBinding | None, + profile: str = DEFAULT_TOOL_PROFILE, +) -> str: + instructions = _SERVER_INSTRUCTIONS + if profile == CLIENT_PROFILE: + instructions += _CLIENT_FEEDBACK_INSTRUCTIONS + elif profile == MAINTAINER_PROFILE: + instructions += _MAINTAINER_FEEDBACK_INSTRUCTIONS + if binding is None: + return instructions + return ( + f"{instructions} This server is bound to project " + f"'{binding.project_id}', checkout '{binding.checkout_id}', and repository " + f"root '{binding.repository_root}'. Repository-scoped calls default to this " + "binding and cannot override it." + ) + + +def tool_manifest( + profile: str = DEFAULT_TOOL_PROFILE, + binding: RepositoryBinding | None = None, +) -> dict[str, Any]: """Return the stable MCP tool contract used by tests and fallback output.""" tool_names = _tool_names_for_profile(profile) return { "service": "codemesh-agent-access", "protocol": "mcp", "profile": profile, - "instructions": _SERVER_INSTRUCTIONS, + "instructions": _server_instructions(binding, profile), "tools": [ - {"name": name, "description": _TOOL_CONTRACTS[name]["description"]} + { + "name": name, + "description": _TOOL_CONTRACTS[name]["description"], + "annotations": _tool_annotations(name).model_dump( + mode="json", exclude_none=True + ), + } for name in tool_names ], "contracts": {name: deepcopy(_TOOL_CONTRACTS[name]) for name in tool_names}, + "binding": binding.as_dict() if binding is not None else None, } def build_server( profile: str = DEFAULT_TOOL_PROFILE, call_guard: Callable[[str, dict[str, Any]], None] | None = None, + binding: RepositoryBinding | None = None, + feedback_runtime: SessionRuntime | None = None, + monotonic_clock: Callable[[], float] = monotonic, ) -> Any: try: from mcp.server.fastmcp import FastMCP @@ -368,15 +535,84 @@ def build_server( ) from exc tool_names = set(_tool_names_for_profile(profile)) - server = FastMCP("codemesh-agent-access", instructions=_SERVER_INSTRUCTIONS) + if profile in FEEDBACK_PROFILES: + if binding is None or feedback_runtime is None: + raise ValueError( + "Development feedback MCP profiles require an explicit binding and " + "validated session runtime." + ) + if feedback_runtime.profile != profile or feedback_runtime.binding != binding: + raise ValueError( + "Development feedback runtime does not match the MCP profile." + ) + feedback_runtime.assert_active() + elif feedback_runtime is not None: + raise ValueError("Feedback session runtime is not valid for this MCP profile.") + server = FastMCP( + "codemesh-agent-access", + instructions=_server_instructions(binding, profile), + ) + + recent_tool_events: deque[dict[str, Any]] = deque(maxlen=32) + request_times: deque[float] = deque() + trace_sequence = 0 + latest_accepted_provenance: dict[str, Any] | None = None def register_tool(name: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: def register(function: Callable[..., Any]) -> Callable[..., Any]: if name not in tool_names: return function - return server.tool(description=_TOOL_CONTRACTS[name]["description"])( - function - ) + + @wraps(function) + async def traced(*args: Any, **kwargs: Any) -> Any: + nonlocal latest_accepted_provenance, trace_sequence + try: + result = await function(*args, **kwargs) + except BaseException: + if name in _NORMAL_TOOL_NAMES: + trace_sequence += 1 + recent_tool_events.append( + { + "sequence": trace_sequence, + "tool": name, + "outcome": "failed", + "diagnostic_code": "tool_error", + } + ) + raise + if name in _NORMAL_TOOL_NAMES: + trace_sequence += 1 + failed = bool(getattr(result, "isError", False)) + if isinstance(result, dict) and result.get("ok") is False: + failed = True + recent_tool_events.append( + { + "sequence": trace_sequence, + "tool": name, + "outcome": "failed" if failed else "succeeded", + "diagnostic_code": "tool_error" if failed else None, + } + ) + if not failed: + derived = _feedback_provenance_from_result(name, result) + if derived is not None: + latest_accepted_provenance = derived + return result + + registered = server.tool( + description=_TOOL_CONTRACTS[name]["description"], + annotations=_tool_annotations(name), + )(traced) + if name in _STRICT_ARGUMENT_TOOLS: + registered_tool = server._tool_manager.get_tool(name) + assert registered_tool is not None + argument_model = registered_tool.fn_metadata.arg_model + argument_model.model_config["extra"] = "forbid" + argument_model.model_rebuild(force=True) + registered_tool.parameters = argument_model.model_json_schema( + by_alias=True + ) + return registered return register @@ -384,6 +620,29 @@ def check(name: str, arguments: dict[str, Any]) -> None: if call_guard is not None: call_guard(name, arguments) + def check_feedback_rate() -> None: + assert feedback_runtime is not None + bounds = feedback_runtime.assert_active().bounds + current = monotonic_clock() + while request_times and current - request_times[0] >= 60: + request_times.popleft() + if len(request_times) >= bounds.max_requests_per_minute: + raise ValueError( + "Development feedback request-rate limit has been reached." + ) + request_times.append(current) + + def repository_reference( + repository_id: str | None, + *, + required_when_unbound: bool = False, + ) -> str | None: + return bound_repository_reference( + binding, + repository_id, + required_when_unbound=required_when_unbound, + ) + @register_tool("codemesh_status") async def codemesh_status() -> dict[str, Any]: """Return CodeMesh Agent Access health and configured read endpoints.""" @@ -411,6 +670,7 @@ async def codemesh_search_context( context_relationship_limit: int = 8, ) -> dict[str, Any]: """Search CodeMesh context for code relevant to an agent task.""" + repository_id = repository_reference(repository_id) check( "codemesh_search_context", {"query": query, "repository_id": repository_id, "limit": limit}, @@ -454,6 +714,11 @@ async def codemesh_get_context_package( output_format: Literal["json", "agent"] = "json", ) -> CallToolResult: """Build an agent-ready context package with snippets, relationships, and repository metadata.""" + nonlocal latest_accepted_provenance + repository_id = repository_reference( + repository_id, + required_when_unbound=profile == DEFAULT_TOOL_PROFILE, + ) check( "codemesh_get_context_package", {"query": query, "repository_id": repository_id, "limit": limit}, @@ -478,7 +743,15 @@ async def codemesh_get_context_package( definition_limit=definition_limit, uses_type_limit=uses_type_limit, include_repository=include_repository, + binding=binding, ) + if result.ok and isinstance(result.data, dict): + derived = _feedback_provenance_from_data( + result.data, + accepted=True, + ) + if derived is not None: + latest_accepted_provenance = derived if output_format == "agent": text = ( format_context_package_for_agent( @@ -510,6 +783,7 @@ async def codemesh_search_symbols( include_declarations: bool = True, ) -> dict[str, Any]: """Search indexed CodeMesh symbols by name, id, display signature, namespace, or file path.""" + repository_id = repository_reference(repository_id) check( "codemesh_search_symbols", {"query": query, "repository_id": repository_id, "limit": limit}, @@ -533,6 +807,10 @@ async def codemesh_get_node( include_neighbors: bool = True, ) -> dict[str, Any]: """Fetch a CodeMesh node with optional content and immediate relationships.""" + repository_id = repository_reference( + repository_id, + required_when_unbound=profile == DEFAULT_TOOL_PROFILE, + ) check( "codemesh_get_node", {"node_id": node_id, "repository_id": repository_id}, @@ -543,6 +821,7 @@ async def codemesh_get_node( repository_id=repository_id, include_content=include_content, include_neighbors=include_neighbors, + binding=binding, ) ).model_dump(mode="json") @@ -554,6 +833,7 @@ async def codemesh_get_neighbors( relationship_kinds: list[str] | None = None, ) -> dict[str, Any]: """Fetch relationships around a CodeMesh node.""" + repository_id = repository_reference(repository_id) check( "codemesh_get_neighbors", {"node_id": node_id, "repository_id": repository_id, "depth": depth}, @@ -571,25 +851,48 @@ async def codemesh_get_neighbors( async def codemesh_list_repositories(limit: int = 50) -> dict[str, Any]: """List repositories indexed by CodeMesh.""" check("codemesh_list_repositories", {"limit": limit}) - return (await tools.list_repositories(limit=limit)).model_dump(mode="json") + return (await tools.list_repositories(limit=limit, binding=binding)).model_dump( + mode="json" + ) @register_tool("codemesh_get_repository") async def codemesh_get_repository(repository_id: str) -> dict[str, Any]: """Fetch one indexed CodeMesh repository by id.""" + repository_id = repository_reference( + repository_id, + required_when_unbound=True, + ) + assert repository_id is not None check("codemesh_get_repository", {"repository_id": repository_id}) return (await tools.get_repository(repository_id)).model_dump(mode="json") @register_tool("codemesh_get_repository_status") - async def codemesh_get_repository_status(repository_id: str) -> dict[str, Any]: + async def codemesh_get_repository_status( + repository_id: str | None = None, + ) -> dict[str, Any]: """Fetch repository status, counts, latest run, summary coverage, and freshness diagnostics.""" - check("codemesh_get_repository_status", {"repository_id": repository_id}) - return (await tools.get_repository_status(repository_id)).model_dump( - mode="json" + repository_id = repository_reference( + repository_id, + required_when_unbound=True, ) + assert repository_id is not None + check("codemesh_get_repository_status", {"repository_id": repository_id}) + payload = ( + await tools.get_repository_status(repository_id, binding=binding) + ).model_dump(mode="json") + status = payload.get("data") + status = status if isinstance(status, dict) else {} + status["codemesh_next_action"] = _repository_status_next_action(status) + return payload @register_tool("codemesh_get_summary_coverage") async def codemesh_get_summary_coverage(repository_id: str) -> dict[str, Any]: """Fetch generated node summary coverage for one indexed CodeMesh repository.""" + repository_id = repository_reference( + repository_id, + required_when_unbound=True, + ) + assert repository_id is not None check("codemesh_get_summary_coverage", {"repository_id": repository_id}) return (await tools.get_summary_coverage(repository_id)).model_dump(mode="json") @@ -599,6 +902,7 @@ async def codemesh_list_ingestion_runs( limit: int = 50, ) -> dict[str, Any]: """List CodeMesh ingestion runs, optionally scoped to a repository.""" + repository_id = repository_reference(repository_id) check( "codemesh_list_ingestion_runs", {"repository_id": repository_id, "limit": limit}, @@ -607,11 +911,326 @@ async def codemesh_list_ingestion_runs( await tools.list_ingestion_runs(repository_id=repository_id, limit=limit) ).model_dump(mode="json") + @register_tool("codemesh_get_feedback_session") + async def codemesh_get_feedback_session() -> dict[str, Any]: + """Inspect the active local development-feedback session and its bounds.""" + check("codemesh_get_feedback_session", {}) + check_feedback_rate() + assert feedback_runtime is not None + payload = feedback_runtime.assert_active() + return { + "schema_version": "codemesh-feedback-session-discovery/v1", + "session_id": payload.session_id, + "enabled": True, + "expires_at": payload.expires_at.isoformat(), + "local_only": payload.local_only, + "provider_free": payload.provider_free, + "profile": profile, + "allowed_categories": [ + "setup", + "binding", + "freshness", + "missing-language", + "missing-relationship", + "ranking", + "budget", + "validation", + "other", + ], + "bounds": payload.bounds.model_dump(mode="json"), + "instructions": ( + "Record only sanitized observations after targeted source verification. " + "Do not include raw prompts, MCP payloads, source excerpts, secrets, " + "credentials, or environment values. Human review is required." + ), + } + + @register_tool("codemesh_record_feedback") + async def codemesh_record_feedback( + confidence: Literal["low", "medium", "high"], + task_family: str, + issue_category: Literal[ + "setup", + "binding", + "freshness", + "missing-language", + "missing-relationship", + "ranking", + "budget", + "validation", + "other", + ], + validation_outcome: Literal["passed", "failed", "not-run", "inconclusive"], + helpful_paths: list[str] | None = None, + incorrect_paths: list[str] | None = None, + missed_paths: list[str] | None = None, + stale_paths: list[str] | None = None, + ambiguous_paths: list[str] | None = None, + fallback_reason: str | None = None, + minimal_reproduction: str | None = None, + expected_targets: list[str] | None = None, + proposed_correction: str | None = None, + supersedes_feedback_ids: list[str] | None = None, + ) -> dict[str, Any]: + """Record one sanitized local feedback packet with server-derived provenance.""" + arguments = { + "confidence": confidence, + "task_family": task_family, + "issue_category": issue_category, + "validation_outcome": validation_outcome, + } + check("codemesh_record_feedback", arguments) + check_feedback_rate() + assert feedback_runtime is not None and binding is not None + if issue_category in {"setup", "binding", "freshness"}: + provenance = { + "status": "unavailable", + "diagnostic_code": f"{issue_category}_reported_before_context_acceptance", + } + else: + provenance = latest_accepted_provenance or { + "status": "unavailable", + "diagnostic_code": "no_accepted_context_provenance", + } + packet, output = feedback.record_development_feedback( + runtime=feedback_runtime, + confidence=confidence, + task_family=task_family, + issue_category=issue_category, + validation_outcome=validation_outcome, + provenance=provenance, + recent_tool_events=list(recent_tool_events), + helpful_paths=helpful_paths, + incorrect_paths=incorrect_paths, + missed_paths=missed_paths, + stale_paths=stale_paths, + ambiguous_paths=ambiguous_paths, + fallback_reason=fallback_reason, + minimal_reproduction=minimal_reproduction, + expected_targets=expected_targets, + proposed_correction=proposed_correction, + supersedes_feedback_ids=supersedes_feedback_ids, + ) + return { + "recorded": True, + "feedback_id": packet.feedback_id, + "packet_path": output.relative_to(Path(binding.repository_root)).as_posix(), + "local_only": True, + "human_review_required": True, + } + + @register_tool("codemesh_list_feedback") + async def codemesh_list_feedback( + include_legacy: bool = False, + state: Literal["open", "resolved", "verified"] | None = None, + category: Literal[ + "setup", + "binding", + "freshness", + "missing-language", + "missing-relationship", + "ranking", + "budget", + "validation", + "other", + ] + | None = None, + task_family: str | None = None, + language: str | None = None, + reporter_role: str | None = None, + feedback_id: str | None = None, + offset: int = 0, + limit: int = 50, + ) -> dict[str, Any]: + """List validated feedback from active-session allowlisted outboxes.""" + check("codemesh_list_feedback", {"offset": offset, "limit": limit}) + check_feedback_rate() + assert feedback_runtime is not None + return feedback.list_session_feedback( + feedback_runtime, + include_legacy=include_legacy, + state=state, + category=category, + task_family=task_family, + language=language, + reporter_role=reporter_role, + feedback_id=feedback_id, + offset=offset, + limit=limit, + ) + + @register_tool("codemesh_get_feedback") + async def codemesh_get_feedback( + feedback_id: str, + include_legacy: bool = False, + ) -> dict[str, Any]: + """Get one exact validated packet from an allowlisted client outbox.""" + check("codemesh_get_feedback", {"feedback_id": feedback_id}) + check_feedback_rate() + assert feedback_runtime is not None + return feedback.get_session_feedback( + feedback_runtime, + feedback_id, + include_legacy=include_legacy, + ) + + @register_tool("codemesh_prepare_feedback_resolution") + async def codemesh_prepare_feedback_resolution( + feedback_ids: list[str], + reproduction_state: Literal["reproduced", "not-reproduced", "blocked"], + change_summary: str, + proposed_paths: list[str], + risks: list[str] | None = None, + required_tests: list[str] | None = None, + unresolved_questions: list[str] | None = None, + include_legacy: bool = False, + ) -> dict[str, Any]: + """Validate and hash a non-authorizing proposed feedback resolution.""" + check( + "codemesh_prepare_feedback_resolution", + {"feedback_ids": feedback_ids, "reproduction_state": reproduction_state}, + ) + check_feedback_rate() + assert feedback_runtime is not None + return feedback.prepare_feedback_resolution( + feedback_runtime, + feedback_ids=feedback_ids, + reproduction_state=reproduction_state, + change_summary=change_summary, + proposed_paths=proposed_paths, + risks=risks, + required_tests=required_tests, + unresolved_questions=unresolved_questions, + include_legacy=include_legacy, + ) + return server -def run(profile: str = DEFAULT_TOOL_PROFILE) -> None: - server = build_server(profile=profile) +def _feedback_provenance_from_result( + tool_name: str, + result: Any, +) -> dict[str, Any] | None: + if tool_name not in { + "codemesh_get_repository_status", + "codemesh_get_context_package", + }: + return None + if isinstance(result, dict): + payload = result + else: + payload = getattr(result, "structuredContent", None) + if not isinstance(payload, dict) or payload.get("ok") is False: + return None + data = payload.get("data", payload) + if not isinstance(data, dict): + return None + return _feedback_provenance_from_data( + data, + accepted=( + tool_name == "codemesh_get_context_package" + or _status_payload_is_accepted(data) + ), + ) + + +def _status_payload_is_accepted(data: dict[str, Any]) -> bool: + freshness = data.get("freshness") + freshness = freshness if isinstance(freshness, dict) else {} + assessment = freshness.get("binding") + assessment = assessment if isinstance(assessment, dict) else {} + return ( + freshness.get("status") == "fresh" + and freshness.get("is_stale") is False + and assessment.get("status") == "accepted" + ) + + +def _feedback_provenance_from_data( + data: dict[str, Any], + *, + accepted: bool, +) -> dict[str, Any] | None: + repository = data.get("repository") + repository = repository if isinstance(repository, dict) else {} + latest_run = data.get("latest_run") + latest_run = latest_run if isinstance(latest_run, dict) else {} + snapshot_id = repository.get("snapshot_id") + language = latest_run.get("language") + languages = ( + sorted( + { + item.strip().casefold() + for item in str(language or "").split(",") + if item.strip() + } + ) + if language + else [] + ) + parser_profile = latest_run.get("parser_name") + if not accepted or not snapshot_id or not languages or not parser_profile: + return None + return { + "status": "accepted", + "snapshot_id": snapshot_id, + "languages": languages, + "parser_profile": parser_profile, + } + + +def _repository_status_next_action(payload: dict[str, Any]) -> dict[str, Any]: + freshness = payload.get("freshness") + freshness = freshness if isinstance(freshness, dict) else {} + binding = freshness.get("binding") + binding = binding if isinstance(binding, dict) else {} + eligible = ( + freshness.get("status") == "fresh" and binding.get("status") == "accepted" + ) + if eligible: + return { + "use_codemesh": True, + "next_tool": "codemesh_get_context_package", + "instruction": ( + "Binding is accepted and the index is fresh. Continue now with " + "one codemesh_get_context_package call using a concise task-shaped " + "query, output_format='agent', and default limits. Stop retrieving " + "when it covers every required facet. Only issue a focused " + "follow-up for a specific unresolved implementation or test facet; " + "use at most three packages and avoid overlapping queries." + ), + } + + return { + "use_codemesh": False, + "next_tool": None, + "instruction": ( + "Do not use CodeMesh context for this task because the exact binding " + "is not both accepted and fresh. Fall back to direct source inspection." + ), + } + + +def run( + profile: str = DEFAULT_TOOL_PROFILE, + binding: RepositoryBinding | None = None, + feedback_runtime: SessionRuntime | None = None, +) -> None: + if profile == DEFAULT_TOOL_PROFILE and binding is None: + raise ValueError( + "The normal MCP profile requires an explicit project, checkout, and " + "repository-root binding." + ) + if profile in FEEDBACK_PROFILES and binding is None: + raise ValueError( + "Development feedback MCP profiles require an explicit project, " + "checkout, and repository-root binding." + ) + server = build_server( + profile=profile, + binding=binding, + feedback_runtime=feedback_runtime, + ) server.run() diff --git a/agent-access/codemesh_agent_access/probe.py b/agent-access/codemesh_agent_access/probe.py new file mode 100644 index 0000000..4b8e9a3 --- /dev/null +++ b/agent-access/codemesh_agent_access/probe.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import asyncio +from datetime import timedelta +import json +import os +from pathlib import Path +from time import perf_counter +from typing import Any + +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + +from .feedback_session import CLIENT_PROFILE, FEEDBACK_PROFILES, inspect_session +from .installer import InstallationPlan, verify_plan_applied + + +class RuntimeProbeError(RuntimeError): + """Raised when the configured MCP process fails a required live gate.""" + + +async def run_feedback_runtime_probe( + plan: InstallationPlan, + *, + timeout_seconds: int = 30, +) -> dict[str, Any]: + verify_plan_applied(plan) + if plan.profile not in FEEDBACK_PROFILES or plan.session is None: + raise RuntimeProbeError( + "The feedback runtime probe requires a pinned feedback installation plan." + ) + manifest_path = plan.session["manifest_path"] + manifest_hash = plan.session["manifest_file_sha256"] + session_status = inspect_session(manifest_path, manifest_hash) + if session_status["status"] != "active": + raise RuntimeProbeError( + "The reviewed development feedback session is not active." + ) + launch = plan.launch + expected_tools = sorted(str(value) for value in launch["expected_tools"]) + _require_launch_value(launch["args"], "--feedback-session", manifest_path) + _require_launch_value( + launch["args"], + "--feedback-session-sha256", + manifest_hash, + ) + parameters = StdioServerParameters( + command=str(launch["command"]), + args=[str(value) for value in launch["args"]], + cwd=Path(str(launch["cwd"])), + env={**os.environ, "CODEMESH_MODEL_PROVIDER": "none"}, + ) + timeout = timedelta(seconds=timeout_seconds) + started = perf_counter() + try: + async with asyncio.timeout(timeout_seconds): + async with stdio_client(parameters) as (read_stream, write_stream): + async with ClientSession(read_stream, write_stream) as session: + initialized = await session.initialize() + instructions = initialized.instructions or "" + _require_instruction_binding(instructions, plan.binding) + if "explicitly enabled local" not in instructions: + raise RuntimeProbeError( + "Feedback MCP instructions do not advertise the active session." + ) + listed = await session.list_tools() + actual_tools = sorted(tool.name for tool in listed.tools) + if actual_tools != expected_tools: + raise RuntimeProbeError( + "Feedback MCP tool surface differs from the reviewed plan." + ) + if plan.profile == CLIENT_PROFILE: + payload = _tool_payload( + await session.call_tool( + "codemesh_get_feedback_session", + arguments={}, + read_timeout_seconds=timeout, + ) + ) + if ( + payload.get("enabled") is not True + or payload.get("session_id") != session_status["session_id"] + or payload.get("local_only") is not True + ): + raise RuntimeProbeError( + "Feedback session discovery differs from the reviewed session." + ) + else: + payload = _tool_payload( + await session.call_tool( + "codemesh_list_feedback", + arguments={"limit": 1}, + read_timeout_seconds=timeout, + ) + ) + if payload.get("session_id") != session_status["session_id"]: + raise RuntimeProbeError( + "Maintainer feedback intake differs from the reviewed session." + ) + except RuntimeProbeError: + raise + except TimeoutError as exc: + raise RuntimeProbeError( + f"CodeMesh feedback runtime probe exceeded {timeout_seconds} seconds." + ) from exc + except Exception as exc: + nested = _nested_probe_error(exc) + if nested is not None: + raise nested from exc + raise RuntimeProbeError( + "Could not start or verify the configured feedback MCP process: " + f"{type(exc).__name__}: {exc}" + ) from exc + + return { + "schema_version": "codemesh-feedback-runtime-probe-v1", + "passed": True, + "provider_mode": "none", + "profile": plan.profile, + "server_name": plan.server_name, + "session_id": session_status["session_id"], + "manifest_file_sha256": manifest_hash, + "project_id": plan.binding.get("project_id"), + "checkout_id": plan.binding.get("checkout_id"), + "repository_root": plan.binding.get("repository_root"), + "tools": expected_tools, + "duration_ms": round((perf_counter() - started) * 1000, 3), + } + + +async def run_runtime_probe( + plan: InstallationPlan, + *, + timeout_seconds: int = 30, + query: str = "repository implementation entry points", + expected_paths: list[str] | None = None, +) -> dict[str, Any]: + verify_plan_applied(plan) + if plan.profile != "normal": + raise RuntimeProbeError( + "The product runtime probe requires the normal profile." + ) + launch = plan.launch + binding = plan.binding + expected_tools = sorted(str(value) for value in launch["expected_tools"]) + parameters = StdioServerParameters( + command=str(launch["command"]), + args=[str(value) for value in launch["args"]], + cwd=Path(str(launch["cwd"])), + env={**os.environ, "CODEMESH_MODEL_PROVIDER": "none"}, + ) + timeout = timedelta(seconds=timeout_seconds) + started = perf_counter() + + try: + async with asyncio.timeout(timeout_seconds): + async with stdio_client(parameters) as (read_stream, write_stream): + async with ClientSession(read_stream, write_stream) as session: + initialized = await session.initialize() + instructions = initialized.instructions or "" + _require_instruction_binding(instructions, binding) + + listed = await session.list_tools() + actual_tools = sorted(tool.name for tool in listed.tools) + if actual_tools != expected_tools: + raise RuntimeProbeError( + "Configured MCP tool surface differs from the reviewed plan: " + f"expected {expected_tools}, got {actual_tools}." + ) + + repositories = _tool_payload( + await session.call_tool( + "codemesh_list_repositories", + arguments={"limit": 10}, + read_timeout_seconds=timeout, + ) + ) + _require_bound_repository(repositories, binding) + + status = _tool_payload( + await session.call_tool( + "codemesh_get_repository_status", + arguments={}, + read_timeout_seconds=timeout, + ) + ) + status_data = _data(status) + _require_fresh_status(status_data, binding) + + package = await session.call_tool( + "codemesh_get_context_package", + arguments={ + "query": query, + "max_characters": 4000, + "include_content": False, + "include_relationships": True, + "output_format": "json", + }, + read_timeout_seconds=timeout, + ) + if package.isError: + raise RuntimeProbeError( + "The bound context-package call returned an MCP error." + ) + package_data = _data(_tool_payload(package)) + items = package_data.get("items") or [] + if not items: + raise RuntimeProbeError( + "The bound context-package probe returned no context items." + ) + returned_paths = sorted( + { + path + for item in items + if isinstance(item, dict) + for path in [_context_item_path(item)] + if path + } + ) + missing_paths = sorted( + set(expected_paths or []) - set(returned_paths) + ) + if missing_paths: + raise RuntimeProbeError( + "The bound context-package probe missed expected path(s): " + + ", ".join(missing_paths) + + f". Returned: {', '.join(returned_paths)}" + ) + except RuntimeProbeError: + raise + except TimeoutError as exc: + raise RuntimeProbeError( + f"CodeMesh MCP runtime probe exceeded {timeout_seconds} seconds." + ) from exc + except Exception as exc: + nested = _nested_probe_error(exc) + if nested is not None: + raise nested from exc + raise RuntimeProbeError( + "Could not start or verify the configured CodeMesh MCP process: " + f"{type(exc).__name__}: {exc}" + ) from exc + + freshness = status_data.get("freshness") or {} + repository = status_data.get("repository") or {} + return { + "schema_version": "codemesh-runtime-probe-v1", + "passed": True, + "provider_mode": "none", + "profile": plan.profile, + "server_name": plan.server_name, + "project_id": binding.get("project_id"), + "checkout_id": binding.get("checkout_id"), + "repository_root": binding.get("repository_root"), + "snapshot_id": repository.get("snapshot_id"), + "source_view_hash": repository.get("source_view_hash"), + "indexed_commit": freshness.get("indexed_commit"), + "current_commit": freshness.get("current_commit"), + "freshness_status": freshness.get("status"), + "binding_status": (freshness.get("binding") or {}).get("status"), + "tools": expected_tools, + "context_query": query, + "context_item_count": len(items), + "returned_paths": returned_paths, + "expected_paths": sorted(expected_paths or []), + "duration_ms": round((perf_counter() - started) * 1000, 3), + } + + +async def run_runtime_rejection_probe( + plan: InstallationPlan, + *, + timeout_seconds: int = 30, + project_id: str | None = None, + checkout_id: str | None = None, + repository_root: str | None = None, + source_view_hash: str | None = None, + expected_substrings: list[str] | None = None, +) -> dict[str, Any]: + verify_plan_applied(plan) + if plan.profile != "normal": + raise RuntimeProbeError( + "The product rejection probe requires the normal profile." + ) + overrides = { + "--project-id": project_id, + "--checkout-id": checkout_id, + "--repository-root": repository_root, + "--source-view-hash": source_view_hash, + } + launch = plan.launch + args = _overridden_args([str(value) for value in launch["args"]], overrides) + parameters = StdioServerParameters( + command=str(launch["command"]), + args=args, + cwd=Path(str(launch["cwd"])), + env={**os.environ, "CODEMESH_MODEL_PROVIDER": "none"}, + ) + timeout = timedelta(seconds=timeout_seconds) + started = perf_counter() + try: + async with asyncio.timeout(timeout_seconds): + async with stdio_client(parameters) as (read_stream, write_stream): + async with ClientSession(read_stream, write_stream) as session: + await session.initialize() + listed = await session.list_tools() + actual_tools = sorted(tool.name for tool in listed.tools) + expected_tools = sorted( + str(value) for value in launch["expected_tools"] + ) + if actual_tools != expected_tools: + raise RuntimeProbeError( + "Rejection probe tool surface differs from the reviewed plan." + ) + result = await session.call_tool( + "codemesh_get_repository_status", + arguments={}, + read_timeout_seconds=timeout, + ) + message = _result_text(result) + rejection_issues: list[str] = [] + rejected = bool(result.isError) + if not rejected: + payload = _tool_payload(result) + if payload.get("ok") is False: + rejected = True + else: + status_data = _data(payload) + assessment = (status_data.get("freshness") or {}).get( + "binding" + ) or {} + rejection_issues = [ + str(value) for value in assessment.get("issues") or [] + ] + effective_binding = { + **plan.binding, + "project_id": project_id + or plan.binding.get("project_id"), + "checkout_id": checkout_id + or plan.binding.get("checkout_id"), + "repository_root": repository_root + or plan.binding.get("repository_root"), + "source_view_hash": source_view_hash + if source_view_hash is not None + else plan.binding.get("source_view_hash"), + } + try: + _require_fresh_status(status_data, effective_binding) + except RuntimeProbeError as rejection: + rejected = True + detail = (status_data.get("freshness") or {}).get( + "detail" + ) + message = " ".join( + value + for value in [str(detail or ""), str(rejection)] + if value + ) + if not rejected: + raise RuntimeProbeError( + "Unsafe binding was accepted by the normal MCP profile." + ) + missing = [ + value + for value in expected_substrings or [] + if value.casefold() not in message.casefold() + ] + if missing: + raise RuntimeProbeError( + "Binding was rejected without expected diagnostic(s): " + + ", ".join(missing) + + f". Returned: {message}" + ) + except RuntimeProbeError: + raise + except TimeoutError as exc: + raise RuntimeProbeError( + f"CodeMesh MCP rejection probe exceeded {timeout_seconds} seconds." + ) from exc + except Exception as exc: + nested = _nested_probe_error(exc) + if nested is not None: + raise nested from exc + raise RuntimeProbeError( + "Could not run the configured CodeMesh rejection probe: " + f"{type(exc).__name__}: {exc}" + ) from exc + + return { + "schema_version": "codemesh-runtime-rejection-probe-v1", + "passed": True, + "rejected": True, + "provider_mode": "none", + "profile": plan.profile, + "project_id": project_id or plan.binding.get("project_id"), + "checkout_id": checkout_id or plan.binding.get("checkout_id"), + "repository_root": repository_root or plan.binding.get("repository_root"), + "source_view_hash": source_view_hash + if source_view_hash is not None + else plan.binding.get("source_view_hash"), + "expected_substrings": sorted(expected_substrings or []), + "rejection_issues": sorted(rejection_issues), + "diagnostic": message, + "duration_ms": round((perf_counter() - started) * 1000, 3), + } + + +def _require_instruction_binding( + instructions: str, + binding: dict[str, str | None], +) -> None: + required = [ + binding.get("project_id"), + binding.get("checkout_id"), + binding.get("repository_root"), + ] + missing = [value for value in required if value and value not in instructions] + if missing: + raise RuntimeProbeError( + "MCP initialization instructions do not identify the reviewed binding." + ) + + +def _require_bound_repository( + payload: dict[str, Any], + binding: dict[str, str | None], +) -> None: + repositories = _data(payload).get("repositories") or [] + if len(repositories) != 1: + raise RuntimeProbeError( + "Bound MCP repository listing must expose exactly one checkout." + ) + repository = repositories[0] + if ( + repository.get("project_id") != binding.get("project_id") + or repository.get("checkout_id") != binding.get("checkout_id") + or not _same_path( + repository.get("root_path"), + binding.get("repository_root"), + ) + ): + raise RuntimeProbeError( + "MCP repository listing does not match the reviewed binding." + ) + + +def _require_fresh_status( + status: dict[str, Any], + binding: dict[str, str | None], +) -> None: + repository = status.get("repository") or {} + freshness = status.get("freshness") or {} + assessment = freshness.get("binding") or {} + if ( + status.get("repository_id") != binding.get("project_id") + or repository.get("project_id") != binding.get("project_id") + or repository.get("checkout_id") != binding.get("checkout_id") + or freshness.get("status") != "fresh" + or freshness.get("is_stale") is not False + or assessment.get("status") != "accepted" + ): + issues = assessment.get("issues") or [] + raise RuntimeProbeError( + "Configured repository binding or freshness was rejected" + + (f": {', '.join(str(value) for value in issues)}" if issues else ".") + ) + + +def _tool_payload(result: Any) -> dict[str, Any]: + if result.isError: + raise RuntimeProbeError("A required MCP probe tool returned an error.") + structured = getattr(result, "structuredContent", None) + if isinstance(structured, dict): + return structured + for item in result.content: + text = getattr(item, "text", None) + if not text: + continue + try: + payload = json.loads(text) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + return payload + raise RuntimeProbeError("A required MCP probe tool returned no JSON object.") + + +def _data(payload: dict[str, Any]) -> dict[str, Any]: + data = payload.get("data", payload) + if not isinstance(data, dict): + raise RuntimeProbeError("A required MCP probe payload has an invalid shape.") + return data + + +def _same_path(left: Any, right: Any) -> bool: + if not left or not right: + return False + return os.path.normcase(os.path.realpath(str(left))) == os.path.normcase( + os.path.realpath(str(right)) + ) + + +def _context_item_path(item: dict[str, Any]) -> str | None: + hit = item.get("hit") if isinstance(item.get("hit"), dict) else {} + path = hit.get("file_path") + if path: + return str(path).replace("\\", "/") + node = item.get("node") if isinstance(item.get("node"), dict) else {} + span = node.get("span") if isinstance(node.get("span"), dict) else {} + path = span.get("file_path") + return str(path).replace("\\", "/") if path else None + + +def _overridden_args( + arguments: list[str], overrides: dict[str, str | None] +) -> list[str]: + result = list(arguments) + for flag, value in overrides.items(): + if value is None: + continue + if flag in result: + index = result.index(flag) + if index + 1 >= len(result): + raise RuntimeProbeError(f"Reviewed launch has no value for {flag}.") + result[index + 1] = value + else: + result.extend([flag, value]) + return result + + +def _require_launch_value(arguments: list[Any], flag: str, expected: str) -> None: + values = [str(value) for value in arguments] + if flag not in values: + raise RuntimeProbeError(f"Reviewed launch has no {flag} value.") + index = values.index(flag) + if index + 1 >= len(values) or values[index + 1] != expected: + raise RuntimeProbeError(f"Reviewed launch {flag} value changed.") + + +def _result_text(result: Any) -> str: + parts = [] + structured = getattr(result, "structuredContent", None) + if structured is not None: + parts.append(json.dumps(structured, sort_keys=True, default=str)) + for item in getattr(result, "content", []): + text = getattr(item, "text", None) + if text: + parts.append(str(text)) + return " ".join(parts) + + +def _nested_probe_error(error: BaseException) -> RuntimeProbeError | None: + if isinstance(error, RuntimeProbeError): + return error + if isinstance(error, BaseExceptionGroup): + for nested in error.exceptions: + match = _nested_probe_error(nested) + if match is not None: + return match + return None diff --git a/agent-access/codemesh_agent_access/rest.py b/agent-access/codemesh_agent_access/rest.py index 7dc5a82..914bf97 100644 --- a/agent-access/codemesh_agent_access/rest.py +++ b/agent-access/codemesh_agent_access/rest.py @@ -5,6 +5,7 @@ from fastapi import Depends, FastAPI +from . import __version__ from .models import ( ContextQuery, ContextResponse, @@ -38,7 +39,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: app = FastAPI( title="CodeMesh Agent Access", - version="0.1.0", + version=__version__, description="REST API for AI agents to read CodeMesh graph, content, and vector intelligence.", lifespan=lifespan, ) diff --git a/agent-access/codemesh_agent_access/store.py b/agent-access/codemesh_agent_access/store.py index 7fa83dd..12e8ca8 100644 --- a/agent-access/codemesh_agent_access/store.py +++ b/agent-access/codemesh_agent_access/store.py @@ -4,8 +4,19 @@ import json import os import subprocess -from typing import Any - +import sys +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +from time import perf_counter +from typing import Any, Awaitable, Callable, Iterator, TypeVar + +from .binding import ( + RepositoryBinding, + assess_binding, + bound_repository_reference, + require_accepted_binding, +) from .config import Settings from .content_store import MongoContentStore, PyMongoError from .embeddings import QueryEmbeddingProvider @@ -104,6 +115,116 @@ "unknown": 0.40, } +_CONTEXT_PACKAGE_TIMING_ENV = "CODEMESH_CONTEXT_PACKAGE_TIMINGS" +_CONTEXT_PACKAGE_TIMING_PREFIX = "CODEMESH_CONTEXT_PACKAGE_TIMING " +_T = TypeVar("_T") + + +@dataclass +class _ContextPackageTiming: + enabled: bool + clock: Callable[[], float] + started: float = field(init=False) + stages_ms: dict[str, float] = field(default_factory=dict) + operations_ms: dict[str, float] = field(default_factory=dict) + counts: dict[str, int] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.started = self.clock() + + def start(self) -> float: + return self.clock() + + def record_stage(self, name: str, started: float) -> None: + if self.enabled: + self.stages_ms[name] = _elapsed_ms(self.clock() - started) + + def record_operation(self, name: str, started: float) -> None: + if self.enabled: + self.operations_ms[name] = round( + self.operations_ms.get(name, 0.0) + + max(0.0, (self.clock() - started) * 1000), + 3, + ) + + def emit(self, outcome: str) -> None: + if not self.enabled: + return + payload = { + "event": "context_package_timing", + "outcome": outcome, + "total_ms": _elapsed_ms(self.clock() - self.started), + "stages_ms": self.stages_ms, + "operations_ms": self.operations_ms, + "counts": self.counts, + } + print( + _CONTEXT_PACKAGE_TIMING_PREFIX + + json.dumps(payload, sort_keys=True, separators=(",", ":")), + file=sys.stderr, + flush=True, + ) + + +_active_package_timing: ContextVar[_ContextPackageTiming | None] = ContextVar( + "codemesh_package_timing", default=None +) + + +@contextmanager +def _search_operation(name: str) -> Iterator[None]: + timing = _active_package_timing.get() + if timing is None or not timing.enabled: + yield + return + started = timing.start() + count_name = f"{name}_calls" + timing.counts[count_name] = timing.counts.get(count_name, 0) + 1 + try: + yield + finally: + timing.record_operation(name, started) + + +async def _timed_search_operation(name: str, awaitable: Awaitable[_T]) -> _T: + with _search_operation(name): + return await awaitable + + +@dataclass(frozen=True) +class _HydratedContextPackageItem: + hit: ContextHit + node: dict[str, Any] | None + content: dict[str, Any] | None + content_document: dict[str, Any] | None + summary: dict[str, Any] | None + relationships: list[dict[str, Any]] + + +def _elapsed_ms(seconds: float) -> float: + return round(max(0.0, seconds * 1000), 3) + + +def _context_package_timing_enabled() -> bool: + return os.getenv(_CONTEXT_PACKAGE_TIMING_ENV, "").strip().casefold() in { + "1", + "true", + "yes", + "on", + } + + +async def _timed_operation( + timing: _ContextPackageTiming, + name: str, + awaitable: Awaitable[_T], +) -> _T: + started = timing.start() + try: + return await awaitable + finally: + timing.record_operation(name, started) + class CodeMeshReadStore: """Read-side facade for graph, content, and vector intelligence.""" @@ -115,6 +236,7 @@ def __init__(self, settings: Settings) -> None: self._embeddings = QueryEmbeddingProvider(settings) self._graph = Neo4jGraphStore(settings) self._vectors = QdrantSearchClient(settings) + self._context_package_clock = perf_counter async def close(self) -> None: await self._graph.close() @@ -139,28 +261,45 @@ async def health(self) -> HealthResponse: return HealthResponse(status=status, components=components) async def search_context(self, query: ContextQuery) -> ContextResponse: - query = await self._resolve_context_query_repository(query) + query = await _timed_search_operation( + "search_repository_resolution", + self._resolve_context_query_repository(query), + ) vector = query.vector embedding_attempted = False embedding_source = "provided" if vector else "none" if not vector and self._embeddings.is_enabled and query.query.strip(): embedding_attempted = True - vector = await self._embeddings.embed_query(query.query) + vector = await _timed_search_operation( + "search_embedding", self._embeddings.embed_query(query.query) + ) if vector: embedding_source = "model_provider" vector_task = ( - asyncio.create_task(self._search_vector_context(query, vector)) + asyncio.create_task( + _timed_search_operation( + "search_vector_pipeline", self._search_vector_context(query, vector) + ) + ) if vector else None ) lexical_task = ( - asyncio.create_task(self._search_lexical_context(query)) + asyncio.create_task( + _timed_search_operation( + "search_lexical_pipeline", self._search_lexical_context(query) + ) + ) if query.query.strip() else None ) summary_task = ( - asyncio.create_task(self._search_summary_context(query)) + asyncio.create_task( + _timed_search_operation( + "search_summary_pipeline", self._search_summary_context(query) + ) + ) if query.query.strip() else None ) @@ -169,7 +308,8 @@ async def search_context(self, query: ContextQuery) -> ContextResponse: summary_hits = await summary_task if summary_task else [] hits = [*vector_hits, *lexical_hits, *summary_hits] - ranked_hits = _ranked_unique_hits(hits, query.limit) + with _search_operation("search_ranking"): + ranked_hits = _ranked_unique_hits(hits, query.limit) vector_hit_count = _source_hit_count(ranked_hits, "vector_score") lexical_hit_count = _source_hit_count(ranked_hits, "lexical_score") summary_hit_count = _source_hit_count(ranked_hits, "summary_score") @@ -198,118 +338,207 @@ async def search_context(self, query: ContextQuery) -> ContextResponse: return ContextResponse(query=query, hits=ranked_hits, diagnostics=diagnostics) async def get_context_package( - self, query: ContextPackageQuery + self, + query: ContextPackageQuery, + binding: RepositoryBinding | None = None, ) -> ContextPackageResponse: - requested_repository_id = query.repository_id - query = await self._resolve_package_query_repository(query) - search = await self.search_context( - ContextQuery( - query=query.query, - repository_id=query.repository_id, - limit=query.limit, - kinds=query.kinds, - filters=query.filters, - vector=query.vector, - expand_context=True, - context_depth=query.context_depth, - include_context_relationships=False, - ) + timing = _ContextPackageTiming( + enabled=_context_package_timing_enabled(), + clock=getattr(self, "_context_package_clock", perf_counter), ) - repository_summary = None - repository = None - latest_run = None - if requested_repository_id: - repository_summary = await self.get_repository(requested_repository_id) - if query.include_repository: - repository = ( - repository_summary.model_dump(mode="json") - if repository_summary - else None + timing_token = _active_package_timing.set(timing) + outcome = "failed" + try: + repository_started = timing.start() + requested_repository_id = bound_repository_reference( + binding, + query.repository_id, + required_when_unbound=False, + ) + repository_summary = None + if binding is not None: + status = await self.get_repository_status( + binding.project_id, + binding=binding, ) - if query.include_repository and requested_repository_id: - runs = await self.list_ingestion_runs( - repository_id=( - repository_summary.repository_id - if repository_summary - else requested_repository_id + require_accepted_binding(status.freshness.get("binding", {})) + repository_summary = status.repository + if repository_summary is None or not repository_summary.snapshot_id: + raise RuntimeError( + "CodeMesh accepted a repository binding without an active snapshot." + ) + query = query.model_copy( + update={"repository_id": repository_summary.snapshot_id} + ) + else: + query = await self._resolve_package_query_repository(query) + timing.record_stage("repository_resolution", repository_started) + + search_started = timing.start() + search = await self.search_context( + ContextQuery( + query=query.query, + repository_id=query.repository_id, + limit=query.limit, + kinds=query.kinds, + filters=query.filters, + vector=query.vector, + expand_context=True, + context_depth=query.context_depth, + include_context_relationships=False, + ) + ) + timing.record_stage("search", search_started) + timing.counts["hits"] = len(search.hits) + timing.counts["hydration_concurrency"] = 1 + + metadata_started = timing.start() + repository = None + latest_run = None + if requested_repository_id: + if repository_summary is None: + repository_summary = await self.get_repository( + requested_repository_id + ) + if query.include_repository: + repository = ( + repository_summary.model_dump(mode="json") + if repository_summary + else None + ) + if query.include_repository and requested_repository_id: + runs = await self.list_ingestion_runs( + repository_id=( + repository_summary.repository_id + if repository_summary + else requested_repository_id + ), + limit=1, + ) + latest_run = runs.runs[0].model_dump(mode="json") if runs.runs else None + timing.record_stage("repository_metadata", metadata_started) + + hydration_started = timing.start() + hydrated_items = [ + await self._hydrate_context_package_item(hit, query, timing) + for hit in search.hits + ] + timing.record_stage("item_hydration", hydration_started) + + assembly_started = timing.start() + remaining_characters = max(0, query.max_characters) + items: list[ContextPackageItem] = [] + total_snippet_characters = 0 + for index, hydrated in enumerate(hydrated_items): + snippet = None + if ( + query.include_content + and hydrated.content_document + and remaining_characters > 0 + ): + remaining_items = max(1, len(hydrated_items) - index) + item_budget = max(0, remaining_characters // remaining_items) + snippet = _snippet_for_hit( + hydrated.content_document, hydrated.hit, item_budget + ) + total_snippet_characters += len(snippet or "") + remaining_characters = max( + 0, remaining_characters - len(snippet or "") + ) + + items.append( + ContextPackageItem( + hit=hydrated.hit, + node=hydrated.node, + content=hydrated.content, + summary=hydrated.summary, + snippet=snippet, + relationships=hydrated.relationships, + relationship_groups=_group_relationships( + hydrated.relationships + ), + rationale={ + "rank": index + 1, + "score": hydrated.hit.score, + "score_components": hydrated.hit.score_components, + "expanded": hydrated.hit.expanded_context is not None, + }, + ) + ) + timing.record_stage("assembly", assembly_started) + outcome = "ok" + return ContextPackageResponse( + query=query, + repository=repository, + latest_run=latest_run, + items=items, + validation_recommendations=_validation_recommendations( + items, repository_summary ), - limit=1, + total_snippet_characters=total_snippet_characters, ) - latest_run = runs.runs[0].model_dump(mode="json") if runs.runs else None - - max_characters = max(0, query.max_characters) - remaining_characters = max_characters - items: list[ContextPackageItem] = [] - total_snippet_characters = 0 + finally: + _active_package_timing.reset(timing_token) + timing.emit(outcome) - for index, hit in enumerate(search.hits): - repository_id = hit.snapshot_id or query.repository_id - node_response = await self.get_node( + async def _hydrate_context_package_item( + self, + hit: ContextHit, + query: ContextPackageQuery, + timing: _ContextPackageTiming, + ) -> _HydratedContextPackageItem: + repository_id = hit.snapshot_id or query.repository_id + node_response = await _timed_operation( + timing, + "node", + self.get_node( NodeLookup( node_id=hit.node_id, repository_id=repository_id, include_content=query.include_content, include_neighbors=False, ) - ) - content = _content_metadata(node_response.content) - snippet = None - content_document = node_response.content - if query.include_content and hit.content_hash: - if ( - content_document is None - or str(content_document.get("hash") or "") != hit.content_hash - ): - content_document = await self._get_content(hit.content_hash) - content = _content_metadata(content_document) - - if query.include_content and content_document and remaining_characters > 0: - remaining_items = max(1, len(search.hits) - index) - item_budget = max(0, remaining_characters // remaining_items) - snippet = _snippet_for_hit(content_document, hit, item_budget) - total_snippet_characters += len(snippet or "") - remaining_characters = max(0, remaining_characters - len(snippet or "")) - - summary = await self._get_node_summary( - repository_id, hit.node_id, hit.content_hash - ) + ), + ) + content_document = node_response.content + if query.include_content and hit.content_hash: + if ( + content_document is None + or str(content_document.get("hash") or "") != hit.content_hash + ): + timing.counts["content_fallbacks"] = ( + timing.counts.get("content_fallbacks", 0) + 1 + ) + content_document = await _timed_operation( + timing, + "content_fallback", + self._get_content(hit.content_hash), + ) - relationships = [] - if query.include_relationships: - relationships = await self._get_context_relationship_summaries( + summary = await _timed_operation( + timing, + "summary", + self._get_node_summary(repository_id, hit.node_id, hit.content_hash), + ) + relationships = [] + if query.include_relationships: + relationships = await _timed_operation( + timing, + "relationships", + self._get_context_relationship_summaries( hit.node_id, repository_id, query.relationship_limit, _relationship_group_limits(query), - ) - - items.append( - ContextPackageItem( - hit=hit, - node=node_response.node, - content=content, - summary=summary, - snippet=snippet, - relationships=relationships, - relationship_groups=_group_relationships(relationships), - rationale={ - "rank": index + 1, - "score": hit.score, - "score_components": hit.score_components, - "expanded": hit.expanded_context is not None, - }, - ) + ), ) - - return ContextPackageResponse( - query=query, - repository=repository, - latest_run=latest_run, - items=items, - validation_recommendations=_validation_recommendations( - items, repository_summary - ), - total_snippet_characters=total_snippet_characters, + return _HydratedContextPackageItem( + hit=hit, + node=node_response.node, + content=_content_metadata(content_document), + content_document=content_document, + summary=summary, + relationships=relationships, ) async def search_symbols(self, query: SymbolQuery) -> SymbolSearchResponse: @@ -362,8 +591,30 @@ async def search_symbols(self, query: SymbolQuery) -> SymbolSearchResponse: return SymbolSearchResponse(query=query, symbols=symbols) - async def get_node(self, lookup: NodeLookup) -> NodeResponse: - lookup = await self._resolve_node_lookup_repository(lookup) + async def get_node( + self, + lookup: NodeLookup, + binding: RepositoryBinding | None = None, + ) -> NodeResponse: + if binding is not None: + bound_repository_reference( + binding, + lookup.repository_id, + required_when_unbound=False, + ) + status = await self.get_repository_status( + binding.project_id, + binding=binding, + ) + require_accepted_binding(status.freshness.get("binding", {})) + repository = status.repository + if repository is None or not repository.snapshot_id: + raise RuntimeError( + "CodeMesh accepted a repository binding without an active snapshot." + ) + lookup = lookup.model_copy(update={"repository_id": repository.snapshot_id}) + else: + lookup = await self._resolve_node_lookup_repository(lookup) node = await self._get_node_properties(lookup.node_id, lookup.repository_id) if node is None or not _matches_repository(node, lookup.repository_id): return NodeResponse() @@ -392,7 +643,19 @@ async def get_neighbors(self, query: NeighborQuery) -> NodeResponse: ) return NodeResponse(relationships=relationships) - async def list_repositories(self, limit: int = 50) -> RepositoryListResponse: + async def list_repositories( + self, + limit: int = 50, + binding: RepositoryBinding | None = None, + ) -> RepositoryListResponse: + if binding is not None: + repository = await self._content.get_bound_repository( + binding.project_id, + binding.checkout_id, + ) + return RepositoryListResponse( + repositories=[repository] if repository is not None else [] + ) return await self._content.list_repositories(limit) async def get_repository(self, repository_id: str) -> RepositorySummary | None: @@ -455,24 +718,45 @@ async def list_ingestion_runs( return await self._content.list_ingestion_runs(repository_id, limit) async def get_repository_status( - self, repository_id: str + self, + repository_id: str, + binding: RepositoryBinding | None = None, ) -> RepositoryStatusResponse: - repository = await self.get_repository(repository_id) + resolved_reference = bound_repository_reference( + binding, + repository_id, + required_when_unbound=True, + ) + repository = ( + await self._content.get_bound_repository( + binding.project_id, + binding.checkout_id, + ) + if binding is not None + else await self.get_repository(resolved_reference or repository_id) + ) if repository is None: + freshness = { + "status": "missing", + "is_stale": None, + "detail": "Repository was not found in the CodeMesh registry.", + } + if binding is not None: + freshness["binding"] = assess_binding(binding, None, freshness) return RepositoryStatusResponse( - repository_id=repository_id, - freshness={ - "status": "missing", - "is_stale": None, - "detail": "Repository was not found in the CodeMesh registry.", - }, + repository_id=resolved_reference or repository_id, + freshness=freshness, refresh=_refresh_status(None), ) runs = await self.list_ingestion_runs(repository.repository_id, limit=1) latest_run = runs.runs[0] if runs.runs else None - summary_coverage = await self.get_summary_coverage(repository.repository_id) + summary_coverage = await self.get_summary_coverage( + repository.snapshot_id or repository.repository_id + ) freshness = await asyncio.to_thread(_repository_freshness, repository) + if binding is not None: + freshness["binding"] = assess_binding(binding, repository, freshness) return RepositoryStatusResponse( repository_id=repository.repository_id, @@ -709,13 +993,16 @@ async def _build_context_hit( context_relationship_limit: int = 8, ) -> ContextHit: source = source_node or node - selected, expansion_reason = await self._select_context_node( - node, - repository_id, - expand_context, - context_depth, + selected, expansion_reason = await _timed_search_operation( + "search_expansion", + self._select_context_node( + node, repository_id, expand_context, context_depth + ), + ) + declaration = await _timed_search_operation( + "search_declaration", + self._get_primary_declaration_node(selected, repository_id), ) - declaration = await self._get_primary_declaration_node(selected, repository_id) content_node = declaration or selected content_hash = str( @@ -724,7 +1011,9 @@ async def _build_context_hit( or source.get("contentHash") or "" ) - content = await self._get_content(content_hash) + content = await _timed_search_operation( + "search_content", self._get_content(content_hash) + ) selected_id = str(selected.get("id") or "") source_id = str(source.get("id") or "") declaration_id = str(declaration.get("id") or "") if declaration else "" @@ -745,10 +1034,11 @@ async def _build_context_hit( } if include_context_relationships: - relationships = await self._get_context_relationship_summaries( - selected_id, - repository_id, - context_relationship_limit, + relationships = await _timed_search_operation( + "search_relationships", + self._get_context_relationship_summaries( + selected_id, repository_id, context_relationship_limit + ), ) if expanded_context is None: expanded_context = { @@ -774,23 +1064,27 @@ async def _build_context_hit( ) async def _search_lexical_context(self, query: ContextQuery) -> list[ContextHit]: - search_text = query.query.casefold() + search_text = query.query kinds = [kind.casefold() for kind in query.kinds] candidate_limit = min(max(query.limit * 4, 16), 200) raw_limit = min(max(query.limit * 16, 64), 500) repository_id = query.repository_id - nodes = await self._graph.search_context_nodes( - search_text, kinds, repository_id, raw_limit + nodes = await _timed_search_operation( + "search_lexical_fetch", + self._graph.search_context_nodes( + search_text, kinds, repository_id, raw_limit + ), ) - nodes = [ - node - for node in nodes - if _matches_filters(node, query.filters) - and _matches_repository(node, query.repository_id) - ] - nodes.sort(key=lambda node: _lexical_score(node, search_text), reverse=True) - nodes = _diversify_lexical_nodes(nodes, candidate_limit) + with _search_operation("search_lexical_selection"): + nodes = [ + node + for node in nodes + if _matches_filters(node, query.filters) + and _matches_repository(node, query.repository_id) + ] + nodes.sort(key=lambda node: _lexical_score(node, search_text), reverse=True) + nodes = _diversify_lexical_nodes(nodes, candidate_limit) hits = [] for node in nodes: @@ -812,8 +1106,11 @@ async def _search_lexical_context(self, query: ContextQuery) -> list[ContextHit] async def _search_vector_context( self, query: ContextQuery, vector: list[float] ) -> list[ContextHit]: - qdrant_hits = await self._vectors.search( - vector, _candidate_limit(query.limit), _effective_filters(query) + qdrant_hits = await _timed_search_operation( + "search_vector_fetch", + self._vectors.search( + vector, _candidate_limit(query.limit), _effective_filters(query) + ), ) hits: list[ContextHit] = [] @@ -823,7 +1120,9 @@ async def _search_vector_context( repository_id = ( query.repository_id or str(item.get("repositoryId") or "") or None ) - node = await self._get_node_properties(node_id, repository_id) + node = await _timed_search_operation( + "search_vector_node", self._get_node_properties(node_id, repository_id) + ) if node is None: node = { @@ -857,11 +1156,11 @@ async def _search_vector_context( async def _search_summary_context(self, query: ContextQuery) -> list[ContextHit]: search_text = query.query.casefold() kinds = [kind for kind in query.kinds if kind] - summaries = await self._summaries.search_summaries( - query.query, - query.repository_id, - kinds, - _candidate_limit(query.limit), + summaries = await _timed_search_operation( + "search_summary_fetch", + self._summaries.search_summaries( + query.query, query.repository_id, kinds, _candidate_limit(query.limit) + ), ) hits: list[ContextHit] = [] @@ -873,7 +1172,9 @@ async def _search_summary_context(self, query: ContextQuery) -> list[ContextHit] repository_id = ( query.repository_id or str(summary.get("repository_id") or "") or None ) - node = await self._get_node_properties(node_id, repository_id) + node = await _timed_search_operation( + "search_summary_node", self._get_node_properties(node_id, repository_id) + ) if node is None: node = _node_from_summary(summary) @@ -917,22 +1218,42 @@ def _repository_freshness(repository: RepositorySummary) -> dict[str, Any]: "snapshotId" ) observed_snapshot_id = repository.metadata.get("observedSnapshotId") + indexed_working_tree_dirty = _optional_bool( + repository.metadata.get("workingTreeDirty") + ) git = _git_snapshot(root_path) if git["available"]: current_commit = git.get("current_commit") is_dirty = bool(git.get("working_tree_dirty")) commit_mismatch = bool(indexed_commit and indexed_commit != current_commit) - source_view_mismatch = ( + snapshot_mismatch = ( observed_snapshot_id != selected_snapshot_id if observed_snapshot_id and selected_snapshot_id else None ) + dirty_state_mismatch = ( + indexed_working_tree_dirty != is_dirty + if indexed_working_tree_dirty is not None + else None + ) + source_view_mismatch = ( + True + if snapshot_mismatch is True or dirty_state_mismatch is True + else None + if snapshot_mismatch is None and dirty_state_mismatch is None + else False + ) if indexed_commit: - is_stale = commit_mismatch or is_dirty or source_view_mismatch is True + unverifiable_dirty_view = is_dirty or indexed_working_tree_dirty is True + is_stale = ( + commit_mismatch + or unverifiable_dirty_view + or source_view_mismatch is True + ) status = "stale" if is_stale else "fresh" detail = ( - "Indexed commit differs from the local checkout or the working tree has uncommitted changes." + "Indexed commit, checkout state, or source-view provenance does not match a verifiable clean checkout." if is_stale else "Indexed commit matches the local checkout and the working tree is clean." ) @@ -947,7 +1268,9 @@ def _repository_freshness(repository: RepositorySummary) -> dict[str, Any]: "selected_snapshot_id": selected_snapshot_id, "indexed_source_view_hash": repository.source_view_hash or repository.metadata.get("sourceViewHash"), + "indexed_working_tree_dirty": indexed_working_tree_dirty, "commit_mismatch": commit_mismatch, + "dirty_state_mismatch": dirty_state_mismatch, "source_view_mismatch": source_view_mismatch, "provenance_status": "known" if selected_snapshot_id else "unknown", "is_stale": is_stale, @@ -961,7 +1284,9 @@ def _repository_freshness(repository: RepositorySummary) -> dict[str, Any]: "selected_snapshot_id": selected_snapshot_id, "indexed_source_view_hash": repository.source_view_hash or repository.metadata.get("sourceViewHash"), + "indexed_working_tree_dirty": indexed_working_tree_dirty, "commit_mismatch": None, + "dirty_state_mismatch": None, "source_view_mismatch": None, "provenance_status": "unavailable", "is_stale": None, @@ -1016,6 +1341,19 @@ def _run_git(root_path: str, *args: str) -> str | None: return result.stdout.strip() +def _optional_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if value is None: + return None + normalized = str(value).strip().casefold() + if normalized == "true": + return True + if normalized == "false": + return False + return None + + def _refresh_status( latest_run: Any | None, freshness: dict[str, Any] | None = None ) -> dict[str, Any]: @@ -1131,6 +1469,30 @@ def _validation_recommendations( recommendations: list[dict[str, Any]] = [] is_codemesh = repository is not None and repository.alias == "code_mesh" + if any(_is_rust_path(path) for path in file_paths): + recommendations.extend( + [ + _validation_command( + "rust-format", + "cargo fmt --all -- --check", + "Check Rust formatting from the repository or workspace root.", + file_paths, + ), + _validation_command( + "rust-clippy", + "cargo clippy --all-targets --all-features -- -D warnings", + "Run the Rust lint gate for affected targets and features.", + file_paths, + ), + _validation_command( + "rust-tests", + "cargo test --all-targets --all-features", + "Run Rust tests after native implementation or PyO3 boundary changes.", + file_paths, + ), + ] + ) + if not is_codemesh: if any(_is_dotnet_path(path) for path in file_paths): recommendations.append( @@ -1233,6 +1595,10 @@ def _validation_applies(identifier: str, path: str) -> bool: and _is_deployment_path(path) ) or (identifier == "markdown-dry-run" and _is_markdown_path(path)) + or ( + identifier in {"rust-format", "rust-clippy", "rust-tests"} + and _is_rust_path(path) + ) ) @@ -1272,6 +1638,11 @@ def _is_markdown_path(path: str) -> bool: return lower.endswith(".md") or lower.endswith(".markdown") +def _is_rust_path(path: str) -> bool: + lower = path.casefold() + return lower.endswith(".rs") or lower.endswith("cargo.toml") + + def _normalize_node(node: dict[str, Any]) -> dict[str, Any]: metadata = _metadata(node.get("metadataJson")) return { @@ -1547,11 +1918,50 @@ def _ranked_unique_hits(hits: list[ContextHit], limit: int) -> list[ContextHit]: else: best_by_node_id[hit.node_id] = _merge_hit_score_components(existing, hit) - return sorted( + ranked = sorted( best_by_node_id.values(), - key=lambda hit: (hit.score, _kind_weight_from_name(hit.kind), -hit.start_line), + key=lambda hit: ( + hit.score, + _raw_relevance_score(hit), + _kind_weight_from_name(hit.kind), + -hit.start_line, + ), reverse=True, - )[:limit] + ) + return _diversify_ranked_hits(ranked, limit) + + +def _raw_relevance_score(hit: ContextHit) -> float: + return max( + ( + float(hit.score_components.get(name, 0.0)) + for name in ("vector_score", "lexical_score", "summary_score") + ), + default=0.0, + ) + + +def _diversify_ranked_hits(hits: list[ContextHit], limit: int) -> list[ContextHit]: + if limit <= 0: + return [] + + buckets: dict[str, list[ContextHit]] = {} + for hit in hits: + key = hit.file_path.replace("\\", "/").casefold() or hit.node_id + buckets.setdefault(key, []).append(hit) + + selected: list[ContextHit] = [] + while len(selected) < limit: + added = False + for bucket in buckets.values(): + if bucket: + selected.append(bucket.pop(0)) + added = True + if len(selected) >= limit: + break + if not added: + break + return selected def _merge_hit_score_components( @@ -1786,44 +2196,78 @@ def _lexical_score(node: dict[str, Any], search_text: str) -> float: if not search_text: return 0.5 + normalized_search_text = search_text.casefold() + + metadata = _metadata(node.get("metadataJson")) fields = [ str(node.get("id") or "").casefold(), str(node.get("stableKey") or "").casefold(), str(node.get("name") or "").casefold(), str(node.get("filePath") or "").casefold(), + str(node.get("language") or "").casefold(), + str(metadata.get("signature") or "").casefold(), + str(metadata.get("display") or "").casefold(), + str(metadata.get("pythonExportName") or "").casefold(), + str(metadata.get("attributes") or "").casefold(), ] - if any(field == search_text for field in fields): + if any(field == normalized_search_text for field in fields): return 1.0 - if any(field.startswith(search_text) for field in fields): + if any(field.startswith(normalized_search_text) for field in fields): return 0.85 - if any(search_text in field for field in fields): + if any(normalized_search_text in field for field in fields): return 0.65 terms = lexical_terms(search_text) if not terms: return 0.0 - matched_terms = sum(1 for term in terms if any(term in field for field in fields)) + matched_terms = sum(1 for term in terms if _lexical_term_matches(term, fields)) coverage = matched_terms / len(terms) best_field_coverage = max( - (sum(1 for term in terms if term in field) / len(terms) for field in fields), + ( + sum(1 for term in terms if _lexical_term_matches(term, [field])) + / len(terms) + for field in fields + ), default=0.0, ) file_name = fields[3].rsplit("/", 1)[-1].rsplit("\\", 1)[-1] identifier_fields = [fields[2], file_name] identifier_coverage = sum( - 1 for term in terms if any(term in field for field in identifier_fields) + 1 for term in terms if _lexical_term_matches(term, identifier_fields) + ) / len(terms) + semantic_fields = fields[4:] + semantic_coverage = sum( + 1 for term in terms if _lexical_term_matches(term, semantic_fields) ) / len(terms) return min( 0.95, 0.15 + (coverage * 0.55) + (best_field_coverage * 0.10) - + (identifier_coverage * 0.25), + + (identifier_coverage * 0.25) + + (semantic_coverage * 0.18), ) +def _lexical_term_matches(term: str, fields: list[str]) -> bool: + return any( + variant in field for variant in _lexical_term_variants(term) for field in fields + ) + + +def _lexical_term_variants(term: str) -> tuple[str, ...]: + variants = [term] + if len(term) > 4 and term.endswith("ing"): + variants.append(term[:-3]) + if len(term) > 4 and term.endswith("ies"): + variants.append(f"{term[:-3]}y") + elif len(term) > 4 and term.endswith("s") and not term.endswith("ss"): + variants.append(term[:-1]) + return tuple(dict.fromkeys(variants)) + + def _symbol_score(node: dict[str, Any], search_text: str) -> float: lexical_score = _symbol_lexical_score(node, search_text) return max(0.0, min(1.0, lexical_score * _kind_weight(node))) diff --git a/agent-access/codemesh_agent_access/tools.py b/agent-access/codemesh_agent_access/tools.py index 42afcf1..5804092 100644 --- a/agent-access/codemesh_agent_access/tools.py +++ b/agent-access/codemesh_agent_access/tools.py @@ -1,5 +1,6 @@ from __future__ import annotations +from .binding import RepositoryBinding from .dependencies import get_store from .models import ( ContextPackageQuery, @@ -70,7 +71,9 @@ async def get_context_package( definition_limit: int | None = None, uses_type_limit: int | None = None, include_repository: bool = True, + binding: RepositoryBinding | None = None, ) -> ToolResult: + binding_arguments = {"binding": binding} if binding is not None else {} result = await get_store().get_context_package( ContextPackageQuery( query=query, @@ -92,7 +95,8 @@ async def get_context_package( definition_limit=definition_limit, uses_type_limit=uses_type_limit, include_repository=include_repository, - ) + ), + **binding_arguments, ) return ToolResult(data=result.model_dump(mode="json")) @@ -123,14 +127,17 @@ async def get_node( repository_id: str | None = None, include_content: bool = True, include_neighbors: bool = True, + binding: RepositoryBinding | None = None, ) -> ToolResult: + binding_arguments = {"binding": binding} if binding is not None else {} result = await get_store().get_node( NodeLookup( node_id=node_id, repository_id=repository_id, include_content=include_content, include_neighbors=include_neighbors, - ) + ), + **binding_arguments, ) return ToolResult(data=result.model_dump(mode="json")) @@ -152,8 +159,15 @@ async def get_neighbors( return ToolResult(data=result.model_dump(mode="json")) -async def list_repositories(limit: int = 50) -> ToolResult: - result = await get_store().list_repositories(limit=limit) +async def list_repositories( + limit: int = 50, + binding: RepositoryBinding | None = None, +) -> ToolResult: + binding_arguments = {"binding": binding} if binding is not None else {} + result = await get_store().list_repositories( + limit=limit, + **binding_arguments, + ) return ToolResult(data=result.model_dump(mode="json")) @@ -174,8 +188,15 @@ async def get_summary_coverage(repository_id: str) -> ToolResult: return ToolResult(data=result.model_dump(mode="json")) -async def get_repository_status(repository_id: str) -> ToolResult: - result = await get_store().get_repository_status(repository_id) +async def get_repository_status( + repository_id: str, + binding: RepositoryBinding | None = None, +) -> ToolResult: + binding_arguments = {"binding": binding} if binding is not None else {} + result = await get_store().get_repository_status( + repository_id, + **binding_arguments, + ) return ToolResult(data=result.model_dump(mode="json")) diff --git a/agent-access/codemesh_agent_access/web.py b/agent-access/codemesh_agent_access/web.py index e09af52..0aefaf4 100644 --- a/agent-access/codemesh_agent_access/web.py +++ b/agent-access/codemesh_agent_access/web.py @@ -95,7 +95,7 @@ async def repository_detail(

{heading}

Repository status and recent ingestion history.

- JSON + JSON
@@ -180,7 +180,7 @@ async def search(
- +
@@ -316,7 +316,7 @@ def _page(title: str, body: str, *, active: str) -> HTMLResponse: def _nav_link(label: str, href: str, active: bool) -> str: class_name = "active" if active else "" - return f'{escape(label)}' + return f'{escape(label)}' def _repository_table(repositories: list, *, detailed: bool = False) -> str: @@ -335,7 +335,7 @@ def _repository_table(repositories: list, *, detailed: bool = False) -> str: ) rows.append( "" - f'{_value(identity)}
{_value(repository.repository_id)}' + f'{_value(identity)}
{_value(repository.repository_id)}' f"{_value(repository.node_count)}" f"{_value(repository.relationship_count)}" f"{_value(repository.content_count)}" @@ -359,7 +359,7 @@ def _run_table(runs: list) -> str: rows.append( "" f'{_value(run.run_id)}
{_value(run.language)} / {_value(run.parser_name)}' - f'{_value(run.repository_id)}' + f'{_value(run.repository_id)}' f"{_badge(run.stage)}" f"{_value(run.node_count)}" f"{_value(run.relationship_count)}" diff --git a/agent-access/pyproject.toml b/agent-access/pyproject.toml index deae5c8..d9a7cb2 100644 --- a/agent-access/pyproject.toml +++ b/agent-access/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "codemesh-agent-access" version = "0.1.0" -description = "REST and MCP read access to local CodeMesh intelligence." +description = "Local CodeMesh Agent Access and development-feedback MCP integration." requires-python = ">=3.12" dependencies = [ "fastapi>=0.115", @@ -28,3 +28,6 @@ build-backend = "hatchling.build" [tool.pytest.ini_options] pythonpath = ["."] + +[tool.ruff] +target-version = "py312" diff --git a/agent-access/tests/test_capped_codex.py b/agent-access/tests/test_capped_codex.py new file mode 100644 index 0000000..43f6b9a --- /dev/null +++ b/agent-access/tests/test_capped_codex.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import httpx +import pytest + +import codemesh_agent_access.evaluation.capped_codex as capped_codex +from codemesh_agent_access.evaluation.capped_codex import ( + CappedCodexRunner, + CappedProxyError, + CappedResponsesProxy, + HardCapLedger, + _capped_provider_config, + _validate_response_request, +) +from codemesh_agent_access.evaluation.codex import ( + REPORTED_TOKEN_ACCOUNTING_ID, + AgentRunRequest, + AgentRunResult, +) + + +class BrokenStream(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"response.created"}\n\n' + raise httpx.ReadError("synthetic interrupted stream") + + +class StaticStream(httpx.AsyncByteStream): + def __init__(self, content: bytes) -> None: + self._content = content + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._content + + +def test_hard_cap_ledger_enforces_exact_boundary_and_accounting() -> None: + ledger = HardCapLedger(10) + reservation, output_allowance = ledger.reserve(4, 20) + + assert output_allowance == 3 + assert ledger.snapshot()["remaining_tokens"] == 0 + + ledger.settle( + reservation, + { + "input_tokens": 4, + "input_tokens_details": {"cached_tokens": 1}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 3}, + }, + ) + + assert ledger.snapshot() == { + "accounting": REPORTED_TOKEN_ACCOUNTING_ID, + "ceiling": 10, + "committed_tokens": 10, + "reported_tokens": 10, + "retained_reservation_tokens": 0, + "remaining_tokens": 0, + "request_count": 1, + "completed_request_count": 1, + "usage": { + "input_tokens": 4, + "cached_input_tokens": 1, + "output_tokens": 3, + "reasoning_output_tokens": 3, + }, + "request_max_retries": 0, + "stream_max_retries": 0, + "failures": [], + "complete": True, + } + with pytest.raises(CappedProxyError, match="Insufficient remaining"): + ledger.reserve(0, 1) + + +def test_hard_cap_ledger_supports_multiple_requests_and_rejects_disagreement() -> None: + ledger = HardCapLedger(30) + first, _ = ledger.reserve(4, 10) + ledger.settle( + first, + { + "input_tokens": 4, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 1}, + }, + ) + second, allowance = ledger.reserve(5, 4) + assert allowance == 4 + with pytest.raises(CappedProxyError, match="disagreed"): + ledger.settle( + second, + {"input_tokens": 6, "output_tokens": 1}, + ) + + +def test_hard_cap_ledger_clamps_to_provider_output_maximum() -> None: + ledger = HardCapLedger(1_000_000) + + reservation, allowance = ledger.reserve(10, None) + + assert allowance == 128_000 + assert reservation.reserved_tokens == 256_010 + + +def test_proxy_clamps_streamed_request_and_records_completed_usage() -> None: + forwarded: list[dict[str, Any]] = [] + + def upstream(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/responses/input_tokens"): + return httpx.Response(200, json={"input_tokens": 4}) + forwarded.append(json.loads(request.content)) + event = { + "type": "response.completed", + "response": { + "usage": { + "input_tokens": 4, + "input_tokens_details": {"cached_tokens": 1}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 2}, + } + }, + } + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=StaticStream(f"data: {json.dumps(event)}\n\n".encode()), + ) + + proxy, upstream_client = _proxy(upstream, ceiling=14) + + async def exercise() -> None: + transport = httpx.ASGITransport(app=proxy.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://proxy" + ) as client: + response = await client.post( + "/v1/responses", + headers={"authorization": "Bearer local-secret"}, + json={ + "model": "test-model", + "input": "count all submitted input", + "stream": True, + "max_output_tokens": 99, + "tools": [{"type": "function", "name": "local_tool"}], + }, + ) + assert response.status_code == 200 + await upstream_client.aclose() + + asyncio.run(exercise()) + + assert forwarded[0]["max_output_tokens"] == 5 + snapshot = proxy.ledger.snapshot() + assert snapshot["complete"] is True + assert snapshot["reported_tokens"] == 11 + assert snapshot["remaining_tokens"] == 3 + + +def test_proxy_retains_reservation_after_interrupted_stream() -> None: + def upstream(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/responses/input_tokens"): + return httpx.Response(200, json={"input_tokens": 4}) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=BrokenStream(), + ) + + proxy, upstream_client = _proxy(upstream, ceiling=10) + + async def exercise() -> None: + transport = httpx.ASGITransport(app=proxy.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://proxy" + ) as client: + with pytest.raises(httpx.ReadError): + await client.post( + "/v1/responses", + headers={"authorization": "Bearer local-secret"}, + json={"model": "test-model", "input": "x", "stream": True}, + ) + await upstream_client.aclose() + + asyncio.run(exercise()) + + snapshot = proxy.ledger.snapshot() + assert snapshot["complete"] is False + assert snapshot["committed_tokens"] == 10 + assert snapshot["retained_reservation_tokens"] == 10 + assert "interrupted" in " ".join(snapshot["failures"]).lower() + + +def test_proxy_rejects_missing_usage_backend_failure_and_duplicate_retry() -> None: + response_calls = 0 + + def upstream(request: httpx.Request) -> httpx.Response: + nonlocal response_calls + if request.url.path.endswith("/responses/input_tokens"): + return httpx.Response(200, json={"input_tokens": 2}) + response_calls += 1 + event = {"type": "response.completed", "response": {}} + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=StaticStream(f"data: {json.dumps(event)}\n\n".encode()), + ) + + proxy, upstream_client = _proxy(upstream, ceiling=8) + body = {"model": "test-model", "input": "x", "stream": True} + + async def exercise() -> None: + transport = httpx.ASGITransport(app=proxy.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://proxy" + ) as client: + first = await client.post( + "/v1/responses", + headers={"authorization": "Bearer local-secret"}, + json=body, + ) + assert first.status_code == 200 + second = await client.post( + "/v1/responses", + headers={"authorization": "Bearer local-secret"}, + json=body, + ) + assert second.status_code == 400 + await upstream_client.aclose() + + asyncio.run(exercise()) + + assert response_calls == 1 + snapshot = proxy.ledger.snapshot() + assert snapshot["committed_tokens"] == 8 + assert snapshot["completed_request_count"] == 0 + assert any("usage" in failure for failure in snapshot["failures"]) + assert any("retry" in failure for failure in snapshot["failures"]) + + +def test_proxy_fails_closed_without_exposing_secrets() -> None: + def upstream(_request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("backend failed with upstream-secret") + + upstream_client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + proxy = CappedResponsesProxy( + ceiling=10, + local_token="local-secret", + upstream_api_key="upstream-secret", + upstream_base_url="https://upstream.test/v1", + upstream_client=upstream_client, + ) + + async def exercise() -> str: + transport = httpx.ASGITransport(app=proxy.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://proxy" + ) as client: + response = await client.post( + "/v1/responses", + headers={"authorization": "Bearer local-secret"}, + json={"model": "test-model", "input": "x"}, + ) + await upstream_client.aclose() + return response.text + + response_text = asyncio.run(exercise()) + retained = json.dumps(proxy.ledger.snapshot()) + response_text + assert "local-secret" not in retained + assert "upstream-secret" not in retained + assert "Input-token counting failed closed" in retained + + +def test_proxy_handles_cost_free_model_discovery_without_poisoning_ledger() -> None: + proxy, upstream_client = _proxy(lambda _request: httpx.Response(500), ceiling=10) + + async def exercise() -> dict[str, Any]: + transport = httpx.ASGITransport(app=proxy.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://proxy" + ) as client: + response = await client.get( + "/v1/models?client_version=test", + headers={"authorization": "Bearer local-secret"}, + ) + await upstream_client.aclose() + assert response.status_code == 200 + return response.json() + + assert asyncio.run(exercise()) == {"models": []} + assert proxy.ledger.failures == [] + + +def test_capped_runner_declares_cap_and_zero_retry_provider_configuration() -> None: + runner = CappedCodexRunner() + + assert runner.capabilities.hard_reported_token_cap is True + assert runner.capabilities.reported_token_accounting == REPORTED_TOKEN_ACCOUNTING_ID + config = _capped_provider_config( + "http://127.0.0.1:1234/v1", Path("/tmp/capped-models.json") + ) + assert "model_providers.codemesh_capped.request_max_retries=0" in config + assert "model_providers.codemesh_capped.stream_max_retries=0" in config + assert "model_providers.codemesh_capped.supports_websockets=false" in config + assert any(value.startswith("model_catalog_json=") for value in config) + + +@pytest.mark.parametrize( + ("body", "message"), + [ + ({"background": True}, "Background"), + ({"context_management": [{"type": "compaction"}]}, "compaction"), + ({"prompt": {"id": "prompt_123"}}, "counting contract"), + ({"future_input": "unreviewed"}, "counting contract"), + ({"tools": [{"type": "web_search"}]}, "cost-bearing"), + ({"tools": "not-an-array"}, "array"), + ], +) +def test_capped_proxy_rejects_unsupported_cost_activity( + body: dict[str, Any], message: str +) -> None: + assert message in str(_validate_response_request(body)) + + +def test_capped_runner_rejects_codex_proxy_usage_disagreement( + monkeypatch: Any, tmp_path: Path +) -> None: + ledger = HardCapLedger(100) + reservation, _ = ledger.reserve(10, 20) + ledger.settle( + reservation, + { + "input_tokens": 10, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 2}, + }, + ) + + class FakeProxy: + def __init__(self, **_kwargs: Any) -> None: + self.ledger = ledger + + @asynccontextmanager + async def serve(self) -> AsyncIterator[str]: + yield "http://127.0.0.1:1234/v1" + + class FakeCodexRunner: + def __init__(self, *_args: Any, **kwargs: Any) -> None: + assert kwargs["environment_overrides"].keys() == { + "CODEMESH_CAPPED_PROXY_TOKEN" + } + assert ( + "model_providers.codemesh_capped.request_max_retries=0" + in kwargs["config_overrides"] + ) + + async def run(self, _request: AgentRunRequest) -> AgentRunResult: + return AgentRunResult( + completed=True, + exit_code=0, + duration_ms=1, + usage={ + "input_tokens": 10, + "cached_input_tokens": 0, + "output_tokens": 4, + "reasoning_output_tokens": 2, + }, + ) + + monkeypatch.setenv("OPENAI_API_KEY", "test-upstream-secret") + monkeypatch.setattr(capped_codex, "CappedResponsesProxy", FakeProxy) + monkeypatch.setattr(capped_codex, "CodexRunner", FakeCodexRunner) + monkeypatch.setattr( + CappedCodexRunner, + "_load_bundled_model", + lambda _self, model: {"slug": model}, + ) + request = AgentRunRequest( + prompt="test", + cwd=tmp_path, + condition="control", + model="test-model", + reasoning_effort="medium", + sandbox="read-only", + timeout_seconds=30, + mcp_cwd=tmp_path, + max_reported_tokens=100, + ) + + result = asyncio.run(CappedCodexRunner().run(request)) + + assert result.completed is False + assert result.hard_cap_ledger is not None + assert result.hard_cap_ledger["complete"] is False + assert any("disagreed" in failure for failure in result.failures) + assert "test-upstream-secret" not in json.dumps(result.hard_cap_ledger) + + +def _proxy( + handler: Any, *, ceiling: int +) -> tuple[CappedResponsesProxy, httpx.AsyncClient]: + upstream_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return ( + CappedResponsesProxy( + ceiling=ceiling, + local_token="local-secret", + upstream_api_key="upstream-secret", + upstream_base_url="https://upstream.test/v1", + upstream_client=upstream_client, + ), + upstream_client, + ) diff --git a/agent-access/tests/test_cli_output.py b/agent-access/tests/test_cli_output.py index 7e981c9..9ecbed3 100644 --- a/agent-access/tests/test_cli_output.py +++ b/agent-access/tests/test_cli_output.py @@ -2,6 +2,7 @@ import json import sys +from types import SimpleNamespace from typing import Any from codemesh_agent_access import cli @@ -316,18 +317,131 @@ def test_cli_prints_mcp_manifest(monkeypatch: Any, capsys: Any) -> None: ] -def test_cli_uses_normal_mcp_profile_by_default(monkeypatch: Any) -> None: - captured: dict[str, str] = {} +def test_cli_uses_normal_mcp_profile_by_default(monkeypatch: Any, tmp_path) -> None: + captured: dict[str, Any] = {} - def fake_run(profile: str) -> None: + def fake_run(profile: str, binding: Any) -> None: captured["profile"] = profile + captured["binding"] = binding monkeypatch.setattr(cli.mcp, "run", fake_run) - monkeypatch.setattr(sys, "argv", ["codemesh-agent-access", "mcp"]) + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "mcp", + "--project-id", + "project:test", + "--checkout-id", + "checkout:test", + "--repository-root", + str(tmp_path), + ], + ) cli.main() assert captured["profile"] == "normal" + assert captured["binding"].project_id == "project:test" + assert captured["binding"].checkout_id == "checkout:test" + + +def test_cli_runs_configured_agent_preflight_without_model( + monkeypatch: Any, capsys: Any, tmp_path +) -> None: + captured: dict[str, Any] = {} + suite = SimpleNamespace(kind="agent", name="configured") + plan = object() + + def fake_load_suite(value: str, kind: str) -> tuple[Any, Any]: + assert value == "configured-suite.json" + assert kind == "agent" + return suite, tmp_path / "configured-suite.json" + + async def fake_preflight(received_suite: Any, **kwargs: Any) -> dict[str, Any]: + captured["suite"] = received_suite + captured.update(kwargs) + return { + "schema_version": "codemesh-agent-preflight-v1", + "passed": True, + "integration_mode": "configured", + } + + monkeypatch.setattr(cli.evaluation, "load_suite", fake_load_suite) + monkeypatch.setattr(cli, "load_installation_plan", lambda _path: plan) + monkeypatch.setattr(cli.evaluation, "run_agent_preflight", fake_preflight) + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "eval", + "agent", + "--suite", + "configured-suite.json", + "--configured-plan", + "plan.json", + "--configured-guidance-path", + "AGENTS.md", + "--configured-expected-path", + "src/A.py", + "--max-reported-tokens", + "1600000", + "--preflight-only", + ], + ) + + cli.main() + + payload = json.loads(capsys.readouterr().out) + assert payload["passed"] is True + assert captured["suite"] is suite + assert captured["configured_plan"] is plan + assert captured["configured_guidance_paths"] == ["AGENTS.md"] + assert captured["configured_expected_paths"] == ["src/A.py"] + assert captured["max_reported_tokens"] == 1600000 + assert isinstance(captured["runner"], cli.evaluation.CodexRunner) + + +def test_cli_selects_capped_codex_runner_for_agent_preflight( + monkeypatch: Any, capsys: Any, tmp_path +) -> None: + captured: dict[str, Any] = {} + suite = SimpleNamespace(kind="agent", name="configured") + + monkeypatch.setattr( + cli.evaluation, + "load_suite", + lambda _value, _kind: (suite, tmp_path / "suite.json"), + ) + + async def fake_preflight(_suite: Any, **kwargs: Any) -> dict[str, Any]: + captured.update(kwargs) + return {"passed": True} + + monkeypatch.setattr(cli.evaluation, "run_agent_preflight", fake_preflight) + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "eval", + "agent", + "--suite", + "suite.json", + "--runner", + "capped-codex", + "--max-reported-tokens", + "100", + "--preflight-only", + ], + ) + + cli.main() + + assert json.loads(capsys.readouterr().out)["passed"] is True + assert isinstance(captured["runner"], cli.evaluation.CappedCodexRunner) def test_cli_filter_requires_key_value_pair(monkeypatch: Any) -> None: @@ -343,3 +457,18 @@ def test_cli_filter_requires_key_value_pair(monkeypatch: Any) -> None: assert str(exc) == "Invalid --filter value 'broken'. Expected KEY=VALUE." else: raise AssertionError("Expected invalid filter to exit.") + + +def test_version_exits_before_configuration_or_store_access(monkeypatch, capsys): + import pytest + from codemesh_agent_access import __version__ + + def unexpected_settings(*args, **kwargs): + raise AssertionError("version must not configure services") + + monkeypatch.setattr(cli, "Settings", unexpected_settings) + monkeypatch.setattr(sys, "argv", ["codemesh-agent-access", "--version"]) + with pytest.raises(SystemExit) as result: + cli.main() + assert result.value.code == 0 + assert capsys.readouterr().out == f"codemesh-agent-access {__version__}\n" diff --git a/agent-access/tests/test_context_ranking.py b/agent-access/tests/test_context_ranking.py index 7e2df1f..a0ee3bc 100644 --- a/agent-access/tests/test_context_ranking.py +++ b/agent-access/tests/test_context_ranking.py @@ -3,6 +3,8 @@ import asyncio import json +import pytest + from codemesh_agent_access.config import Settings from codemesh_agent_access.graph_store import lexical_terms from codemesh_agent_access.models import ( @@ -17,6 +19,7 @@ from codemesh_agent_access.store import ( CodeMeshReadStore, _diversify_lexical_nodes, + _diversify_ranked_hits, _lexical_score, _prioritize_context_relationships, _ranked_unique_hits, @@ -204,6 +207,69 @@ def test_ranked_unique_hits_keeps_best_expanded_context() -> None: assert _ranked_unique_hits([lower, higher], 5) == [higher] +def test_ranked_hits_cover_distinct_files_before_repeating() -> None: + hits = [ + ContextHit( + node_id="a-1", + stable_key="a-1", + kind="Method", + file_path="A.py", + start_line=1, + end_line=2, + score=0.9, + ), + ContextHit( + node_id="a-2", + stable_key="a-2", + kind="Method", + file_path="A.py", + start_line=3, + end_line=4, + score=0.8, + ), + ContextHit( + node_id="b-1", + stable_key="b-1", + kind="Method", + file_path="B.rs", + start_line=1, + end_line=2, + score=0.7, + ), + ] + + assert [hit.node_id for hit in _diversify_ranked_hits(hits, 3)] == [ + "a-1", + "b-1", + "a-2", + ] + + +def test_ranked_hits_break_saturated_scores_with_raw_relevance() -> None: + incidental = ContextHit( + node_id="incidental", + stable_key="incidental", + kind="Method", + file_path="MediaTagInjector.cs", + start_line=1, + end_line=2, + score=1.0, + score_components={"lexical_score": 0.745}, + ) + exact = ContextHit( + node_id="exact", + stable_key="exact", + kind="Method", + file_path="VideoDownloader.cs", + start_line=100, + end_line=101, + score=1.0, + score_components={"lexical_score": 0.87}, + ) + + assert _ranked_unique_hits([incidental, exact], 2) == [exact, incidental] + + def test_lexical_score_ranks_multi_term_coverage() -> None: query = "repository cleanup removes indexed data" primary = { @@ -242,6 +308,43 @@ def test_lexical_score_prefers_terms_in_symbol_and_file_names() -> None: ) +def test_lexical_score_uses_language_signature_and_inflection_for_native_boundary() -> ( + None +): + query = ( + "Allow four workers in native feed-hub replay parallel processing across " + "Python PyO3 Rust telemetry evidence and tests" + ) + rust_boundary = { + "id": "rust:fn:crate::native::feed_hub_replay::src:process_mbo_v3_parallel", + "stableKey": ( + "rust:fn:crate::native::feed_hub_replay::src:process_mbo_v3_parallel" + ), + "name": "process_mbo_v3_parallel", + "filePath": "native/feed_hub_replay/src/lib.rs", + "language": "rust", + "metadataJson": json.dumps( + { + "signature": ( + "fn process_mbo_v3_parallel(py: Python, worker_count: usize)" + ), + "attributes": "#[allow(clippy::too_many_arguments)]", + } + ), + } + broad_python_test = { + "id": "python:testcase:tests.test_feed_hub_native_replay_adapter", + "stableKey": "python:testcase:tests.test_feed_hub_native_replay_adapter", + "name": "test_adapter_validates_parallel_limits_and_telemetry", + "filePath": "tests/test_feed_hub_native_replay_adapter.py", + "language": "python", + } + + assert _lexical_score(rust_boundary, query) > _lexical_score( + broad_python_test, query + ) + + def test_lexical_terms_remove_grammatical_noise() -> None: assert lexical_terms("trace the callers of a method and its writes") == [ "trace", @@ -252,6 +355,17 @@ def test_lexical_terms_remove_grammatical_noise() -> None: ] +def test_lexical_terms_split_language_identifiers_for_file_name_matching() -> None: + assert lexical_terms("NativeReplayAdapter processMboV3") == [ + "native", + "replay", + "adapter", + "process", + "mbo", + "v3", + ] + + def test_lexical_candidates_cover_distinct_files_before_repeating() -> None: nodes = [ {"id": "a-1", "filePath": "A.cs"}, @@ -407,6 +521,134 @@ def test_context_package_includes_snippet_relationships_and_registry_metadata() assert item.rationale["expanded"] is True +def test_context_package_emits_sanitized_stage_timings_with_injected_delays( + monkeypatch, capsys +) -> None: + monkeypatch.setenv("CODEMESH_CONTEXT_PACKAGE_TIMINGS", "1") + clock = AdvancingClock() + store = TimingReadStore(clock) + + package = asyncio.run( + store.get_context_package( + ContextPackageQuery( + query="secret query text", + repository_id="repo:test", + limit=1, + vector=[1.0, 0.0], + ) + ) + ) + + assert len(package.items) == 1 + timing_line = next( + line + for line in capsys.readouterr().err.splitlines() + if line.startswith("CODEMESH_CONTEXT_PACKAGE_TIMING ") + ) + payload = json.loads(timing_line.partition(" ")[2]) + assert payload["event"] == "context_package_timing" + assert payload["outcome"] == "ok" + assert payload["counts"]["content_fallbacks"] == 1 + assert payload["counts"]["hits"] == 1 + assert payload["counts"]["hydration_concurrency"] == 1 + assert payload["stages_ms"]["repository_resolution"] >= 1 + assert payload["stages_ms"]["search"] >= 2 + assert payload["stages_ms"]["repository_metadata"] >= 7 + assert payload["stages_ms"]["item_hydration"] >= 18 + assert payload["operations_ms"]["node"] >= 5 + assert payload["operations_ms"]["summary"] >= 6 + assert payload["operations_ms"]["relationships"] >= 7 + assert "secret query text" not in timing_line + assert "Downloader.cs" not in timing_line + + +def test_search_timings_attribute_candidate_work_without_changing_results( + monkeypatch, capsys +) -> None: + store = SearchTimingReadStore() + query = ContextPackageQuery( + query="download secret query", repository_id="repo:test" + ) + monkeypatch.delenv("CODEMESH_CONTEXT_PACKAGE_TIMINGS", raising=False) + baseline = asyncio.run(store.get_context_package(query)) + assert capsys.readouterr().err == "" + + monkeypatch.setenv("CODEMESH_CONTEXT_PACKAGE_TIMINGS", "1") + measured = asyncio.run(store.get_context_package(query)) + stderr = capsys.readouterr().err + payload = json.loads(stderr.partition(" ")[2]) + + assert measured.model_dump(exclude={"generated_at"}) == baseline.model_dump( + exclude={"generated_at"} + ) + for name, expected in { + "search_lexical_fetch": 2, + "search_expansion": 3, + "search_declaration": 5, + "search_content": 7, + }.items(): + assert payload["operations_ms"][name] == expected + assert payload["counts"][f"{name}_calls"] == 1 + assert payload["operations_ms"]["search_lexical_pipeline"] == 17 + assert payload["stages_ms"]["search"] == 17 + assert "secret query" not in stderr + assert "Downloader.cs" not in stderr + assert "repo:test" not in stderr + + +def test_search_timings_isolate_overlapping_requests_and_reset_after_failure( + monkeypatch, capsys +) -> None: + monkeypatch.setenv("CODEMESH_CONTEXT_PACKAGE_TIMINGS", "1") + + async def exercise() -> None: + started = asyncio.Event() + release = asyncio.Event() + slow = SearchTimingReadStore() + fast = slow + original = slow._get_context_candidates + pause_next = True + + async def paused_expansion(*args, **kwargs): + nonlocal pause_next + if pause_next: + pause_next = False + started.set() + await release.wait() + return await original(*args, **kwargs) + + monkeypatch.setattr(slow, "_get_context_candidates", paused_expansion) + query = ContextPackageQuery(query="download", repository_id="repo:test") + pending = asyncio.create_task(slow.get_context_package(query)) + await started.wait() + await fast.get_context_package(query) + release.set() + await pending + + async def failing_expansion(*args, **kwargs): + raise RuntimeError("synthetic search failure") + + monkeypatch.setattr(fast, "_get_context_candidates", failing_expansion) + with pytest.raises(RuntimeError, match="synthetic search failure"): + await fast.get_context_package(query) + from codemesh_agent_access.store import _active_package_timing + + assert _active_package_timing.get() is None + + asyncio.run(exercise()) + records = [ + json.loads(line.partition(" ")[2]) + for line in capsys.readouterr().err.splitlines() + ] + assert [record["outcome"] for record in records] == ["ok", "ok", "failed"] + for record in records[:2]: + assert record["operations_ms"]["search_expansion"] >= 3 + assert record["counts"]["search_expansion_calls"] == 1 + assert record["counts"]["search_content_calls"] == 1 + assert record["operations_ms"]["search_lexical_pipeline"] >= 17 + assert records[2]["counts"]["search_expansion_calls"] == 1 + + def test_context_package_accepts_repository_alias() -> None: store = FakeReadStore() @@ -815,6 +1057,98 @@ async def list_ingestion_runs( ) +class AdvancingClock: + def __init__(self) -> None: + self.value = 0.0 + + def __call__(self) -> float: + return self.value + + def advance(self, seconds: float) -> None: + self.value += seconds + + +class SearchTimingReadStore(FakeReadStore): + def __init__(self) -> None: + super().__init__() + self.clock = AdvancingClock() + self._context_package_clock = self.clock + self._graph.search_context_nodes = self._search_nodes + + async def _search_nodes(self, *args): + self.clock.advance(0.002) + return [self.parameter_declaration] + + async def _search_lexical_context(self, query): + return await CodeMeshReadStore._search_lexical_context(self, query) + + async def _get_context_candidates(self, *args, **kwargs): + self.clock.advance(0.003) + return await super()._get_context_candidates(*args, **kwargs) + + async def _get_primary_declaration_node(self, *args, **kwargs): + self.clock.advance(0.005) + return await super()._get_primary_declaration_node(*args, **kwargs) + + async def _get_content(self, *args, **kwargs): + self.clock.advance(0.007) + return await super()._get_content(*args, **kwargs) + + +class TimingReadStore(FakeReadStore): + def __init__(self, clock: AdvancingClock) -> None: + super().__init__() + self.clock = clock + self._context_package_clock = clock + + async def _resolve_package_query_repository( + self, query: ContextPackageQuery + ) -> ContextPackageQuery: + self.clock.advance(0.001) + return await super()._resolve_package_query_repository(query) + + async def search_context(self, query: ContextQuery): + self.clock.advance(0.002) + return await super().search_context(query) + + async def get_repository(self, repository_id: str) -> RepositorySummary | None: + self.clock.advance(0.003) + return await super().get_repository(repository_id) + + async def list_ingestion_runs( + self, + repository_id: str | None = None, + limit: int = 50, + ) -> IngestionRunListResponse: + self.clock.advance(0.004) + return await super().list_ingestion_runs(repository_id, limit) + + async def get_node(self, lookup, binding=None): + self.clock.advance(0.005) + return await super().get_node(lookup, binding) + + async def _get_node_summary( + self, + repository_id: str | None, + node_id: str, + content_hash: str | None = None, + ) -> dict[str, object] | None: + self.clock.advance(0.006) + return await super()._get_node_summary(repository_id, node_id, content_hash) + + async def _get_context_relationship_summaries( + self, + node_id: str, + repository_id: str | None = None, + limit: int = 8, + group_limits: dict[str, int] | None = None, + ) -> list[dict[str, object]]: + self.clock.advance(0.007) + return await super()._get_context_relationship_summaries( + node_id, repository_id, limit, group_limits + ) + + class SnapshotReadStore(FakeReadStore): def __init__(self) -> None: super().__init__() diff --git a/agent-access/tests/test_development_feedback.py b/agent-access/tests/test_development_feedback.py new file mode 100644 index 0000000..5349524 --- /dev/null +++ b/agent-access/tests/test_development_feedback.py @@ -0,0 +1,465 @@ +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime, timedelta +import json +from pathlib import Path +import subprocess + +import pytest +from mcp.server.fastmcp.exceptions import ToolError + +from codemesh_agent_access import feedback as feedback_module +from codemesh_agent_access import mcp +from codemesh_agent_access.binding import RepositoryBinding +from codemesh_agent_access.feedback import ( + FeedbackPacketV2, + get_session_feedback, + list_session_feedback, + prepare_feedback_resolution, + record_development_feedback, + validate_feedback, +) +from codemesh_agent_access.feedback_session import ( + CLIENT_PROFILE, + MAINTAINER_PROFILE, + SessionBounds, + SessionParticipant, + activate_session, + create_session_plan, + create_session_runtime, +) +from codemesh_agent_access.models import ToolResult + + +NOW = datetime(2026, 9, 10, 9, 0, tzinfo=UTC) + + +def test_records_v2_with_server_derived_session_and_repository_provenance( + tmp_path: Path, +) -> None: + client_runtime, _maintainer_runtime, client, _codemesh = _session(tmp_path) + + packet, output = _record(client_runtime) + + assert packet.schema_version == "codemesh-feedback/v2" + assert packet.classification == "diagnostic" + assert packet.session.session_id == client_runtime.payload.session_id + assert packet.repository.root == str(client) + assert packet.codemesh.project_id == "project:client" + assert packet.codemesh.checkout_id == "checkout:client" + assert packet.codemesh.binding.status == "accepted" + assert packet.codemesh.languages == ["csharp", "python"] + assert packet.task.recent_tool_events[0].tool == "codemesh_get_context_package" + assert validate_feedback(output) == packet + + +def test_content_feedback_requires_accepted_provenance_but_setup_does_not( + tmp_path: Path, +) -> None: + client_runtime, _maintainer_runtime, _client, _codemesh = _session(tmp_path) + unavailable = { + "status": "unavailable", + "diagnostic_code": "binding_rejected", + } + + with pytest.raises(ValueError, match="requires accepted snapshot"): + _record(client_runtime, provenance=unavailable) + + packet, _output = _record( + client_runtime, + provenance=unavailable, + issue_category="binding", + ) + assert packet.codemesh.snapshot_id is None + assert packet.codemesh.binding.diagnostic_code == "binding_rejected" + + +def test_v2_validator_reapplies_sanitization_to_handcrafted_packet( + tmp_path: Path, +) -> None: + client_runtime, _maintainer_runtime, _client, _codemesh = _session(tmp_path) + _packet, output = _record(client_runtime) + payload = json.loads(output.read_text(encoding="utf-8")) + payload["result"]["fallback_reason"] = "token=do-not-retain" + unsigned = {key: value for key, value in payload.items() if key != "feedback_id"} + payload["feedback_id"] = feedback_module._content_id(unsigned) + output.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="credential or secret"): + validate_feedback(output) + + +def test_maintainer_lists_gets_and_prepares_stable_non_authorizing_plan( + tmp_path: Path, +) -> None: + client_runtime, maintainer_runtime, _client, _codemesh = _session(tmp_path) + failed, _ = _record(client_runtime) + passed, _ = _record( + client_runtime, + validation_outcome="passed", + supersedes_feedback_ids=[failed.feedback_id], + ) + + listing = list_session_feedback(maintainer_runtime) + assert listing["total"] == 2 + states = {item["feedback_id"]: item["state"] for item in listing["items"]} + assert states[failed.feedback_id] == "resolved" + assert states[passed.feedback_id] == "verified" + + detail = get_session_feedback(maintainer_runtime, failed.feedback_id) + assert detail["state"] == "resolved" + assert detail["session_scope"] == "active_session" + assert detail["packet"]["feedback_id"] == failed.feedback_id + + arguments = { + "runtime": maintainer_runtime, + "feedback_ids": [failed.feedback_id], + "reproduction_state": "reproduced", + "change_summary": "Preserve the missing relationship during retrieval.", + "proposed_paths": ["src/retrieval.py"], + "risks": ["Could alter ranking."], + "required_tests": ["Run the relationship fixture."], + } + first = prepare_feedback_resolution(**arguments) + second = prepare_feedback_resolution(**arguments) + assert first == second + assert first["human_review_required"] is True + assert first["human_approved"] is False + assert len(first["plan_hash"]) == 64 + + +def test_only_matching_passing_rechecks_resolve_feedback(tmp_path: Path) -> None: + client_runtime, maintainer_runtime, _client, _codemesh = _session(tmp_path) + failed, _ = _record(client_runtime) + + with pytest.raises(ValueError, match="Only a passing recheck"): + _record( + client_runtime, + validation_outcome="failed", + supersedes_feedback_ids=[failed.feedback_id], + ) + + _record( + client_runtime, + issue_category="ranking", + validation_outcome="passed", + supersedes_feedback_ids=[failed.feedback_id], + ) + listing = list_session_feedback(maintainer_runtime) + states = {item["feedback_id"]: item["state"] for item in listing["items"]} + assert states[failed.feedback_id] == "open" + + +def test_maintainer_rejects_outside_ids_and_proposal_paths(tmp_path: Path) -> None: + client_runtime, maintainer_runtime, _client, _codemesh = _session(tmp_path) + packet, _ = _record(client_runtime) + + with pytest.raises(ValueError, match="not found"): + get_session_feedback(maintainer_runtime, "feedback-" + "a" * 20) + + with pytest.raises(ValueError, match="repository-relative"): + prepare_feedback_resolution( + maintainer_runtime, + feedback_ids=[packet.feedback_id], + reproduction_state="reproduced", + change_summary="Test unsafe scope.", + proposed_paths=[str(tmp_path / "outside.py")], + ) + + +def test_maintainer_reports_linked_packet_as_invalid_without_reading_it( + tmp_path: Path, +) -> None: + client_runtime, maintainer_runtime, client, _codemesh = _session(tmp_path) + _record(client_runtime) + outside = tmp_path / "outside.json" + outside.write_text('{"secret":"do not inspect"}', encoding="utf-8") + (client / ".codemesh-feedback" / "linked.json").symlink_to(outside) + + listing = list_session_feedback(maintainer_runtime) + + assert listing["total"] == 1 + assert listing["invalid_packet_count"] == 1 + assert listing["invalid_packets"][0]["code"] == "invalid_packet" + + +def test_mcp_feedback_profiles_are_exact_and_record_through_the_client_tool( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client_runtime, maintainer_runtime, client, _codemesh = _session(tmp_path) + + async def accepted_status(repository_id: str, **_kwargs) -> ToolResult: + return ToolResult( + data={ + "repository_id": repository_id, + "repository": {"snapshot_id": "snapshot:mcp"}, + "latest_run": { + "language": "python", + "parser_name": "python-tree-sitter", + }, + "freshness": { + "status": "fresh", + "is_stale": False, + "binding": {"status": "accepted", "issues": []}, + }, + } + ) + + monkeypatch.setattr(mcp.tools, "get_repository_status", accepted_status) + client_server = mcp.build_server( + profile=CLIENT_PROFILE, + binding=client_runtime.binding, + feedback_runtime=client_runtime, + ) + client_tools = {tool.name: tool for tool in asyncio.run(client_server.list_tools())} + assert set(client_tools) == set(client_runtime.payload.profiles[CLIENT_PROFILE]) + assert client_tools["codemesh_record_feedback"].annotations.readOnlyHint is False + assert client_tools["codemesh_record_feedback"].annotations.destructiveHint is False + assert ( + client_tools["codemesh_record_feedback"].inputSchema["additionalProperties"] + is False + ) + assert "explicitly enabled" in client_server.instructions + + asyncio.run(client_server.call_tool("codemesh_get_repository_status", {})) + + _content, recorded = asyncio.run( + client_server.call_tool( + "codemesh_record_feedback", + { + "confidence": "high", + "task_family": "discovery", + "issue_category": "ranking", + "validation_outcome": "failed", + "missed_paths": ["src/retrieval.py"], + }, + ) + ) + assert recorded["recorded"] is True + assert recorded["packet_path"].startswith(".codemesh-feedback/") + assert Path(client, recorded["packet_path"]).is_file() + recorded_packet = validate_feedback(Path(client, recorded["packet_path"])) + assert isinstance(recorded_packet, FeedbackPacketV2) + assert recorded_packet.task.recent_tool_events[0].tool == ( + "codemesh_get_repository_status" + ) + with pytest.raises(ToolError, match="classification|repository_root"): + asyncio.run( + client_server.call_tool( + "codemesh_record_feedback", + { + "confidence": "high", + "task_family": "discovery", + "issue_category": "binding", + "validation_outcome": "failed", + "classification": "controlled-evaluation", + "repository_root": str(client), + }, + ) + ) + + maintainer_server = mcp.build_server( + profile=MAINTAINER_PROFILE, + binding=maintainer_runtime.binding, + feedback_runtime=maintainer_runtime, + ) + maintainer_tools = { + tool.name: tool for tool in asyncio.run(maintainer_server.list_tools()) + } + assert set(maintainer_tools) == set( + maintainer_runtime.payload.profiles[MAINTAINER_PROFILE] + ) + assert all(tool.annotations.readOnlyHint for tool in maintainer_tools.values()) + _content, listing = asyncio.run( + maintainer_server.call_tool("codemesh_list_feedback", {}) + ) + assert listing["total"] == 1 + + +def test_feedback_mcp_profiles_fail_closed_without_session_runtime() -> None: + binding = RepositoryBinding("project:test", "checkout:test", "/does/not/matter") + for profile in (CLIENT_PROFILE, MAINTAINER_PROFILE): + with pytest.raises(ValueError, match="validated session runtime"): + mcp.build_server(profile=profile, binding=binding) + + +def test_agent_format_context_package_supplies_feedback_provenance( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client_runtime, _maintainer_runtime, client, _codemesh = _session(tmp_path) + + async def context_package(query: str, **_kwargs) -> ToolResult: + return ToolResult( + data={ + "query": {"query": query}, + "repository": {"snapshot_id": "snapshot:agent"}, + "latest_run": { + "language": "python,rust", + "parser_name": "composite-v1", + }, + "items": [], + "validation_recommendations": [], + "total_snippet_characters": 0, + "generated_at": NOW.isoformat(), + } + ) + + monkeypatch.setattr(mcp.tools, "get_context_package", context_package) + server = mcp.build_server( + profile=CLIENT_PROFILE, + binding=client_runtime.binding, + feedback_runtime=client_runtime, + ) + package = asyncio.run( + server.call_tool( + "codemesh_get_context_package", + {"query": "find bridge", "output_format": "agent"}, + ) + ) + assert package.isError is False + _content, recorded = asyncio.run( + server.call_tool( + "codemesh_record_feedback", + { + "confidence": "high", + "task_family": "cross-language-impact", + "issue_category": "missing-relationship", + "validation_outcome": "failed", + "missed_paths": ["src/retrieval.py"], + }, + ) + ) + packet = validate_feedback(client / recorded["packet_path"]) + assert isinstance(packet, FeedbackPacketV2) + assert packet.codemesh.snapshot_id == "snapshot:agent" + assert packet.codemesh.languages == ["python", "rust"] + + +def test_feedback_session_enforces_request_and_packet_bounds(tmp_path: Path) -> None: + bounds = SessionBounds(max_requests_per_minute=1, max_packets=1) + client_runtime, _maintainer_runtime, _client, _codemesh = _session( + tmp_path, + bounds=bounds, + ) + server = mcp.build_server( + profile=CLIENT_PROFILE, + binding=client_runtime.binding, + feedback_runtime=client_runtime, + monotonic_clock=lambda: 10.0, + ) + asyncio.run(server.call_tool("codemesh_get_feedback_session", {})) + with pytest.raises(ToolError, match="request-rate limit"): + asyncio.run(server.call_tool("codemesh_get_feedback_session", {})) + + unavailable = {"status": "unavailable", "diagnostic_code": "binding_rejected"} + _record(client_runtime, issue_category="binding", provenance=unavailable) + with pytest.raises(ValueError, match="packet limit"): + _record( + client_runtime, + issue_category="binding", + provenance=unavailable, + ) + + +def _record( + runtime, + *, + provenance=None, + issue_category="missing-relationship", + validation_outcome="failed", + supersedes_feedback_ids=None, +) -> tuple[FeedbackPacketV2, Path]: + return record_development_feedback( + runtime=runtime, + confidence="high", + task_family="change-impact", + issue_category=issue_category, + validation_outcome=validation_outcome, + provenance=provenance + or { + "status": "accepted", + "snapshot_id": "snapshot:test", + "languages": ["python", "csharp"], + "parser_profile": "mixed-v1", + }, + recent_tool_events=[ + { + "sequence": 1, + "tool": "codemesh_get_context_package", + "outcome": "succeeded", + } + ], + missed_paths=["src/retrieval.py"], + minimal_reproduction="Request callers for the selected method.", + proposed_correction="Include the missing caller edge.", + supersedes_feedback_ids=supersedes_feedback_ids, + ) + + +def _session(tmp_path: Path, *, bounds: SessionBounds | None = None): + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=True) + plan = create_session_plan( + manifest_path=tmp_path / "session.json", + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[_participant(client, "client", "Primary")], + expires_at=NOW + timedelta(hours=4), + bounds=bounds, + now=NOW, + ) + activate_session(plan, plan.plan_hash) + client_runtime = create_session_runtime( + plan.manifest_path, + plan.manifest_file_sha256, + profile=CLIENT_PROFILE, + binding=RepositoryBinding( + "project:client", + "checkout:client", + str(client), + ), + clock=lambda: NOW, + ) + maintainer_runtime = create_session_runtime( + plan.manifest_path, + plan.manifest_file_sha256, + profile=MAINTAINER_PROFILE, + binding=RepositoryBinding( + "project:codemesh", + "checkout:codemesh", + str(codemesh), + ), + clock=lambda: NOW, + ) + return client_runtime, maintainer_runtime, client, codemesh + + +def _participant(root: Path, name: str, role: str) -> SessionParticipant: + return SessionParticipant( + project_id=f"project:{name}", + checkout_id=f"checkout:{name}", + repository_root=str(root), + reporter_role=role, + ) + + +def _repository(path: Path, *, ignored: bool) -> Path: + path.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=path, check=True) + subprocess.run( + ["git", "config", "user.email", "codemesh-tests@example.invalid"], + cwd=path, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "CodeMesh Tests"], cwd=path, check=True + ) + (path / "src").mkdir() + (path / "src" / "retrieval.py").write_text("pass\n", encoding="utf-8") + if ignored: + (path / ".gitignore").write_text("/.codemesh-feedback/\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=path, check=True) + subprocess.run(["git", "commit", "-m", "test: fixture"], cwd=path, check=True) + return path.resolve() diff --git a/agent-access/tests/test_development_feedback_integration.py b/agent-access/tests/test_development_feedback_integration.py new file mode 100644 index 0000000..f13bcfc --- /dev/null +++ b/agent-access/tests/test_development_feedback_integration.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +import json +import os +from pathlib import Path +import subprocess +from typing import Any + +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + +from codemesh_agent_access.binding import RepositoryBinding +from codemesh_agent_access.feedback_session import ( + CLIENT_PROFILE, + MAINTAINER_PROFILE, + SessionParticipant, + activate_session, + create_session_plan, + revoke_session, +) +from codemesh_agent_access.installer import create_installation_plan + + +def test_provider_free_stdio_failure_to_recheck_flow(tmp_path: Path) -> None: + asyncio.run(_exercise_provider_free_stdio_flow(tmp_path)) + + +async def _exercise_provider_free_stdio_flow(tmp_path: Path) -> None: + codemesh_root = Path(__file__).resolve().parents[2] + client = _repository(tmp_path / "client") + client_binding = RepositoryBinding( + "project:client", + "checkout:client", + str(client), + ) + maintainer_binding = RepositoryBinding( + "project:codemesh", + "checkout:codemesh", + str(codemesh_root), + ) + session_plan = create_session_plan( + manifest_path=tmp_path / "session.json", + codemesh=SessionParticipant( + project_id=maintainer_binding.project_id, + checkout_id=maintainer_binding.checkout_id, + repository_root=maintainer_binding.repository_root, + reporter_role="maintainer", + ), + clients=[ + SessionParticipant( + project_id=client_binding.project_id, + checkout_id=client_binding.checkout_id, + repository_root=client_binding.repository_root, + reporter_role="integration-client", + ) + ], + expires_at=datetime.now(UTC) + timedelta(minutes=10), + ) + activate_session(session_plan, session_plan.plan_hash) + client_plan = create_installation_plan( + client, + codemesh_root, + client_binding, + profile=CLIENT_PROFILE, + include_onboarding=False, + feedback_session_path=session_plan.manifest_path, + feedback_session_sha256=session_plan.manifest_file_sha256, + ) + maintainer_plan = create_installation_plan( + codemesh_root, + codemesh_root, + maintainer_binding, + profile=MAINTAINER_PROFILE, + include_onboarding=False, + feedback_session_path=session_plan.manifest_path, + feedback_session_sha256=session_plan.manifest_file_sha256, + ) + before = _git_status(codemesh_root) + + async with _session(client_plan) as client_session: + discovery = _payload( + await client_session.call_tool("codemesh_get_feedback_session", {}) + ) + assert discovery["session_id"] == session_plan.manifest.payload.session_id + failed = _payload( + await client_session.call_tool( + "codemesh_record_feedback", + { + "confidence": "high", + "task_family": "agent-access-setup", + "issue_category": "binding", + "validation_outcome": "failed", + "fallback_reason": "The configured binding was rejected.", + }, + ) + ) + assert failed["human_review_required"] is True + + async with _session(maintainer_plan) as maintainer_session: + listing = _payload( + await maintainer_session.call_tool("codemesh_list_feedback", {}) + ) + assert listing["total"] == 1 + detail = _payload( + await maintainer_session.call_tool( + "codemesh_get_feedback", + {"feedback_id": failed["feedback_id"]}, + ) + ) + assert detail["packet"]["feedback_id"] == failed["feedback_id"] + resolution = _payload( + await maintainer_session.call_tool( + "codemesh_prepare_feedback_resolution", + { + "feedback_ids": [failed["feedback_id"]], + "reproduction_state": "reproduced", + "change_summary": "Clarify rejected binding diagnostics.", + "proposed_paths": ["agent-access/codemesh_agent_access/binding.py"], + "required_tests": ["Run the Agent Access suite."], + }, + ) + ) + assert resolution["human_approved"] is False + assert len(resolution["plan_hash"]) == 64 + + async with _session(client_plan) as client_session: + passed = _payload( + await client_session.call_tool( + "codemesh_record_feedback", + { + "confidence": "high", + "task_family": "agent-access-setup", + "issue_category": "binding", + "validation_outcome": "passed", + "supersedes_feedback_ids": [failed["feedback_id"]], + }, + ) + ) + assert passed["feedback_id"] != failed["feedback_id"] + + async with _session(maintainer_plan) as maintainer_session: + listing = _payload( + await maintainer_session.call_tool("codemesh_list_feedback", {}) + ) + states = {item["feedback_id"]: item["state"] for item in listing["items"]} + assert states[failed["feedback_id"]] == "resolved" + + revoke_session( + session_plan.manifest_path, + session_plan.manifest_file_sha256, + reason="Integration rehearsal complete.", + ) + rejected = await client_session.call_tool("codemesh_get_feedback_session", {}) + assert rejected.isError is True + + assert _git_status(codemesh_root) == before + assert len(list((client / ".codemesh-feedback").glob("feedback-*.json"))) == 2 + + +@asynccontextmanager +async def _session(plan: Any): + launch = plan.launch + parameters = StdioServerParameters( + command=str(launch["command"]), + args=[str(value) for value in launch["args"]], + cwd=Path(str(launch["cwd"])), + env={**os.environ, "CODEMESH_MODEL_PROVIDER": "none"}, + ) + async with stdio_client(parameters) as (read_stream, write_stream): + async with ClientSession(read_stream, write_stream) as session: + await session.initialize() + yield session + + +def _payload(result: Any) -> dict[str, Any]: + assert result.isError is False + if isinstance(result.structuredContent, dict): + return result.structuredContent + for item in result.content: + text = getattr(item, "text", None) + if text: + payload = json.loads(text) + if isinstance(payload, dict): + return payload + raise AssertionError("MCP tool returned no JSON payload.") + + +def _git_status(root: Path) -> bytes: + return subprocess.run( + ["git", "status", "--porcelain=v1", "-z", "--untracked-files=normal"], + cwd=root, + check=True, + capture_output=True, + ).stdout + + +def _repository(path: Path) -> Path: + path.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=path, check=True) + subprocess.run( + ["git", "config", "user.email", "codemesh-tests@example.invalid"], + cwd=path, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "CodeMesh Tests"], cwd=path, check=True + ) + (path / ".gitignore").write_text("/.codemesh-feedback/\n", encoding="utf-8") + (path / "README.md").write_text("# Client fixture\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=path, check=True) + subprocess.run(["git", "commit", "-m", "test: fixture"], cwd=path, check=True) + return path.resolve() diff --git a/agent-access/tests/test_effectiveness_evaluation.py b/agent-access/tests/test_effectiveness_evaluation.py index ac1ffbf..af5982f 100644 --- a/agent-access/tests/test_effectiveness_evaluation.py +++ b/agent-access/tests/test_effectiveness_evaluation.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import hashlib import json import subprocess from pathlib import Path @@ -10,24 +11,38 @@ import pytest from pydantic import ValidationError +import codemesh_agent_access.evaluation.agent as agent_evaluation +from codemesh_agent_access.binding import RepositoryBinding from codemesh_agent_access.evaluation.agent import ( + AgentEvaluationContext, + ConfiguredGuidanceFile, _condition_prompt, _grade_answer, _grade_command_policy, + _prepare_agent_context, + _prepare_configured_integration, _run_task_condition, classify_agent_results, + run_agent_evaluation, + run_agent_preflight, ) from codemesh_agent_access.evaluation.codex import ( + REPORTED_TOKEN_ACCOUNTING_ID, AgentRunRequest, AgentRunResult, + AgentRunnerCapabilities, CodexRunner, + count_reported_tokens, parse_codex_jsonl, sanitize_text, ) from codemesh_agent_access.evaluation.live import ( EvaluationInfrastructureError, McpEvaluationClient, + _context_package_timing_summary, + _parse_context_package_timings, _run_live_case, + get_repository_preflight, get_tool_source_identity, ) from codemesh_agent_access.evaluation.metrics import ( @@ -47,12 +62,19 @@ load_suite, suite_json_schema, ) +from codemesh_agent_access.installer import ( + apply_installation_plan, + create_installation_plan, +) def test_built_in_suites_and_schemas_are_valid() -> None: live, live_path = load_suite("codemesh-live", "live") external_live, external_live_path = load_suite("youtube-downloader-live", "live") impact_live, impact_live_path = load_suite("youtube-downloader-impact-live", "live") + adaptive_impact_live, adaptive_impact_live_path = load_suite( + "youtube-downloader-impact-adaptive-live", "live" + ) agent, agent_path = load_suite("codemesh-agent", "agent") external_agent, external_agent_path = load_suite( "youtube-downloader-agent", "agent" @@ -63,15 +85,27 @@ def test_built_in_suites_and_schemas_are_valid() -> None: assisted_agent, assisted_agent_path = load_suite( "youtube-downloader-impact-assisted-agent", "agent" ) + configured_impact_agent, configured_impact_agent_path = load_suite( + "youtube-downloader-impact-configured-agent", "agent" + ) independent_live, independent_live_path = load_suite( "config-net-cache-live", "live" ) + independent_focused_live, independent_focused_live_path = load_suite( + "config-net-cache-focused-live", "live" + ) independent_agent, independent_agent_path = load_suite( "config-net-cache-agent", "agent" ) independent_assisted_agent, independent_assisted_agent_path = load_suite( "config-net-cache-assisted-agent", "agent" ) + independent_configured_agent, independent_configured_agent_path = load_suite( + "config-net-cache-configured-agent", "agent" + ) + onenine_configured_agent, onenine_configured_agent_path = load_suite( + "onenine-native-replay-configured-agent", "agent" + ) model, model_path = load_suite("codemesh-model", "model-benchmark") assert live.name == "codemesh-live" @@ -80,6 +114,8 @@ def test_built_in_suites_and_schemas_are_valid() -> None: assert external_live.repository_commit assert len(impact_live.cases) == 2 assert impact_live.repository_commit == external_live.repository_commit + assert len(adaptive_impact_live.cases) == 5 + assert adaptive_impact_live.repository_commit == external_live.repository_commit assert agent.name == "codemesh-agent" assert {task.task_type for task in agent.tasks} == {"answer", "change"} assert len(external_agent.tasks) == 3 @@ -88,13 +124,46 @@ def test_built_in_suites_and_schemas_are_valid() -> None: assert impact_agent.repository_commit == external_live.repository_commit assert assisted_agent.treatment_guidance assert assisted_agent.repository_commit == external_live.repository_commit + assert configured_impact_agent.resolved_integration_mode == "configured" + assert configured_impact_agent.repository_commit == external_live.repository_commit + assert configured_impact_agent.treatment_guidance is None + assert configured_impact_agent.tasks == impact_agent.tasks assert independent_agent.treatment_guidance is None assert independent_agent.repository_commit == independent_live.repository_commit + assert independent_focused_live.repository_commit == ( + independent_live.repository_commit + ) + assert len(independent_focused_live.cases) == 5 + assert { + target.file_path + for case in independent_focused_live.cases + for target in case.targets + } == { + "src/Config.Net/Core/IoHandler.cs", + "src/Config.Net/Core/LazyVar.cs", + "src/Config.Net/Core/DynamicWriter.cs", + "src/Config.Net/Core/InterfaceInterceptor.cs", + "src/Config.Net/ConfigurationBuilder.cs", + "src/Config.Net.Tests/LogicTest.cs", + "src/Config.Net.Tests/ConfigurableMethodsTest.cs", + } assert independent_assisted_agent.treatment_guidance assert ( independent_assisted_agent.repository_commit == independent_live.repository_commit ) + assert independent_configured_agent.resolved_integration_mode == "configured" + assert independent_configured_agent.repository_commit == ( + independent_live.repository_commit + ) + assert independent_configured_agent.treatment_guidance is None + assert len(independent_configured_agent.tasks) == 1 + assert onenine_configured_agent.resolved_integration_mode == "configured" + assert onenine_configured_agent.repository_commit == ( + "fe2f761583bf78601961ff17934185c4b6a632f9" + ) + assert onenine_configured_agent.treatment_guidance is None + assert len(onenine_configured_agent.tasks) == 1 assert independent_live.repository_url != external_live.repository_url assert model.name == "codemesh-model" assert len(model.questions) == 24 @@ -102,13 +171,18 @@ def test_built_in_suites_and_schemas_are_valid() -> None: assert live_path.is_file() assert external_live_path.is_file() assert impact_live_path.is_file() + assert adaptive_impact_live_path.is_file() assert agent_path.is_file() assert external_agent_path.is_file() assert impact_agent_path.is_file() assert assisted_agent_path.is_file() + assert configured_impact_agent_path.is_file() + assert independent_configured_agent_path.is_file() assert independent_live_path.is_file() + assert independent_focused_live_path.is_file() assert independent_agent_path.is_file() assert independent_assisted_agent_path.is_file() + assert onenine_configured_agent_path.is_file() assert model_path.is_file() assert suite_json_schema("live")["title"] == "LiveSuite" assert suite_json_schema("agent")["title"] == "AgentSuite" @@ -142,6 +216,462 @@ def test_suite_models_reject_incomplete_targets_and_change_tasks() -> None: ) +def test_agent_suite_integration_modes_preserve_prompt_contracts() -> None: + task = AgentTask( + id="answer", + description="answer", + task_type="answer", + prompt="answer", + targets=[RetrievalTarget(file_path="src/A.cs")], + ) + + assert AgentSuite(name="default", tasks=[task]).resolved_integration_mode == ( + "spontaneous" + ) + assert ( + AgentSuite( + name="assisted", + tasks=[task], + treatment_guidance="Use CodeMesh first.", + ).resolved_integration_mode + == "assisted" + ) + configured = AgentSuite( + name="configured", + tasks=[task], + integration_mode="configured", + repository_commit="commit:test", + baseline_mcp_servers=["graphify"], + ) + assert configured.resolved_integration_mode == "configured" + assert configured.baseline_mcp_servers == ["graphify"] + assert _condition_prompt(task.prompt, "control", None) == task.prompt + assert _condition_prompt(task.prompt, "treatment", None) == task.prompt + + with pytest.raises(ValidationError, match="shipped repository guidance"): + AgentSuite( + name="invalid-configured", + tasks=[task], + integration_mode="configured", + repository_commit="commit:test", + treatment_guidance="Evaluator-only guidance.", + ) + with pytest.raises(ValidationError, match="require evaluator treatment guidance"): + AgentSuite( + name="invalid-assisted", + tasks=[task], + integration_mode="assisted", + ) + with pytest.raises(ValidationError, match="pinned repository commit"): + AgentSuite( + name="unpinned-configured", + tasks=[task], + integration_mode="configured", + ) + with pytest.raises(ValidationError, match="only by configured suites"): + AgentSuite( + name="invalid-baseline", + tasks=[task], + baseline_mcp_servers=["graphify"], + ) + with pytest.raises(ValidationError, match="non-empty and unique"): + AgentSuite( + name="duplicate-baseline", + tasks=[task], + integration_mode="configured", + repository_commit="commit:test", + baseline_mcp_servers=["graphify", "graphify"], + ) + with pytest.raises(ValidationError, match="letters, digits"): + AgentSuite( + name="invalid-baseline-name", + tasks=[task], + integration_mode="configured", + repository_commit="commit:test", + baseline_mcp_servers=["graphify.server"], + ) + + +def test_agent_context_rejects_dirty_and_stale_candidates( + monkeypatch: Any, tmp_path: Path +) -> None: + target = tmp_path / "target" + target.mkdir() + tracked = target / "tracked.txt" + tracked.write_text("clean\n", encoding="utf-8") + _git(target, "init", "--quiet") + _git(target, "config", "user.name", "CodeMesh Test") + _git(target, "config", "user.email", "codemesh@example.invalid") + _git(target, "add", "tracked.txt") + _git(target, "commit", "--quiet", "-m", "fixture") + base_commit = _git(target, "rev-parse", "HEAD").strip() + suite = AgentSuite( + name="configured", + repository_commit=base_commit, + integration_mode="configured", + tasks=[ + AgentTask( + id="answer", + description="answer", + task_type="answer", + prompt="answer", + targets=[RetrievalTarget(file_path="tracked.txt")], + ) + ], + ) + + async def stale_preflight(_repository_id: str) -> dict[str, Any]: + return { + "freshness": { + "indexed_commit": "stale", + "current_commit": base_commit, + "working_tree_dirty": False, + } + } + + tracked.write_text("dirty\n", encoding="utf-8") + with pytest.raises(EvaluationInfrastructureError, match="clean source checkout"): + asyncio.run( + _prepare_agent_context( + suite, + repository_root=target, + repository_id=None, + configured_plan=None, + configured_guidance_paths=None, + configured_probe_query="query", + configured_expected_paths=None, + configured_probe_timeout_seconds=30, + ) + ) + + tracked.write_text("clean\n", encoding="utf-8") + monkeypatch.setattr(agent_evaluation, "get_repository_preflight", stale_preflight) + with pytest.raises(EvaluationInfrastructureError, match="indexed commit"): + asyncio.run( + _prepare_agent_context( + suite, + repository_root=target, + repository_id=None, + configured_plan=None, + configured_guidance_paths=None, + configured_probe_query="query", + configured_expected_paths=None, + configured_probe_timeout_seconds=30, + ) + ) + + +def test_configured_integration_requires_exact_plan_binding_and_probes( + monkeypatch: Any, tmp_path: Path +) -> None: + target = tmp_path / "target" + target.mkdir() + (target / "AGENTS.md").write_text("# Configured guidance\n", encoding="utf-8") + (target / ".gitignore").write_text("/AGENTS.override.md\n", encoding="utf-8") + (target / "AGENTS.override.md").write_text( + "# Installed CodeMesh guidance\n", encoding="utf-8" + ) + (target / ".codex").mkdir() + (target / ".codex" / "config.toml").write_text( + "\n".join( + [ + "[mcp_servers.graphify]", + 'command = "graphify"', + 'args = ["mcp"]', + f'cwd = "{target.as_posix()}"', + "required = true", + 'enabled_tools = ["graphify_search"]', + 'env_vars = ["GRAPHIFY_URL"]', + "", + ] + ), + encoding="utf-8", + ) + _git(target, "init", "--quiet") + _git(target, "config", "user.name", "CodeMesh Test") + _git(target, "config", "user.email", "codemesh@example.invalid") + _git(target, "add", "AGENTS.md", ".gitignore") + _git(target, "commit", "--quiet", "-m", "fixture") + base_commit = _git(target, "rev-parse", "HEAD").strip() + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding( + "project:test", + "checkout:test", + str(target), + "view:test", + ) + plan = create_installation_plan( + target, + codemesh_root, + binding, + include_onboarding=False, + ) + apply_installation_plan(plan, plan.plan_hash) + preflight = { + "repository_id": "project:test", + "repository": { + "project_id": "project:test", + "checkout_id": "checkout:test", + "root_path": str(target), + "source_view_hash": "view:test", + }, + } + + async def fake_positive(received_plan: Any, **kwargs: Any) -> dict[str, Any]: + assert received_plan == plan + assert kwargs["query"] == "configured query" + assert kwargs["expected_paths"] == ["src/A.cs"] + return { + "schema_version": "codemesh-runtime-probe-v1", + "passed": True, + "provider_mode": "none", + "profile": "normal", + "server_name": "codemesh", + "project_id": "project:test", + "checkout_id": "checkout:test", + "repository_root": str(target), + "snapshot_id": "snapshot:test", + "source_view_hash": "view:test", + "indexed_commit": base_commit, + "current_commit": base_commit, + "freshness_status": "fresh", + "binding_status": "accepted", + "tools": plan.launch["expected_tools"], + "context_query": "configured query", + "context_item_count": 1, + "returned_paths": ["src/A.cs"], + "expected_paths": ["src/A.cs"], + "duration_ms": 1.5, + } + + async def fake_negative(received_plan: Any, **kwargs: Any) -> dict[str, Any]: + assert received_plan == plan + assert kwargs["checkout_id"].endswith(":codemesh-eval-rejection") + assert kwargs["expected_substrings"] == ["bound_repository_missing"] + return { + "schema_version": "codemesh-runtime-rejection-probe-v1", + "passed": True, + "rejected": True, + "provider_mode": "none", + "profile": "normal", + "checkout_id": kwargs["checkout_id"], + "expected_substrings": ["bound_repository_missing"], + "diagnostic": "binding rejected: bound_repository_missing", + "duration_ms": 1.0, + } + + monkeypatch.setattr(agent_evaluation, "run_runtime_probe", fake_positive) + monkeypatch.setattr(agent_evaluation, "run_runtime_rejection_probe", fake_negative) + + metadata, control_overrides, treatment_overrides, guidance_files = asyncio.run( + _prepare_configured_integration( + plan, + root=target, + base_commit=base_commit, + codemesh_root=codemesh_root, + preflight=preflight, + guidance_paths=["AGENTS.override.md"], + probe_query="configured query", + expected_paths=["src/A.cs"], + timeout_seconds=30, + baseline_mcp_servers=["graphify"], + ) + ) + + assert metadata["plan_hash"] == plan.plan_hash + assert metadata["profile"] == "normal" + assert metadata["guidance"][0]["path"] == "AGENTS.override.md" + assert len(metadata["guidance"][0]["sha256"]) == 64 + assert metadata["guidance"][0]["source"] == "installed-ignored" + assert guidance_files[0].content == b"# Installed CodeMesh guidance\n" + assert len(metadata["positive_probe"]["report_sha256"]) == 64 + assert "diagnostic" not in metadata["rejection_probe"] + assert metadata["baseline_mcp_servers"][0]["name"] == "graphify" + assert any("mcp_servers.graphify.command" in value for value in control_overrides) + assert not any("mcp_servers.codemesh" in value for value in control_overrides) + assert treatment_overrides[: len(control_overrides)] == control_overrides + assert any('"--profile", "normal"' in value for value in treatment_overrides) + assert any("default_tools_approval_mode" in value for value in treatment_overrides) + + mismatched = json.loads(json.dumps(preflight)) + mismatched["repository"]["checkout_id"] = "checkout:other" + with pytest.raises(EvaluationInfrastructureError, match="checkout"): + asyncio.run( + _prepare_configured_integration( + plan, + root=target, + base_commit=base_commit, + codemesh_root=codemesh_root, + preflight=mismatched, + guidance_paths=["AGENTS.md"], + probe_query="configured query", + expected_paths=["src/A.cs"], + timeout_seconds=30, + baseline_mcp_servers=["graphify"], + ) + ) + + (target / ".codex" / "config.toml").write_text( + "# changed after review\n", encoding="utf-8" + ) + with pytest.raises(EvaluationInfrastructureError, match="not applied unchanged"): + asyncio.run( + _prepare_configured_integration( + plan, + root=target, + base_commit=base_commit, + codemesh_root=codemesh_root, + preflight=preflight, + guidance_paths=["AGENTS.md"], + probe_query="configured query", + expected_paths=["src/A.cs"], + timeout_seconds=30, + baseline_mcp_servers=["graphify"], + ) + ) + + +def test_configured_agent_report_retains_identity_and_prompt_parity( + monkeypatch: Any, tmp_path: Path +) -> None: + repository = tmp_path / "repository" + (repository / "src").mkdir(parents=True) + (repository / "src" / "A.cs").write_text("class A {}\n", encoding="utf-8") + _git(repository, "init", "--quiet") + _git(repository, "config", "user.name", "CodeMesh Test") + _git(repository, "config", "user.email", "codemesh@example.invalid") + _git(repository, "add", "src/A.cs") + _git(repository, "commit", "--quiet", "-m", "fixture") + base_commit = _git(repository, "rev-parse", "HEAD").strip() + suite = AgentSuite( + name="configured", + repository_commit=base_commit, + integration_mode="configured", + tasks=[ + AgentTask( + id="answer", + description="answer", + task_type="answer", + prompt="Explain A.", + targets=[RetrievalTarget(file_path="src/A.cs")], + ) + ], + ) + configured_identity = { + "schema_version": "codemesh-configured-agent-integration-v1", + "plan_hash": "plan:test", + "profile": "normal", + } + context = AgentEvaluationContext( + root=repository, + base_commit=base_commit, + preflight={"repository_id": "project:test"}, + freshness={"current_branch": "main"}, + mcp_cwd=tmp_path, + codemesh_root=tmp_path, + codemesh_source_commit="codemesh:test", + codemesh_source_branch="main", + integration_mode="configured", + control_mcp_config=( + "mcp_servers={}", + 'mcp_servers.graphify.command="graphify"', + ), + treatment_mcp_config=('mcp_servers.codemesh.command="uv"',), + configured_integration=configured_identity, + configured_guidance_files=( + ConfiguredGuidanceFile( + path="AGENTS.override.md", + content=b"# Installed CodeMesh guidance\n", + sha256=hashlib.sha256(b"# Installed CodeMesh guidance\n").hexdigest(), + ), + ), + ) + + async def fake_context(*_args: Any, **_kwargs: Any) -> AgentEvaluationContext: + return context + + requests: list[AgentRunRequest] = [] + observed_guidance: list[str] = [] + + class FakeRunner: + @property + def capabilities(self) -> AgentRunnerCapabilities: + return AgentRunnerCapabilities( + hard_reported_token_cap=True, + reported_token_accounting=REPORTED_TOKEN_ACCOUNTING_ID, + ) + + async def run(self, request: AgentRunRequest) -> AgentRunResult: + requests.append(request) + observed_guidance.append( + (request.cwd / "AGENTS.override.md").read_text(encoding="utf-8") + ) + return AgentRunResult( + completed=True, + exit_code=0, + duration_ms=10, + final_message=json.dumps( + { + "answer": "A is implemented here.", + "citations": [ + { + "file_path": "src/A.cs", + "start_line": 1, + "end_line": 1, + "stable_key": None, + } + ], + } + ), + usage={ + "input_tokens": 5, + "cached_input_tokens": 3, + "output_tokens": 2, + "reasoning_output_tokens": 0, + }, + mcp_calls=( + ["codemesh.codemesh_get_context_package"] + if request.condition == "treatment" + else [] + ), + ) + + monkeypatch.setattr(agent_evaluation, "_prepare_agent_context", fake_context) + report = asyncio.run( + run_agent_evaluation( + suite, + tmp_path / "suite.json", + model="test-model", + repetitions=1, + max_reported_tokens=100, + output_dir=tmp_path / "artifacts", + runner=FakeRunner(), + ) + ) + + assert report.metadata["adoption_mode"] == "configured" + assert report.metadata["integration_mode"] == "configured" + assert report.metadata["prompt_parity"] is True + assert report.metadata["treatment_guidance"] is None + assert report.metadata["configured_integration"] == configured_identity + assert len(requests) == 2 + assert {request.prompt for request in requests} == {"Explain A."} + assert observed_guidance == [ + "# Installed CodeMesh guidance\n", + "# Installed CodeMesh guidance\n", + ] + control = next(request for request in requests if request.condition == "control") + treatment = next( + request for request in requests if request.condition == "treatment" + ) + assert control.mcp_config == context.control_mcp_config + assert treatment.mcp_config == context.treatment_mcp_config + assert [request.max_reported_tokens for request in requests] == [100, 93] + assert report.summary["aggregate_reported_tokens"] == 14 + assert report.summary["remaining_reported_tokens"] == 86 + assert report.summary["automatic_retries"] is False + + def test_ranking_metrics_and_stability_are_deterministic() -> None: candidates = [ CandidateRecord(file_path="src/Other.cs"), @@ -211,10 +741,91 @@ async def call_tool(self, *_args: Any, **_kwargs: Any) -> Any: asyncio.run(client.call("codemesh_search_context", {})) -def test_live_case_scores_results_and_omits_raw_payloads() -> None: +def test_context_package_timing_records_are_parsed_and_summarized() -> None: + stderr = "\n".join( + [ + "unrelated log", + 'CODEMESH_CONTEXT_PACKAGE_TIMING {"event":"context_package_timing",' + '"outcome":"ok","total_ms":20,"stages_ms":{"item_hydration":12},' + '"operations_ms":{"node":7},"counts":{"hits":2}}', + 'CODEMESH_CONTEXT_PACKAGE_TIMING {"event":"context_package_timing",' + '"outcome":"ok","total_ms":30,"stages_ms":{"item_hydration":18},' + '"operations_ms":{"node":9},"counts":{"hits":2}}', + ] + ) + + records = _parse_context_package_timings(stderr) + summary = _context_package_timing_summary(records, [None, "task-shaped"]) + + assert summary["record_count"] == 1 + case = summary["cases"]["task-shaped"] + assert case["calls"] == 1 + assert case["successful_calls"] == 1 + assert case["p50_total_ms"] == 30 + assert case["p50_stages_ms"] == {"item_hydration": 18} + assert case["p50_operations_ms"] == {"node": 9} + assert case["counts"] == {"hits": 2} + + +def test_context_package_timing_summary_fails_on_missing_sideband_record() -> None: + with pytest.raises( + EvaluationInfrastructureError, + match="record count did not match", + ): + _context_package_timing_summary([], ["task-shaped"]) + + +def test_repository_preflight_stale_diagnostic_retains_candidate_identity( + monkeypatch: Any, +) -> None: class FakeClient: + async def __aenter__(self) -> "FakeClient": + return self + + async def __aexit__(self, *_args: Any) -> None: + return None + + def connect(self) -> "FakeClient": + return self + async def call( self, _tool: str, _arguments: dict[str, Any] + ) -> tuple[dict[str, Any], float]: + return ( + { + "data": { + "freshness": { + "status": "stale", + "detail": "Candidate identity does not match.", + "indexed_commit": "indexed:test", + "current_commit": "current:test", + "working_tree_dirty": False, + "indexed_source_view_hash": "view:test", + } + } + }, + 1.0, + ) + + monkeypatch.setattr( + "codemesh_agent_access.evaluation.live.McpEvaluationClient", + lambda **_kwargs: FakeClient(), + ) + + with pytest.raises(EvaluationInfrastructureError) as captured: + asyncio.run(get_repository_preflight("project:test")) + + diagnostic = str(captured.value) + assert "indexed_commit='indexed:test'" in diagnostic + assert "current_commit='current:test'" in diagnostic + assert "working_tree_dirty=False" in diagnostic + assert "indexed_source_view_hash='view:test'" in diagnostic + + +def test_live_case_scores_results_and_omits_raw_payloads() -> None: + class FakeClient: + async def call( + self, _tool: str, _arguments: dict[str, Any], **_kwargs: Any ) -> tuple[dict[str, Any], float]: return ( { @@ -330,6 +941,237 @@ def test_codex_jsonl_parser_collects_usage_tools_and_failures() -> None: assert "malformed JSONL" in result.failures[0] +def test_reported_token_accounting_includes_reasoning_without_readding_cache() -> None: + usage = { + "input_tokens": 100, + "cached_input_tokens": 80, + "output_tokens": 20, + "reasoning_output_tokens": 5, + } + + assert count_reported_tokens(usage) == 125 + + with pytest.raises(ValueError, match="cannot exceed"): + count_reported_tokens({**usage, "cached_input_tokens": 101}) + with pytest.raises(ValueError, match="non-negative integer"): + count_reported_tokens({**usage, "output_tokens": -1}) + with pytest.raises(ValueError, match="missing required token fields"): + count_reported_tokens({"input_tokens": 100}) + + +def test_agent_preflight_refuses_runner_without_hard_cap_before_context( + monkeypatch: Any, +) -> None: + suite = AgentSuite( + name="configured", + repository_commit="commit:test", + integration_mode="configured", + tasks=[], + ) + context_prepared = False + + async def unexpected_context(*_args: Any, **_kwargs: Any) -> AgentEvaluationContext: + nonlocal context_prepared + context_prepared = True + raise AssertionError("repository preflight must not run") + + class UnsupportedRunner: + calls = 0 + + @property + def capabilities(self) -> AgentRunnerCapabilities: + return AgentRunnerCapabilities( + hard_reported_token_cap=False, + reported_token_accounting=REPORTED_TOKEN_ACCOUNTING_ID, + ) + + async def run(self, _request: AgentRunRequest) -> AgentRunResult: + self.calls += 1 + raise AssertionError("model runner must not be called") + + runner = UnsupportedRunner() + monkeypatch.setattr(agent_evaluation, "_prepare_agent_context", unexpected_context) + + with pytest.raises(EvaluationInfrastructureError, match="before any model call"): + asyncio.run( + run_agent_preflight( + suite, + max_reported_tokens=100, + runner=runner, + ) + ) + + assert context_prepared is False + assert runner.calls == 0 + + +def test_agent_campaign_stops_at_reported_token_boundary_without_next_call( + monkeypatch: Any, tmp_path: Path +) -> None: + suite = AgentSuite( + name="budget-boundary", + tasks=[ + AgentTask( + id="answer", + description="answer", + task_type="answer", + prompt="answer", + targets=[RetrievalTarget(file_path="src/A.cs")], + ) + ], + ) + context = AgentEvaluationContext( + root=tmp_path, + base_commit="commit:test", + preflight={"repository_id": "project:test"}, + freshness={"current_branch": "main"}, + mcp_cwd=tmp_path, + codemesh_root=tmp_path, + codemesh_source_commit="codemesh:test", + codemesh_source_branch="main", + integration_mode="spontaneous", + control_mcp_config=None, + treatment_mcp_config=None, + configured_integration=None, + ) + condition_calls: list[dict[str, Any]] = [] + + async def fake_context(*_args: Any, **_kwargs: Any) -> AgentEvaluationContext: + return context + + async def fake_condition(**kwargs: Any) -> AgentExecutionReport: + condition_calls.append(kwargs) + limit = kwargs["max_reported_tokens"] + return AgentExecutionReport( + task_id=kwargs["task"].id, + repetition=kwargs["repetition"], + condition=kwargs["condition"], + completed=True, + passed=True, + input_tokens=limit, + reported_tokens=limit, + reported_token_limit=limit, + ) + + class CapableRunner: + @property + def capabilities(self) -> AgentRunnerCapabilities: + return AgentRunnerCapabilities( + hard_reported_token_cap=True, + reported_token_accounting=REPORTED_TOKEN_ACCOUNTING_ID, + ) + + async def run(self, _request: AgentRunRequest) -> AgentRunResult: + raise AssertionError("patched condition owns this deterministic test") + + monkeypatch.setattr(agent_evaluation, "_prepare_agent_context", fake_context) + monkeypatch.setattr(agent_evaluation, "_run_task_condition", fake_condition) + + report = asyncio.run( + run_agent_evaluation( + suite, + tmp_path / "suite.json", + model="test-model", + repetitions=1, + max_reported_tokens=10, + output_dir=tmp_path / "boundary-artifacts", + runner=CapableRunner(), + ) + ) + + assert len(condition_calls) == 1 + assert condition_calls[0]["max_reported_tokens"] == 10 + assert report.verdict == "insufficient" + assert report.passed is False + assert report.summary["aggregate_reported_tokens"] == 10 + assert report.summary["remaining_reported_tokens"] == 0 + assert report.summary["stopped_for_reported_token_cap"] is True + assert "no retry was attempted" in report.failures[0] + + +def test_agent_campaign_does_not_retry_incomplete_run( + monkeypatch: Any, tmp_path: Path +) -> None: + suite = AgentSuite( + name="no-retry", + tasks=[ + AgentTask( + id="answer", + description="answer", + task_type="answer", + prompt="answer", + targets=[RetrievalTarget(file_path="src/A.cs")], + ) + ], + ) + context = AgentEvaluationContext( + root=tmp_path, + base_commit="commit:test", + preflight={"repository_id": "project:test"}, + freshness={"current_branch": "main"}, + mcp_cwd=tmp_path, + codemesh_root=tmp_path, + codemesh_source_commit="codemesh:test", + codemesh_source_branch="main", + integration_mode="spontaneous", + control_mcp_config=None, + treatment_mcp_config=None, + configured_integration=None, + ) + condition_calls = 0 + + async def fake_context(*_args: Any, **_kwargs: Any) -> AgentEvaluationContext: + return context + + async def fake_condition(**kwargs: Any) -> AgentExecutionReport: + nonlocal condition_calls + condition_calls += 1 + return AgentExecutionReport( + task_id=kwargs["task"].id, + repetition=kwargs["repetition"], + condition=kwargs["condition"], + completed=False, + passed=False, + input_tokens=3, + reported_tokens=3, + reported_token_limit=kwargs["max_reported_tokens"], + ) + + class CapableRunner: + @property + def capabilities(self) -> AgentRunnerCapabilities: + return AgentRunnerCapabilities( + hard_reported_token_cap=True, + reported_token_accounting=REPORTED_TOKEN_ACCOUNTING_ID, + ) + + async def run(self, _request: AgentRunRequest) -> AgentRunResult: + raise AssertionError("patched condition owns this deterministic test") + + monkeypatch.setattr(agent_evaluation, "_prepare_agent_context", fake_context) + monkeypatch.setattr(agent_evaluation, "_run_task_condition", fake_condition) + + report = asyncio.run( + run_agent_evaluation( + suite, + tmp_path / "suite.json", + model="test-model", + repetitions=1, + max_reported_tokens=10, + output_dir=tmp_path / "no-retry-artifacts", + runner=CapableRunner(), + ) + ) + + assert condition_calls == 1 + assert len(report.runs) == 1 + assert report.verdict == "insufficient" + assert report.summary["aggregate_reported_tokens"] == 3 + assert report.summary["remaining_reported_tokens"] == 7 + assert report.summary["automatic_retries"] is False + assert "no retry was attempted" in report.failures[0] + + def test_answer_grading_validates_target_and_line_range(tmp_path: Path) -> None: source = tmp_path / "src" / "Target.cs" source.parent.mkdir() @@ -604,7 +1446,12 @@ async def run(self, request: AgentRunRequest) -> AgentRunResult: completed=True, exit_code=0, duration_ms=10, - usage={"input_tokens": 5, "output_tokens": 2}, + usage={ + "input_tokens": 5, + "cached_input_tokens": 0, + "output_tokens": 2, + "reasoning_output_tokens": 0, + }, ) artifacts = tmp_path / "artifacts" diff --git a/agent-access/tests/test_evaluation_paths.py b/agent-access/tests/test_evaluation_paths.py new file mode 100644 index 0000000..acb06c3 --- /dev/null +++ b/agent-access/tests/test_evaluation_paths.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import asyncio +import json +import shutil +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from codemesh_agent_access.evaluation import agent, benchmark +from codemesh_agent_access.evaluation.codex import ( + REPORTED_TOKEN_ACCOUNTING_ID, + AgentRunnerCapabilities, +) +from codemesh_agent_access.evaluation.models import ( + AgentSuite, + ModelBenchmarkSuite, + load_suite, +) +from codemesh_agent_access.evaluation.paths import ( + managed_artifact_path, + resolve_setup_patch, +) + + +@pytest.mark.parametrize( + "identifier", + [ + "", + ".", + "..", + "../escape", + "/absolute", + "a/b", + "a\\b", + "C:drive", + "item:stream", + "trailing.", + "NUL", + "con.txt", + "LPT1", + "line\nbreak", + ], +) +@pytest.mark.parametrize("kind", ["task", "question", "change"]) +def test_artifact_identifiers_reject_cross_platform_path_syntax(identifier, kind): + agent_suite, _ = load_suite("codemesh-agent", "agent") + model_suite, _ = load_suite("codemesh-model", "model-benchmark") + item = { + "task": agent_suite.tasks[0], + "question": model_suite.questions[0], + "change": model_suite.change_scenarios[0], + }[kind] + payload = item.model_dump() + payload["id"] = identifier + with pytest.raises(ValidationError): + type(item).model_validate(payload) + + +def test_suites_reject_case_insensitive_artifact_collisions(): + suite, _ = load_suite("codemesh-agent", "agent") + payload = suite.model_dump() + duplicate = dict(payload["tasks"][0], id=payload["tasks"][0]["id"].upper()) + payload["tasks"].append(duplicate) + with pytest.raises(ValidationError, match="unique ignoring case"): + AgentSuite.model_validate(payload) + + model_suite, _ = load_suite("codemesh-model", "model-benchmark") + payload = model_suite.model_dump() + payload["change_scenarios"][0]["id"] = payload["questions"][0]["id"] + with pytest.raises(ValidationError, match="unique ignoring case"): + ModelBenchmarkSuite.model_validate(payload) + + +@pytest.mark.parametrize( + "relative", + [ + "../outside.patch", + "/tmp/outside.patch", + "C:/outside.patch", + "patches\\outside.patch", + "patches/../outside.patch", + "a:stream", + ], +) +def test_patch_names_reject_escape_syntax_before_file_access(tmp_path, relative): + with pytest.raises(ValueError, match="suite-relative"): + resolve_setup_patch(tmp_path / "suite.json", relative) + + +def test_patch_resolution_accepts_contained_files_and_rejects_link_escape(tmp_path): + suite = tmp_path / "suite" + (suite / "patches").mkdir(parents=True) + patch = suite / "patches" / "fault with spaces.patch" + patch.write_text("fixture patch", encoding="utf-8") + assert ( + resolve_setup_patch(suite / "suite.json", "patches/fault with spaces.patch") + == patch + ) + + outside = tmp_path / "outside.patch" + outside.write_text("must not be applied", encoding="utf-8") + _symlink(suite / "escape.patch", outside) + with pytest.raises(ValueError, match="within its suite"): + resolve_setup_patch(suite / "suite.json", "escape.patch") + with pytest.raises(ValueError, match="accessible"): + resolve_setup_patch(suite / "suite.json", "missing.patch") + + +@pytest.mark.parametrize("directory", ["workspaces", ".runtime", "answer-1-treatment"]) +def test_generated_artifacts_reject_existing_symlink_escape(tmp_path, directory): + root = tmp_path / "artifacts" + outside = tmp_path / "outside" + root.mkdir() + outside.mkdir() + _symlink(root / directory, outside) + with pytest.raises(ValueError, match="escapes"): + managed_artifact_path(root, directory, "result.json") + assert list(outside.iterdir()) == [] + + +def test_operator_selected_artifact_root_may_resolve_to_a_directory(tmp_path): + root = tmp_path / "selected-root" + root.mkdir() + alias = tmp_path / "alias" + _symlink(alias, root) + assert ( + managed_artifact_path(alias, "answer", "result.json") + == root / "answer" / "result.json" + ) + + +def test_agent_rejects_mutated_identifier_before_preflight_or_runner( + monkeypatch, tmp_path +): + suite, path = load_suite("youtube-downloader-agent", "agent") + suite.tasks[0].id = "../escape" + monkeypatch.setattr(agent, "_prepare_agent_context", _unexpected_effect) + runner = _unused_capped_runner() + with pytest.raises(ValueError, match="safe path segments"): + asyncio.run( + agent.run_agent_evaluation( + suite, path, model="unused", max_reported_tokens=100, runner=runner + ) + ) + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("mode", ["agent", "model"]) +def test_all_setup_patches_are_checked_before_any_repository_or_model_work( + monkeypatch, tmp_path, mode +): + model_suite, original_path = load_suite("codemesh-model", "model-benchmark") + copied = tmp_path / "suites" + shutil.copytree(original_path.parent, copied) + outside = tmp_path / "outside.patch" + outside.write_text("must not be read or applied", encoding="utf-8") + _symlink(copied / "escape.patch", outside) + monkeypatch.setattr(agent, "_prepare_agent_context", _unexpected_effect) + monkeypatch.setattr(benchmark, "_git_root", _unexpected_effect) + + if mode == "agent": + suite, _ = load_suite("codemesh-agent", "agent") + suite.tasks[-1].setup_patch = "escape.patch" + pending = agent.run_agent_evaluation( + suite, + copied / "codemesh-agent.json", + model="unused", + max_reported_tokens=100, + runner=_unused_capped_runner(), + ) + else: + model_suite.change_scenarios[-1].setup_patch = "escape.patch" + pending = benchmark.run_model_benchmark( + model_suite, + copied / original_path.name, + model="unused", + runner=_unused_capped_runner(), + ) + with pytest.raises(ValueError, match="within its suite"): + asyncio.run(pending) + + +def test_suite_loader_rejects_patch_links_before_returning_a_runnable_suite(tmp_path): + suite, _ = load_suite("codemesh-agent", "agent") + outside = tmp_path / "outside.patch" + outside.write_text("not an accepted patch", encoding="utf-8") + directory = tmp_path / "suite" + directory.mkdir() + _symlink(directory / "escape.patch", outside) + payload = suite.model_dump() + payload["tasks"] = [payload["tasks"][0]] + payload["tasks"][0]["setup_patch"] = "escape.patch" + path = directory / "suite.json" + path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="within its suite"): + load_suite(path, "agent") + + +def test_validation_argv_preserves_shell_metacharacters_as_literal_arguments(tmp_path): + marker = tmp_path / "must-not-exist" + argument = f"literal; touch {marker}; $(touch {marker})" + result = agent._run_command( + [ + sys.executable, + "-c", + "import json, sys; print(json.dumps(sys.argv[1:]))", + argument, + ], + cwd=tmp_path, + ) + assert result.returncode == 0 + assert json.loads(result.stdout) == [argument] + assert not marker.exists() + + +def _symlink(link: Path, target: Path) -> None: + try: + link.symlink_to(target, target_is_directory=target.is_dir()) + except OSError as exc: + pytest.skip(f"This platform cannot create the required symlink fixture: {exc}") + + +def _unexpected_effect(*args, **kwargs): + raise AssertionError( + "Unsafe suite reached repository preparation or model execution" + ) + + +def _unused_capped_runner(): + return SimpleNamespace( + capabilities=AgentRunnerCapabilities( + hard_reported_token_cap=True, + reported_token_accounting=REPORTED_TOKEN_ACCOUNTING_ID, + runner_id="unused-fixture", + ), + run=_unexpected_effect, + ) + + +@pytest.mark.parametrize("kind", ["agent", "model-benchmark"]) +@pytest.mark.parametrize("name", ["../escape", "/absolute", "a\\b", "CON"]) +def test_suite_name_cannot_escape_default_artifact_root(kind, name): + built_in = "codemesh-agent" if kind == "agent" else "codemesh-model" + suite, _ = load_suite(built_in, kind) + payload = suite.model_dump() + payload["name"] = name + with pytest.raises(ValidationError): + type(suite).model_validate(payload) + + +def test_mutated_suite_name_rejected_before_preflight(monkeypatch): + suite, path = load_suite("youtube-downloader-agent", "agent") + suite.name = "../escape" + monkeypatch.setattr(agent, "_prepare_agent_context", _unexpected_effect) + with pytest.raises(ValueError, match="safe path segments"): + asyncio.run( + agent.run_agent_evaluation( + suite, + path, + model="unused", + max_reported_tokens=100, + runner=_unused_capped_runner(), + ) + ) + + +def test_codex_launch_preserves_selected_executable_and_literal_arguments( + monkeypatch, tmp_path +): + from codemesh_agent_access.evaluation import codex + + captured = [] + executable = str(tmp_path / "selected codex;literal") + payload = '$(touch never-created); & echo "literal"' + + class Process: + returncode = 0 + + async def communicate(self): + return b"", b"" + + async def spawn(*argv, **kwargs): + captured.append((argv, kwargs)) + return Process() + + monkeypatch.setattr(codex.asyncio, "create_subprocess_exec", spawn) + runner = codex.CodexRunner(executable, supports_ignore_user_config=True) + request = codex.AgentRunRequest( + prompt=payload, + cwd=tmp_path, + condition="control", + model=payload, + reasoning_effort="medium", + sandbox="read-only", + timeout_seconds=5, + mcp_cwd=tmp_path, + ) + asyncio.run(runner.run(request)) + argv, options = captured[0] + assert argv[0] == executable + assert argv[-1] == payload + assert argv[argv.index("--model") + 1] == payload + assert options["cwd"] == tmp_path + assert "shell" not in options + assert len(captured) == 1 diff --git a/agent-access/tests/test_feedback.py b/agent-access/tests/test_feedback.py new file mode 100644 index 0000000..b533121 --- /dev/null +++ b/agent-access/tests/test_feedback.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from codemesh_agent_access import cli, feedback +from codemesh_agent_access.feedback import ( + PACKET_SCHEMA_VERSION, + SUMMARY_SCHEMA_VERSION, + record_feedback, + summarize_feedback, + validate_feedback, +) + + +def test_records_and_validates_sanitized_commit_bound_packet(tmp_path: Path) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + + packet, output = _record(repository) + + assert output.parent == repository / ".codemesh-feedback" + assert packet.schema_version == PACKET_SCHEMA_VERSION + assert packet.repository.root == str(repository) + assert packet.repository.branch == "main" + assert packet.repository.dirty is False + assert packet.codemesh.languages == ["python", "rust"] + assert packet.codemesh.dirty is False + assert packet.result.missed_paths == ["rust/src/lib.rs"] + assert validate_feedback(output) == packet + + +def test_recorder_requires_ignored_outbox(tmp_path: Path) -> None: + repository = _repository(tmp_path / "repo", ignored=False) + + with pytest.raises(ValueError, match="is not ignored"): + _record(repository) + + +def test_recorder_rejects_linked_outbox(tmp_path: Path) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + outside = tmp_path / "outside" + outside.mkdir() + (repository / ".codemesh-feedback").symlink_to(outside, target_is_directory=True) + + with pytest.raises(ValueError, match="outbox must be a local directory"): + _record(repository) + + assert list(outside.iterdir()) == [] + + +@pytest.mark.parametrize("destination_kind", ["file", "symlink", "hardlink"]) +def test_recorder_preserves_existing_packet_destination( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, destination_kind: str +) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + outbox = repository / ".codemesh-feedback" + outbox.mkdir() + feedback_id = "feedback-" + "a" * 20 + monkeypatch.setattr(feedback, "_content_id", lambda _payload: feedback_id) + destination = outbox / f"{feedback_id}.json" + sentinel = tmp_path / "sentinel.json" + sentinel.write_text("preserved", encoding="utf-8") + if destination_kind == "symlink": + destination.symlink_to(sentinel) + elif destination_kind == "hardlink": + destination.hardlink_to(sentinel) + else: + destination.write_text("preserved", encoding="utf-8") + + with pytest.raises(FileExistsError): + _record(repository) + + assert sentinel.read_text(encoding="utf-8") == "preserved" + assert destination.read_text(encoding="utf-8") == "preserved" + + +def test_recorder_rejects_secrets_and_non_relative_paths(tmp_path: Path) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + + with pytest.raises(ValueError, match="credential or secret"): + _record(repository, fallback_reason="token=do-not-store") + + with pytest.raises(ValueError, match="repository-relative paths"): + _record(repository, missed_paths=[str(repository / "src/app.py")]) + + +def test_validator_rejects_tampered_packet(tmp_path: Path) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + _packet, output = _record(repository) + payload = json.loads(output.read_text(encoding="utf-8")) + payload["issue"]["category"] = "ranking" + output.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="Feedback id mismatch"): + validate_feedback(output) + + +def test_summarizer_groups_and_preserves_exact_provenance(tmp_path: Path) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + first, _ = _record(repository, snapshot_id="snapshot:first") + second, _ = _record(repository, snapshot_id="snapshot:second") + + summary = summarize_feedback([repository]) + + assert summary["schema_version"] == SUMMARY_SCHEMA_VERSION + assert summary["packet_count"] == 2 + priority = summary["priorities"][0] + assert priority["issue_category"] == "missing-relationship" + assert priority["occurrences"] == 2 + assert priority["priority"] == "high" + assert priority["status"] == "open" + assert {item["feedback_id"] for item in priority["provenance"]} == { + first.feedback_id, + second.feedback_id, + } + assert {item["snapshot_id"] for item in priority["provenance"]} == { + "snapshot:first", + "snapshot:second", + } + + +def test_successful_recheck_resolves_superseded_failure(tmp_path: Path) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + failed, _ = _record(repository, snapshot_id="snapshot:failed") + _passed, _ = _record( + repository, + snapshot_id="snapshot:recheck", + validation_outcome="passed", + supersedes_feedback_ids=[failed.feedback_id], + ) + + priority = summarize_feedback([repository])["priorities"][0] + + assert priority["status"] == "resolved" + assert priority["priority"] == "low" + assert priority["priority_score"] == 0 + assert priority["superseded_feedback_ids"] == [failed.feedback_id] + assert priority["unresolved_failure_ids"] == [] + + +def test_feedback_cli_records_then_summarizes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + repository = _repository(tmp_path / "repo", ignored=True) + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "feedback", + "record", + "--repository-root", + str(repository), + "--codemesh-root", + str(repository), + "--role", + "Primary", + "--classification", + "diagnostic", + "--confidence", + "medium", + "--project-id", + "project:onenine", + "--checkout-id", + "checkout:primary", + "--snapshot-id", + "snapshot:test", + "--language", + "python", + "--language", + "rust", + "--parser-profile", + "python-rust-v1", + "--task-family", + "test-selection", + "--issue-category", + "ranking", + "--validation-outcome", + "inconclusive", + "--missed-path", + "tests/test_bridge.py", + ], + ) + + cli.main() + + recorded = json.loads(capsys.readouterr().out) + assert recorded["recorded"] is True + monkeypatch.setattr( + sys, + "argv", + ["codemesh-agent-access", "feedback", "summarize", str(repository)], + ) + cli.main() + summary = json.loads(capsys.readouterr().out) + assert summary["packet_count"] == 1 + assert summary["priorities"][0]["task_family"] == "test-selection" + + +def _record( + repository: Path, + *, + snapshot_id: str = "snapshot:test", + fallback_reason: str | None = "Used rg after the expected edge was absent.", + missed_paths: list[str] | None = None, + validation_outcome: str = "failed", + supersedes_feedback_ids: list[str] | None = None, +): + return record_feedback( + repository_root=repository, + codemesh_root=repository, + role="Primary", + classification="diagnostic", + confidence="high", + project_id="project:onenine", + checkout_id="checkout:primary", + snapshot_id=snapshot_id, + languages=["rust", "python", "python"], + parser_profile="python-rust-v1", + task_family="cross-language-impact", + issue_category="missing-relationship", + validation_outcome=validation_outcome, + tools_attempted=["codemesh_get_context_package"], + tools_called=["codemesh_get_context_package"], + missed_paths=missed_paths or ["rust/src/lib.rs"], + fallback_reason=fallback_reason, + minimal_reproduction="Find the Python call into the exported Rust function.", + expected_targets=["python/bridge.py", "rust/src/lib.rs"], + proposed_correction="Resolve aliased PyO3 exports across language results.", + supersedes_feedback_ids=supersedes_feedback_ids, + ) + + +def _repository(path: Path, *, ignored: bool) -> Path: + path.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=path, check=True) + subprocess.run( + ["git", "config", "user.email", "codemesh-tests@example.invalid"], + cwd=path, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "CodeMesh Tests"], cwd=path, check=True + ) + (path / "src").mkdir() + (path / "src" / "app.py").write_text("print('test')\n", encoding="utf-8") + if ignored: + (path / ".gitignore").write_text("/.codemesh-feedback/\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=path, check=True) + subprocess.run(["git", "commit", "-m", "test: fixture"], cwd=path, check=True) + return path.resolve() diff --git a/agent-access/tests/test_feedback_session.py b/agent-access/tests/test_feedback_session.py new file mode 100644 index 0000000..d324a1b --- /dev/null +++ b/agent-access/tests/test_feedback_session.py @@ -0,0 +1,380 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +from codemesh_agent_access import cli +from codemesh_agent_access.binding import RepositoryBinding +from codemesh_agent_access.feedback_session import ( + CLIENT_PROFILE, + MAINTAINER_PROFILE, + SessionParticipant, + activate_session, + create_session_plan, + create_session_runtime, + inspect_session, + load_active_session, + load_session_plan, + revoke_session, + write_session_plan, +) + + +NOW = datetime(2026, 9, 10, 8, 0, tzinfo=UTC) + + +def test_session_plan_requires_reviewed_hash_and_authorizes_exact_bindings( + tmp_path: Path, +) -> None: + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=True) + manifest_path = tmp_path / "sessions" / "active.json" + manifest_path.parent.mkdir() + + plan = create_session_plan( + manifest_path=manifest_path, + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[_participant(client, "client", "Primary")], + expires_at=NOW + timedelta(hours=4), + now=NOW, + ) + + assert not manifest_path.exists() + with pytest.raises(ValueError, match="Approved plan hash"): + activate_session(plan, "0" * 64) + + assert activate_session(plan, plan.plan_hash) == manifest_path + with pytest.raises(FileExistsError): + activate_session(plan, plan.plan_hash) + with pytest.raises(ValueError, match="manifest hash"): + inspect_session(manifest_path, "0" * 64, now=NOW) + status = inspect_session(manifest_path, plan.manifest_file_sha256, now=NOW) + assert status["status"] == "active" + assert status["client_count"] == 1 + + client_binding = RepositoryBinding( + "project:client", + "checkout:client", + str(client), + ) + client_manifest = load_active_session( + manifest_path, + plan.manifest_file_sha256, + profile=CLIENT_PROFILE, + binding=client_binding, + now=NOW, + ) + assert client_manifest.payload.session_id.startswith("session-") + + maintainer_binding = RepositoryBinding( + "project:codemesh", + "checkout:codemesh", + str(codemesh), + ) + runtime = create_session_runtime( + manifest_path, + plan.manifest_file_sha256, + profile=MAINTAINER_PROFILE, + binding=maintainer_binding, + clock=lambda: NOW, + ) + assert runtime.assert_active().codemesh.repository_root == str(codemesh) + + +def test_session_plan_round_trip_and_tamper_rejection(tmp_path: Path) -> None: + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=True) + manifest_path = tmp_path / "active.json" + plan_path = tmp_path / "plan.json" + plan = create_session_plan( + manifest_path=manifest_path, + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[_participant(client, "client", "Primary")], + expires_at=NOW + timedelta(hours=1), + now=NOW, + ) + + write_session_plan(plan, plan_path) + assert load_session_plan(plan_path) == plan + + payload = json.loads(plan_path.read_text(encoding="utf-8")) + payload["manifest"]["payload"]["expires_at"] = ( + NOW + timedelta(hours=2) + ).isoformat() + plan_path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="plan hash|payload hash"): + load_session_plan(plan_path) + + +def test_session_rejects_linked_authority_files(tmp_path: Path) -> None: + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=True) + plan = create_session_plan( + manifest_path=tmp_path / "active.json", + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[_participant(client, "client", "Primary")], + expires_at=NOW + timedelta(hours=1), + now=NOW, + ) + manifest = activate_session(plan, plan.plan_hash) + manifest_link = tmp_path / "manifest-hardlink.json" + manifest_link.hardlink_to(manifest) + with pytest.raises(ValueError, match="unlinked local regular file"): + inspect_session(manifest, plan.manifest_file_sha256, now=NOW) + manifest_link.unlink() + + revocation = revoke_session( + manifest, + plan.manifest_file_sha256, + reason="Owner closed the test session.", + now=NOW, + ) + revocation_link = tmp_path / "revocation-hardlink.json" + revocation_link.hardlink_to(revocation) + with pytest.raises(ValueError, match="unlinked local regular file"): + inspect_session(manifest, plan.manifest_file_sha256, now=NOW) + revocation_link.unlink() + revocation.unlink() + revocation.symlink_to(tmp_path / "missing-revocation.json") + with pytest.raises(ValueError, match="unlinked local regular file"): + inspect_session(manifest, plan.manifest_file_sha256, now=NOW) + + +def test_session_rejects_wrong_binding_expiry_and_revocation(tmp_path: Path) -> None: + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=True) + plan = create_session_plan( + manifest_path=tmp_path / "active.json", + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[_participant(client, "client", "Primary")], + expires_at=NOW + timedelta(hours=1), + now=NOW, + ) + activate_session(plan, plan.plan_hash) + binding = RepositoryBinding("project:client", "checkout:wrong", str(client)) + + with pytest.raises(ValueError, match="does not authorize"): + load_active_session( + plan.manifest_path, + plan.manifest_file_sha256, + profile=CLIENT_PROFILE, + binding=binding, + now=NOW, + ) + + correct = RepositoryBinding("project:client", "checkout:client", str(client)) + with pytest.raises(ValueError, match="expired"): + load_active_session( + plan.manifest_path, + plan.manifest_file_sha256, + profile=CLIENT_PROFILE, + binding=correct, + now=NOW + timedelta(hours=1), + ) + + revoke_session( + plan.manifest_path, + plan.manifest_file_sha256, + reason="Owner closed the development session.", + now=NOW + timedelta(minutes=5), + ) + with pytest.raises(FileExistsError): + revoke_session( + plan.manifest_path, + plan.manifest_file_sha256, + reason="Duplicate close must not overwrite.", + now=NOW + timedelta(minutes=6), + ) + assert ( + inspect_session(plan.manifest_path, plan.manifest_file_sha256, now=NOW)[ + "status" + ] + == "revoked" + ) + with pytest.raises(ValueError, match="revoked"): + load_active_session( + plan.manifest_path, + plan.manifest_file_sha256, + profile=CLIENT_PROFILE, + binding=correct, + now=NOW, + ) + + +def test_session_requires_ignored_unlinked_client_outbox(tmp_path: Path) -> None: + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=False) + + with pytest.raises(ValueError, match="is not ignored"): + create_session_plan( + manifest_path=tmp_path / "active.json", + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[_participant(client, "client", "Primary")], + expires_at=NOW + timedelta(hours=1), + now=NOW, + ) + + (client / ".gitignore").write_text( + "/.codemesh-feedback/\n", + encoding="utf-8", + ) + outside = tmp_path / "outside" + outside.mkdir() + (client / ".codemesh-feedback").symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="outbox must be a local directory"): + create_session_plan( + manifest_path=tmp_path / "active.json", + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[_participant(client, "client", "Primary")], + expires_at=NOW + timedelta(hours=1), + now=NOW, + ) + + +def test_session_rejects_duplicate_clients_and_long_duration(tmp_path: Path) -> None: + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=True) + participant = _participant(client, "client", "Primary") + + with pytest.raises(ValueError, match="must be unique"): + create_session_plan( + manifest_path=tmp_path / "active.json", + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[participant, participant], + expires_at=NOW + timedelta(hours=1), + now=NOW, + ) + + with pytest.raises(ValueError, match="seven days"): + create_session_plan( + manifest_path=tmp_path / "active.json", + codemesh=_participant(codemesh, "codemesh", "maintainer"), + clients=[participant], + expires_at=NOW + timedelta(days=8), + now=NOW, + ) + + +def test_session_cli_plans_activates_inspects_and_revokes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + codemesh = _repository(tmp_path / "codemesh", ignored=False) + client = _repository(tmp_path / "client", ignored=True) + manifest = tmp_path / "active.json" + plan_path = tmp_path / "plan.json" + client_json = json.dumps( + _participant(client, "client", "Primary").model_dump(mode="json") + ) + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "feedback", + "session", + "plan", + "--manifest-path", + str(manifest), + "--codemesh-root", + str(codemesh), + "--codemesh-project-id", + "project:codemesh", + "--codemesh-checkout-id", + "checkout:codemesh", + "--client-json", + client_json, + "--expires-at", + (datetime.now(UTC) + timedelta(hours=1)).isoformat(), + "--output", + str(plan_path), + ], + ) + cli.main() + planned = json.loads(capsys.readouterr().out) + + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "feedback", + "session", + "activate", + str(plan_path), + "--approve-plan-hash", + planned["plan_hash"], + ], + ) + cli.main() + assert json.loads(capsys.readouterr().out)["activated"] is True + + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "feedback", + "session", + "inspect", + "--manifest", + str(manifest), + "--manifest-sha256", + planned["manifest_file_sha256"], + ], + ) + cli.main() + assert json.loads(capsys.readouterr().out)["status"] == "active" + + monkeypatch.setattr( + sys, + "argv", + [ + "codemesh-agent-access", + "feedback", + "session", + "revoke", + "--manifest", + str(manifest), + "--manifest-sha256", + planned["manifest_file_sha256"], + "--reason", + "Owner closed the test session.", + ], + ) + cli.main() + assert json.loads(capsys.readouterr().out)["revoked"] is True + + +def _participant(root: Path, name: str, role: str) -> SessionParticipant: + return SessionParticipant( + project_id=f"project:{name}", + checkout_id=f"checkout:{name}", + repository_root=str(root), + reporter_role=role, + ) + + +def _repository(path: Path, *, ignored: bool) -> Path: + path.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=path, check=True) + subprocess.run( + ["git", "config", "user.email", "codemesh-tests@example.invalid"], + cwd=path, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "CodeMesh Tests"], cwd=path, check=True + ) + (path / "README.md").write_text("# Fixture\n", encoding="utf-8") + if ignored: + (path / ".gitignore").write_text("/.codemesh-feedback/\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=path, check=True) + subprocess.run(["git", "commit", "-m", "test: fixture"], cwd=path, check=True) + return path.resolve() diff --git a/agent-access/tests/test_installer_probe.py b/agent-access/tests/test_installer_probe.py new file mode 100644 index 0000000..0923293 --- /dev/null +++ b/agent-access/tests/test_installer_probe.py @@ -0,0 +1,539 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import UTC, datetime, timedelta +import hashlib +import json +from pathlib import Path +import subprocess +from types import SimpleNamespace +from typing import Any + +import pytest + +from codemesh_agent_access import probe +from codemesh_agent_access.binding import RepositoryBinding +from codemesh_agent_access.installer import ( + SCHEMA_VERSION, + SCHEMA_VERSION_V1, + apply_installation_plan, + codex_mcp_overrides, + create_installation_plan, + load_installation_plan, + verify_plan_applied, + write_installation_plan, +) +from codemesh_agent_access.feedback_session import ( + CLIENT_PROFILE, + SessionParticipant, + activate_session, + create_session_plan, +) + + +def test_installation_requires_reviewed_hash_and_preserves_unmanaged_content( + tmp_path, +) -> None: + target = tmp_path / "target" + target.mkdir() + (target / "AGENTS.md").write_text("# Existing instructions\n", encoding="utf-8") + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:test", "checkout:test", str(target)) + + plan = create_installation_plan(target, codemesh_root, binding) + + assert not (target / ".codex" / "config.toml").exists() + assert (target / "AGENTS.md").read_text(encoding="utf-8") == ( + "# Existing instructions\n" + ) + assert all(item.diff for item in plan.files) + assert "CODEMESH_NEO4J_PASSWORD" in plan.files[0].content + assert 'CODEMESH_MODEL_PROVIDER = "none"' in plan.files[0].content + assert "# Existing instructions" in plan.files[1].content + + with pytest.raises(ValueError, match="Approved plan hash"): + apply_installation_plan(plan, "wrong") + + changed = apply_installation_plan(plan, plan.plan_hash) + + assert changed == [".codex/config.toml", "AGENTS.md"] + verify_plan_applied(plan) + assert "# Existing instructions" in (target / "AGENTS.md").read_text( + encoding="utf-8" + ) + assert "codemesh:onboarding:start" in (target / "AGENTS.md").read_text( + encoding="utf-8" + ) + onboarding = (target / "AGENTS.md").read_text(encoding="utf-8") + assert "1. Start with one `codemesh_get_context_package` call" in onboarding + assert "covers every required facet" in onboarding + assert "at most three" in onboarding + assert "avoid overlapping queries" in onboarding + assert "fails closed when they are not accepted" in onboarding + assert "only for explicit diagnostics" in onboarding + assert onboarding.index("codemesh_get_context_package") < onboarding.index( + "codemesh_get_repository_status" + ) + + overrides = codex_mcp_overrides(plan, approve_tools=True) + assert f'mcp_servers.codemesh.command="{plan.launch["command"]}"' in overrides + assert any('"--profile", "normal"' in value for value in overrides) + assert any("env_vars=" in value for value in overrides) + assert any('CODEMESH_MODEL_PROVIDER = "none"' in value for value in overrides) + assert 'mcp_servers.codemesh.default_tools_approval_mode="approve"' in overrides + assert not any("codemesh_delete_repository" in value for value in overrides) + + tampered = replace( + plan, + launch={**plan.launch, "args": [*plan.launch["args"], "--unexpected"]}, + ) + with pytest.raises(ValueError, match="hash does not match"): + codex_mcp_overrides(tampered, approve_tools=True) + + +def test_installation_plan_round_trip_and_concurrent_change_rejection(tmp_path) -> None: + target = tmp_path / "target" + target.mkdir() + (target / "AGENTS.md").write_text("# Initial\n", encoding="utf-8") + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:test", "checkout:test", str(target)) + plan = create_installation_plan(target, codemesh_root, binding) + plan_path = tmp_path / "reviewed-plan.json" + write_installation_plan(plan, plan_path) + + loaded = load_installation_plan(plan_path) + assert loaded == plan + + (target / "AGENTS.md").write_text("# Changed after review\n", encoding="utf-8") + with pytest.raises(ValueError, match="changed after review"): + apply_installation_plan(loaded, loaded.plan_hash) + + +def test_configuration_only_plan_leaves_repository_guidance_to_instance_manager( + tmp_path, +) -> None: + target = tmp_path / "target" + target.mkdir() + (target / "AGENTS.md").write_text("# Managed elsewhere\n", encoding="utf-8") + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:test", "checkout:test", str(target)) + + plan = create_installation_plan( + target, + codemesh_root, + binding, + include_onboarding=False, + ) + + assert [item.path for item in plan.files] == [".codex/config.toml"] + changed = apply_installation_plan(plan, plan.plan_hash) + assert changed == [".codex/config.toml"] + assert (target / "AGENTS.md").read_text(encoding="utf-8") == ( + "# Managed elsewhere\n" + ) + + +def test_feedback_installation_pins_exact_session_and_onboards_client( + tmp_path: Path, +) -> None: + target = _git_repository(tmp_path / "target", ignored=True) + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:client", "checkout:client", str(target)) + session_plan = create_session_plan( + manifest_path=tmp_path / "session.json", + codemesh=SessionParticipant( + project_id="project:codemesh", + checkout_id="checkout:codemesh", + repository_root=str(codemesh_root), + reporter_role="maintainer", + ), + clients=[ + SessionParticipant( + project_id=binding.project_id, + checkout_id=binding.checkout_id, + repository_root=binding.repository_root, + reporter_role="Primary", + ) + ], + expires_at=datetime.now(UTC) + timedelta(hours=1), + ) + activate_session(session_plan, session_plan.plan_hash) + + plan = create_installation_plan( + target, + codemesh_root, + binding, + profile=CLIENT_PROFILE, + feedback_session_path=session_plan.manifest_path, + feedback_session_sha256=session_plan.manifest_file_sha256, + ) + + assert plan.schema_version == SCHEMA_VERSION + assert plan.session == { + "manifest_path": session_plan.manifest_path, + "manifest_file_sha256": session_plan.manifest_file_sha256, + } + assert "--feedback-session" in plan.launch["args"] + assert "codemesh_record_feedback" in plan.launch["expected_tools"] + onboarding = next(item.content for item in plan.files if item.path == "AGENTS.md") + assert "codemesh_get_feedback_session" in onboarding + assert "Human review remains required" in onboarding + apply_installation_plan(plan, plan.plan_hash) + verify_plan_applied(plan) + + +def test_v1_normal_installation_plan_remains_loadable(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:test", "checkout:test", str(target)) + plan = create_installation_plan(target, codemesh_root, binding) + payload = plan.to_dict() + payload["schema_version"] = SCHEMA_VERSION_V1 + payload.pop("session") + payload.pop("plan_hash") + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + payload["plan_hash"] = hashlib.sha256(encoded).hexdigest() + path = tmp_path / "v1-plan.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + loaded = load_installation_plan(path) + + assert loaded.schema_version == SCHEMA_VERSION_V1 + assert loaded.session is None + assert loaded.profile == "normal" + + +def test_feedback_runtime_probe_checks_session_identity_without_writing( + monkeypatch: Any, + tmp_path: Path, +) -> None: + target = _git_repository(tmp_path / "target", ignored=True) + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:client", "checkout:client", str(target)) + session_plan = create_session_plan( + manifest_path=tmp_path / "session.json", + codemesh=SessionParticipant( + project_id="project:codemesh", + checkout_id="checkout:codemesh", + repository_root=str(codemesh_root), + reporter_role="maintainer", + ), + clients=[ + SessionParticipant( + project_id=binding.project_id, + checkout_id=binding.checkout_id, + repository_root=binding.repository_root, + reporter_role="Primary", + ) + ], + expires_at=datetime.now(UTC) + timedelta(hours=1), + ) + activate_session(session_plan, session_plan.plan_hash) + plan = create_installation_plan( + target, + codemesh_root, + binding, + profile=CLIENT_PROFILE, + feedback_session_path=session_plan.manifest_path, + feedback_session_sha256=session_plan.manifest_file_sha256, + ) + apply_installation_plan(plan, plan.plan_hash) + session = FakeFeedbackProbeSession(plan, session_plan.manifest.payload.session_id) + + @asynccontextmanager + async def fake_stdio_client(parameters: Any): + assert parameters.args == plan.launch["args"] + assert parameters.env["CODEMESH_MODEL_PROVIDER"] == "none" + yield object(), object() + + monkeypatch.setattr(probe, "stdio_client", fake_stdio_client) + monkeypatch.setattr(probe, "ClientSession", lambda *_args: session) + + report = asyncio.run(probe.run_feedback_runtime_probe(plan)) + + assert report["passed"] is True + assert report["profile"] == CLIENT_PROFILE + assert report["session_id"] == session_plan.manifest.payload.session_id + assert session.calls == ["codemesh_get_feedback_session"] + + +def test_runtime_probe_checks_exact_bound_normal_server( + monkeypatch: Any, tmp_path +) -> None: + target = tmp_path / "target" + target.mkdir() + (target / "AGENTS.md").write_text("# Initial\n", encoding="utf-8") + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:test", "checkout:test", str(target)) + plan = create_installation_plan(target, codemesh_root, binding) + apply_installation_plan(plan, plan.plan_hash) + + session = FakeProbeSession(plan) + + @asynccontextmanager + async def fake_stdio_client(parameters: Any): + assert parameters.command == plan.launch["command"] + assert parameters.args == plan.launch["args"] + assert str(parameters.cwd) == plan.launch["cwd"] + assert parameters.env["CODEMESH_MODEL_PROVIDER"] == "none" + yield object(), object() + + monkeypatch.setattr(probe, "stdio_client", fake_stdio_client) + monkeypatch.setattr(probe, "ClientSession", lambda *_args: session) + + report = asyncio.run( + probe.run_runtime_probe(plan, expected_paths=["src/example.py"]) + ) + + assert report["passed"] is True + assert report["profile"] == "normal" + assert report["project_id"] == "project:test" + assert report["checkout_id"] == "checkout:test" + assert report["snapshot_id"] == "snapshot:test" + assert report["context_item_count"] == 1 + assert report["returned_paths"] == ["src/example.py"] + assert report["expected_paths"] == ["src/example.py"] + assert session.calls == [ + "codemesh_list_repositories", + "codemesh_get_repository_status", + "codemesh_get_context_package", + ] + + +def test_rejection_probe_requires_unsafe_binding_to_fail_closed( + monkeypatch: Any, tmp_path +) -> None: + target = tmp_path / "target" + target.mkdir() + (target / "AGENTS.md").write_text("# Initial\n", encoding="utf-8") + codemesh_root = Path(__file__).resolve().parents[2] + binding = RepositoryBinding("project:test", "checkout:test", str(target)) + plan = create_installation_plan(target, codemesh_root, binding) + apply_installation_plan(plan, plan.plan_hash) + session = FakeRejectedSession(plan) + + @asynccontextmanager + async def fake_stdio_client(parameters: Any): + checkout_index = parameters.args.index("--checkout-id") + 1 + assert parameters.args[checkout_index] == "checkout:wrong" + assert parameters.env["CODEMESH_MODEL_PROVIDER"] == "none" + yield object(), object() + + monkeypatch.setattr(probe, "stdio_client", fake_stdio_client) + monkeypatch.setattr(probe, "ClientSession", lambda *_args: session) + + report = asyncio.run( + probe.run_runtime_rejection_probe( + plan, + checkout_id="checkout:wrong", + expected_substrings=["bound_repository_missing"], + ) + ) + + assert report["passed"] is True + assert report["rejected"] is True + assert report["checkout_id"] == "checkout:wrong" + assert report["rejection_issues"] == [ + "bound_repository_missing", + "snapshot_not_fresh", + "snapshot_provenance_unknown", + ] + + +class FakeProbeSession: + def __init__(self, plan: Any) -> None: + self.plan = plan + self.calls: list[str] = [] + + async def __aenter__(self) -> "FakeProbeSession": + return self + + async def __aexit__(self, *args: Any) -> None: + return None + + async def initialize(self) -> Any: + binding = self.plan.binding + return SimpleNamespace( + instructions=( + f"bound {binding['project_id']} {binding['checkout_id']} " + f"{binding['repository_root']}" + ) + ) + + async def list_tools(self) -> Any: + return SimpleNamespace( + tools=[ + SimpleNamespace(name=name) + for name in self.plan.launch["expected_tools"] + ] + ) + + async def call_tool( + self, + name: str, + arguments: dict[str, Any], + read_timeout_seconds: Any, + ) -> Any: + self.calls.append(name) + binding = self.plan.binding + repository = { + "repository_id": binding["project_id"], + "project_id": binding["project_id"], + "checkout_id": binding["checkout_id"], + "snapshot_id": "snapshot:test", + "source_view_hash": "view:test", + "root_path": binding["repository_root"], + } + if name == "codemesh_list_repositories": + data = {"repositories": [repository]} + elif name == "codemesh_get_repository_status": + data = { + "repository_id": binding["project_id"], + "repository": repository, + "freshness": { + "status": "fresh", + "is_stale": False, + "indexed_commit": "abc123", + "current_commit": "abc123", + "binding": {"status": "accepted", "issues": []}, + }, + } + else: + data = { + "query": {"query": arguments["query"]}, + "items": [ + { + "hit": {"file_path": "src/example.py"}, + "relationship_groups": {}, + } + ], + } + payload = {"ok": True, "data": data, "error": None} + return SimpleNamespace( + isError=False, + structuredContent=payload, + content=[SimpleNamespace(text=json.dumps(payload))], + ) + + +class FakeRejectedSession: + def __init__(self, plan: Any) -> None: + self.plan = plan + + async def __aenter__(self) -> "FakeRejectedSession": + return self + + async def __aexit__(self, *args: Any) -> None: + return None + + async def initialize(self) -> Any: + return SimpleNamespace(instructions="bound rejection fixture") + + async def list_tools(self) -> Any: + return SimpleNamespace( + tools=[ + SimpleNamespace(name=name) + for name in self.plan.launch["expected_tools"] + ] + ) + + async def call_tool(self, *_args: Any, **_kwargs: Any) -> Any: + payload = { + "ok": True, + "data": { + "repository_id": self.plan.binding["project_id"], + "repository": None, + "freshness": { + "status": "missing", + "is_stale": True, + "provenance_status": "unknown", + "detail": "Repository was not found in the CodeMesh registry.", + "binding": { + "status": "rejected", + "issues": [ + "bound_repository_missing", + "snapshot_not_fresh", + "snapshot_provenance_unknown", + ], + }, + }, + }, + "error": None, + } + return SimpleNamespace( + isError=False, + structuredContent=payload, + content=[SimpleNamespace(text=json.dumps(payload))], + ) + + +class FakeFeedbackProbeSession: + def __init__(self, plan: Any, session_id: str) -> None: + self.plan = plan + self.session_id = session_id + self.calls: list[str] = [] + + async def __aenter__(self) -> "FakeFeedbackProbeSession": + return self + + async def __aexit__(self, *args: Any) -> None: + return None + + async def initialize(self) -> Any: + binding = self.plan.binding + return SimpleNamespace( + instructions=( + "explicitly enabled local feedback session bound to " + f"{binding['project_id']} {binding['checkout_id']} " + f"{binding['repository_root']}" + ) + ) + + async def list_tools(self) -> Any: + return SimpleNamespace( + tools=[ + SimpleNamespace(name=name) + for name in self.plan.launch["expected_tools"] + ] + ) + + async def call_tool( + self, + name: str, + arguments: dict[str, Any], + read_timeout_seconds: Any, + ) -> Any: + self.calls.append(name) + assert arguments == {} + payload = { + "session_id": self.session_id, + "enabled": True, + "local_only": True, + } + return SimpleNamespace( + isError=False, + structuredContent=payload, + content=[SimpleNamespace(text=json.dumps(payload))], + ) + + +def _git_repository(path: Path, *, ignored: bool) -> Path: + path.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=path, check=True) + subprocess.run( + ["git", "config", "user.email", "codemesh-tests@example.invalid"], + cwd=path, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "CodeMesh Tests"], cwd=path, check=True + ) + (path / "README.md").write_text("# Fixture\n", encoding="utf-8") + if ignored: + (path / ".gitignore").write_text("/.codemesh-feedback/\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=path, check=True) + subprocess.run(["git", "commit", "-m", "test: fixture"], cwd=path, check=True) + return path.resolve() diff --git a/agent-access/tests/test_mcp_contract.py b/agent-access/tests/test_mcp_contract.py index 5740ef8..5a3ac2c 100644 --- a/agent-access/tests/test_mcp_contract.py +++ b/agent-access/tests/test_mcp_contract.py @@ -5,8 +5,10 @@ from typing import Any import pytest +from mcp.server.fastmcp.exceptions import ToolError from codemesh_agent_access import mcp +from codemesh_agent_access.binding import RepositoryBinding from codemesh_agent_access.models import ToolResult @@ -18,7 +20,13 @@ def test_mcp_manifest_matches_registered_tools() -> None: assert manifest["profile"] == "diagnostic" assert server.instructions == manifest["instructions"] assert "codemesh_get_context_package" in manifest["instructions"] + assert "one codemesh_get_context_package call" in manifest["instructions"] + assert "covers every required facet" in manifest["instructions"] + assert "at most three context packages" in manifest["instructions"] + assert "avoid overlapping queries" in manifest["instructions"] assert "targeted, batched checkout reads" in manifest["instructions"] + assert "fails closed" in manifest["instructions"] + assert "only for explicit" in manifest["instructions"] assert {tool["name"] for tool in manifest["tools"]} == set(registered_tools) assert set(manifest["contracts"]) == set(registered_tools) @@ -62,6 +70,7 @@ def test_mcp_manifest_matches_registered_tools() -> None: ] package_contract = manifest["contracts"]["codemesh_get_context_package"] + assert "fails closed before search" in package_contract["description"] assert package_contract["response"]["fields"] == [ "query", "repository", @@ -104,6 +113,7 @@ def test_mcp_manifest_matches_registered_tools() -> None: assert "codemesh_delete_repository" not in manifest["contracts"] status_contract = manifest["contracts"]["codemesh_get_repository_status"] + assert "not a required first step" in status_contract["description"] assert status_contract["response"]["fields"] == [ "repository_id", "repository", @@ -112,6 +122,7 @@ def test_mcp_manifest_matches_registered_tools() -> None: "counts", "freshness", "refresh", + "codemesh_next_action", "generated_at", ] @@ -156,10 +167,68 @@ def test_mcp_unknown_profile_fails_fast() -> None: with pytest.raises(ValueError, match="Unknown MCP tool profile 'admin'"): mcp.build_server(profile="admin") - with pytest.raises(ValueError, match="Expected one of: normal, diagnostic"): + with pytest.raises(ValueError, match="development-feedback"): mcp.tool_manifest(profile="admin") +def test_normal_mcp_runtime_requires_checkout_binding() -> None: + with pytest.raises(ValueError, match="normal MCP profile requires"): + mcp.run(profile="normal") + + +def test_bound_mcp_injects_identity_and_rejects_override( + monkeypatch: Any, + tmp_path, +) -> None: + captured: dict[str, Any] = {} + binding = RepositoryBinding( + "project:test", + "checkout:test", + str(tmp_path), + "view:test", + ) + + async def fake_get_context_package(query: str, **kwargs: Any) -> ToolResult: + captured["query"] = query + captured.update(kwargs) + return ToolResult( + data={ + "query": {"query": query, "repository_id": kwargs["repository_id"]}, + "items": [], + "validation_recommendations": [], + "total_snippet_characters": 0, + "generated_at": "2026-08-31T00:00:00Z", + } + ) + + monkeypatch.setattr(mcp.tools, "get_context_package", fake_get_context_package) + server = mcp.build_server(binding=binding) + + result = asyncio.run( + server.call_tool( + "codemesh_get_context_package", + {"query": "find native replay", "output_format": "json"}, + ) + ) + with pytest.raises(ToolError, match="does not match the configured project"): + asyncio.run( + server.call_tool( + "codemesh_get_context_package", + { + "query": "find native replay", + "repository_id": "project:other", + "output_format": "json", + }, + ) + ) + + assert result.isError is False + assert captured["repository_id"] == "project:test" + assert captured["binding"] is binding + assert server.instructions == mcp.tool_manifest(binding=binding)["instructions"] + assert "checkout:test" in server.instructions + + def test_mcp_context_package_forwards_group_limits(monkeypatch: Any) -> None: captured: dict[str, Any] = {} @@ -375,7 +444,9 @@ async def fake_get_summary_coverage(repository_id: str) -> ToolResult: def test_mcp_repository_status_forwards_repository_id(monkeypatch: Any) -> None: captured: dict[str, Any] = {} - async def fake_get_repository_status(repository_id: str) -> ToolResult: + async def fake_get_repository_status( + repository_id: str, **kwargs: Any + ) -> ToolResult: captured["repository_id"] = repository_id return ToolResult( data={"repository_id": repository_id, "freshness": {"status": "unknown"}} @@ -393,12 +464,65 @@ async def fake_get_repository_status(repository_id: str) -> ToolResult: assert captured["repository_id"] == "sample_repo" assert structured["data"]["freshness"]["status"] == "unknown" + assert structured["data"]["codemesh_next_action"] == { + "use_codemesh": False, + "next_tool": None, + "instruction": ( + "Do not use CodeMesh context for this task because the exact binding " + "is not both accepted and fresh. Fall back to direct source inspection." + ), + } + + +def test_bound_mcp_repository_status_directs_fresh_context_retrieval( + monkeypatch: Any, + tmp_path, +) -> None: + binding = RepositoryBinding( + "project:test", + "checkout:test", + str(tmp_path), + "view:test", + ) + + async def fake_get_repository_status( + repository_id: str, **kwargs: Any + ) -> ToolResult: + assert repository_id == "project:test" + assert kwargs["binding"] is binding + return ToolResult( + data={ + "repository_id": repository_id, + "freshness": { + "status": "fresh", + "binding": {"status": "accepted", "issues": []}, + }, + } + ) + + monkeypatch.setattr(mcp.tools, "get_repository_status", fake_get_repository_status) + + server = mcp.build_server(binding=binding) + _content, structured = asyncio.run( + server.call_tool("codemesh_get_repository_status", {}) + ) + + assert structured["data"]["codemesh_next_action"]["use_codemesh"] is True + assert ( + structured["data"]["codemesh_next_action"]["next_tool"] + == "codemesh_get_context_package" + ) + assert "Continue now" in structured["data"]["codemesh_next_action"]["instruction"] + instruction = structured["data"]["codemesh_next_action"]["instruction"] + assert "specific unresolved implementation or test facet" in instruction + assert "at most three packages" in instruction def test_mcp_profiles_do_not_expose_repository_deletion() -> None: for profile in mcp.tool_profile_names(): - server = mcp.build_server(profile=profile) - registered_tools = {tool.name for tool in asyncio.run(server.list_tools())} + registered_tools = { + tool["name"] for tool in mcp.tool_manifest(profile=profile)["tools"] + } assert "codemesh_delete_repository" not in registered_tools diff --git a/agent-access/tests/test_model_benchmark.py b/agent-access/tests/test_model_benchmark.py index ae392ae..150f985 100644 --- a/agent-access/tests/test_model_benchmark.py +++ b/agent-access/tests/test_model_benchmark.py @@ -117,7 +117,12 @@ def test_query_report_flags_and_scrubs_canary_leakage(tmp_path: Path) -> None: completed=True, exit_code=0, duration_ms=10, - usage={"input_tokens": 10, "output_tokens": 2}, + usage={ + "input_tokens": 10, + "cached_input_tokens": 0, + "output_tokens": 2, + "reasoning_output_tokens": 0, + }, mcp_call_attempts=[ "codemesh.codemesh_search_context", "benchmark_response.submit_benchmark_answer", diff --git a/agent-access/tests/test_rest_contract.py b/agent-access/tests/test_rest_contract.py index 262763e..67717c8 100644 --- a/agent-access/tests/test_rest_contract.py +++ b/agent-access/tests/test_rest_contract.py @@ -1,6 +1,11 @@ from __future__ import annotations +from html.parser import HTMLParser from typing import Any +from urllib.parse import quote + +import pytest +from pydantic import BaseModel from fastapi.testclient import TestClient @@ -463,3 +468,109 @@ async def list_ingestion_runs( ) ] ) + + +class ParsedUi(HTMLParser): + def __init__(self, html: str) -> None: + super().__init__(convert_charrefs=True) + self.elements: list[tuple[str, dict[str, str | None]]] = [] + self.text: list[str] = [] + self.feed(html) + + def handle_starttag(self, tag, attrs): + self.elements.append((tag, dict(attrs))) + + def handle_data(self, data): + self.text.append(data) + + +def _replace_store_text(value, payload): + if isinstance(value, BaseModel): + return value.model_copy( + update={ + name: _replace_store_text(getattr(value, name), payload) + for name in type(value).model_fields + } + ) + if isinstance(value, dict): + return {key: _replace_store_text(item, payload) for key, item in value.items()} + if isinstance(value, list): + return [_replace_store_text(item, payload) for item in value] + return payload if isinstance(value, str) else value + + +@pytest.mark.parametrize( + "payload", + [ + '', + '">', + "javascript:alert(1)", + ""' onclick=alert(2)", + ], +) +def test_ui_keeps_hostile_request_and_store_values_inert(monkeypatch, payload): + store = FakeRestStore() + + def poisoned(method): + async def call(*args, **kwargs): + value = _replace_store_text(await method(*args, **kwargs), payload) + if isinstance(value, ContextResponse): + value.hits[0].preview = payload + if isinstance(value, RepositoryStatusResponse): + value.repository.branch = payload + value.repository.commit = payload + value.counts[payload] = 1 + return value + + return call + + for name in ( + "health", + "list_repositories", + "list_ingestion_runs", + "get_repository_status", + "search_context", + ): + monkeypatch.setattr(store, name, poisoned(getattr(store, name))) + + with rest_client(store) as client: + for route in ( + "/ui", + "/ui/repositories", + f"/ui/repositories/{quote(payload, safe='')}", + "/ui/runs", + "/ui/search", + ): + response = client.get( + route, params={"query": payload, "repository_id": payload} + ) + assert response.status_code == 200 + parsed = ParsedUi(response.text) + assert payload in "".join(parsed.text) + for tag, attrs in parsed.elements: + assert tag not in {"script", "img", "svg", "iframe", "object", "embed"} + assert not any(name.startswith("on") for name in attrs) + for name in ("href", "action"): + if name in attrs: + assert attrs[name].startswith("/") + assert not attrs[name].startswith("//") + if route in {"/ui/search", "/ui/runs"}: + inputs = { + attrs.get("name"): attrs.get("value") + for tag, attrs in parsed.elements + if tag == "input" + } + assert inputs["repository_id"] == payload + if route == "/ui/search": + assert inputs["query"] == payload + + +@pytest.mark.parametrize("limit", ['1" onfocus=alert(1)', "0", "51"]) +def test_ui_search_rejects_non_integer_or_out_of_range_limit(limit): + store = FakeRestStore() + with rest_client(store) as client: + response = client.get( + "/ui/search", params={"query": "download", "limit": limit} + ) + assert response.status_code == 422 + assert store.context_query is None diff --git a/agent-access/tests/test_store_components.py b/agent-access/tests/test_store_components.py index 361de4b..4fc0cad 100644 --- a/agent-access/tests/test_store_components.py +++ b/agent-access/tests/test_store_components.py @@ -3,9 +3,17 @@ import asyncio from typing import Any +import pytest + from codemesh_agent_access import embeddings as embeddings_module +from codemesh_agent_access.binding import ( + RepositoryBinding, + RepositoryBindingError, + assess_binding, +) from codemesh_agent_access.content_store import ( MongoContentStore, + bound_repository_document, normalize_ingestion_run, normalize_repository, repository_alias, @@ -21,7 +29,12 @@ lexical_terms, storage_key, ) -from codemesh_agent_access.models import IngestionRunSummary +from codemesh_agent_access.models import ( + ContextPackageQuery, + IngestionRunSummary, + RepositoryStatusResponse, + RepositorySummary, +) from codemesh_agent_access.store import CodeMeshReadStore, _refresh_status from codemesh_agent_access.summary_store import ( MongoNodeSummaryStore, @@ -176,6 +189,114 @@ def test_content_store_normalizers_preserve_registry_fields() -> None: } +def test_bound_repository_document_uses_exact_checkout_slot() -> None: + document = bound_repository_document( + { + "_id": "project:test", + "alias": "sample_repo", + "rootPath": "/wrong/last-writer", + "metadata": {"language": "python"}, + }, + { + "_id": "checkout:secondary", + "projectId": "project:test", + "rootPath": "/work/secondary", + "lastSeenAtUtc": "2026-08-31T00:00:00Z", + }, + { + "kind": "CheckoutCurrent", + "key": "checkout:secondary", + "snapshotId": "snapshot:secondary", + }, + { + "_id": "snapshot:secondary", + "projectId": "project:test", + "sourceViewHash": "view:secondary", + "nodeCount": 7, + "relationshipCount": 5, + "contentCount": 3, + }, + { + "branch": "feature/secondary", + "commit": "abc123", + "workingTreeDirty": False, + }, + ) + repository = normalize_repository(document) + + assert repository.project_id == "project:test" + assert repository.checkout_id == "checkout:secondary" + assert repository.snapshot_id == "snapshot:secondary" + assert repository.root_path == "/work/secondary" + assert repository.commit == "abc123" + assert repository.source_view_hash == "view:secondary" + assert repository.metadata["workingTreeDirty"] == "false" + assert repository.node_count == 7 + + +def test_binding_assessment_requires_exact_identity_root_and_freshness( + tmp_path, +) -> None: + binding = RepositoryBinding( + "project:test", + "checkout:test", + str(tmp_path), + "view:test", + ) + repository = RepositorySummary( + repository_id="project:test", + project_id="project:test", + checkout_id="checkout:test", + snapshot_id="snapshot:test", + source_view_hash="view:test", + root_path=str(tmp_path), + ) + + accepted = assess_binding( + binding, + repository, + {"status": "fresh", "is_stale": False, "provenance_status": "known"}, + ) + rejected = assess_binding( + binding, + repository.model_copy(update={"checkout_id": "checkout:other"}), + {"status": "stale", "is_stale": True, "provenance_status": "known"}, + ) + + assert accepted["status"] == "accepted" + assert accepted["issues"] == [] + assert rejected["status"] == "rejected" + assert rejected["issues"] == ["checkout_mismatch", "snapshot_not_fresh"] + + +def test_context_package_rejects_unaccepted_binding_before_search(tmp_path) -> None: + store = object.__new__(CodeMeshReadStore) + binding = RepositoryBinding("project:test", "checkout:test", str(tmp_path)) + + async def fake_status(*args: Any, **kwargs: Any) -> RepositoryStatusResponse: + return RepositoryStatusResponse( + repository_id="project:test", + freshness={ + "status": "stale", + "is_stale": True, + "binding": { + "status": "rejected", + "issues": ["snapshot_not_fresh"], + }, + }, + ) + + store.get_repository_status = fake_status # type: ignore[method-assign] + + with pytest.raises(RepositoryBindingError, match="snapshot_not_fresh"): + asyncio.run( + store.get_context_package( + ContextPackageQuery(query="find native replay"), + binding=binding, + ) + ) + + def test_refresh_status_reports_skipped_completed_and_failed_runs() -> None: skipped = _refresh_status( IngestionRunSummary( @@ -238,6 +359,55 @@ def test_graph_storage_key_uses_repository_prefix() -> None: assert storage_key("repo:test", "node:one") == "repo:test:node:one" +@pytest.mark.parametrize("repository_id", [None, "", "snapshot:'}) RETURN secret //"]) +def test_lexical_query_scopes_before_projection_and_parameterizes_input( + monkeypatch, repository_id +) -> None: + calls = [] + node = {"id": "node:test", "repositoryId": repository_id} + + class Session: + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + async def run(self, query, **parameters): + calls.append((query, parameters)) + return self + + async def data(self): + return [{"node": node}] + + class Driver: + def session(self, **kwargs): + return Session() + + store = Neo4jGraphStore(Settings.from_env()) + monkeypatch.setattr(store, "_neo4j", lambda: Driver()) + search_text = "Download 'quoted' input" + result = asyncio.run( + store.search_context_nodes(search_text, ["method"], repository_id, 12) + ) + assert result == [node] + query, parameters = calls[0] + assert parameters == { + "search_text": search_text.casefold(), + "search_terms": ["download", "quoted", "input"], + "kinds": ["method"], + "repository_id": repository_id, + "limit": 12, + } + assert search_text not in query + if repository_id is None: + assert "repositoryId" not in query + else: + assert "repositoryId: $repository_id" in query.split("WITH n", 1)[0] + if repository_id: + assert repository_id not in query + + def test_lexical_terms_normalizes_and_deduplicates_query_words() -> None: assert lexical_terms("Repository cleanup, repository records") == [ "repository", diff --git a/docs/README.md b/docs/README.md index 56b72c7..e5f8b56 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,10 +10,17 @@ readers can tell what CodeMesh does from what it may do later. | --- | --- | --- | | Evaluating CodeMesh | [`README.md`](../README.md) | [MCP Setup](guides/mcp-setup.md), [Project Status](current/project-status.md) | | Using CodeMesh through MCP | [MCP Setup](guides/mcp-setup.md) | [Agent Prompts](guides/agent-prompts.md), [Agent Access Contracts](current/agent-access-contracts.md) | -| Contributing | [`CONTRIBUTING.md`](../CONTRIBUTING.md) | [Testing](evaluation/testing.md), [Engineering Standards](engineering/standards.md) | +| Contributing | [`CONTRIBUTING.md`](../CONTRIBUTING.md) | [`COMMANDS.md`](../COMMANDS.md), [Testing](evaluation/testing.md), [Engineering Standards](engineering/standards.md) | +| Understanding, debugging, or extending the implementation | [Developer Handbook](guides/developer-handbook.md) | [Source Walkthroughs](guides/developer-walkthroughs.md), [Development and Debugging](guides/development-and-debugging.md), [Making Changes](guides/making-changes.md) | | Coding agent working in this repository | [Agent Quickstart](guides/agent-quickstart.md) | [`AGENTS.md`](../AGENTS.md), [Architecture](current/architecture.md) | | Reviewing product direction | [Project Status](current/project-status.md) | [Next Steps](planning/next-steps.md), then the relevant proposal | +The [Backup and Recovery guide](guides/backup-and-recovery.md) records the +verified disposable Linux procedure and the remaining upgrade/acceptance gates. + +The [source review packet](evaluation/review-packet-a158bc0.md) consolidates +local artifact evidence, draft release notes, and remaining acceptance gates. + ## Authority And Document Types When documents overlap, use this order: @@ -22,16 +29,20 @@ When documents overlap, use this order: 2. [Agent Access Contracts](current/agent-access-contracts.md) and [Architecture](current/architecture.md) describe the current supported surfaces and component boundaries. -3. [Testing](evaluation/testing.md), [MCP Setup](guides/mcp-setup.md), and - [Security and Redaction](current/security-and-redaction.md) are current operating - guides. -4. [Project Status](current/project-status.md) records the latest reviewed +3. [`COMMANDS.md`](../COMMANDS.md) indexes supported repository invocations; + [Testing](evaluation/testing.md), [MCP Setup](guides/mcp-setup.md), and + [Security and Redaction](current/security-and-redaction.md) are current + operating guides. +4. [Engineering Standards](engineering/standards.md) and the + [Software Engineering Methodology](engineering/methodology.md) define + contributor policy, evidence states, and authority boundaries. +5. [Project Status](current/project-status.md) records the latest reviewed implementation and evidence position. -5. [Next Steps](planning/next-steps.md) is the execution-order authority for future +6. [Next Steps](planning/next-steps.md) is the execution-order authority for future product work. -6. Proposal and roadmap documents describe unimplemented possibilities unless +7. Proposal and roadmap documents describe unimplemented possibilities unless current source and contracts say otherwise. -7. Dated evidence and publication reviews describe only the commit and time +8. Dated evidence and publication reviews describe only the commit and time they identify. Document labels have these meanings: @@ -63,6 +74,14 @@ Document labels have these meanings: - [`README.md`](../README.md) — public product overview and local demonstration. - [Project Status](current/project-status.md) — current capability and evidence summary. - [Architecture](current/architecture.md) — implemented components, flow, and stores. +- [Identity and Persistence](current/identity-and-persistence.md) — graph/content + identities, physical keys, publication, retention, and partial failures. +- [Ingestion Internals](current/ingestion-internals.md) — selection, each parser, + composition, incremental writes, refresh, and watch. +- [Retrieval Internals](current/retrieval-internals.md) — binding, freshness, + ranking, expansion, package assembly, and output budgets. +- [Enrichment and Evaluation Internals](current/enrichment-and-evaluation.md) — + embeddings, summaries, qualification, onboarding, feedback, and harnesses. - [Agent Access Contracts](current/agent-access-contracts.md) — REST, MCP, CLI-facing, UI, and .NET client contracts. - [Security and Redaction](current/security-and-redaction.md) — implemented redaction, @@ -70,6 +89,14 @@ Document labels have these meanings: ### Setup and use +- [Developer Handbook](guides/developer-handbook.md) — human entry point, mental + model, glossary, chapter index, and recommended reading paths. +- [Developer Walkthroughs](guides/developer-walkthroughs.md) — three source traces + from ingestion through retrieval and changed-source refresh. +- [Development and Debugging](guides/development-and-debugging.md) — setup, + configuration, debugging entry points, symptoms, and inspection findings. +- [Making Changes](guides/making-changes.md) — parser, ranking, interface, and + lifecycle change walkthroughs with validation selection. - [MCP Setup](guides/mcp-setup.md) — services, ingestion, stdio MCP configuration, and troubleshooting. - [Self-Analysis](guides/self-analysis.md) — index and query CodeMesh itself. @@ -93,9 +120,17 @@ Document labels have these meanings: - [Next Steps](planning/next-steps.md) — canonical execution order and capability map. - [Agent Integration Contract](planning/agent-integration-contract.md) — selected - explicit-onboarding contract, implemented tool profiles, and planned checkout - binding, freshness enforcement, runtime probe, installation, and evaluation - classification. + explicit-onboarding contract, implemented checkout binding, freshness + enforcement, tool profiles, runtime probes, reviewable installation, and + evaluation classification. +- [one|nine Adoption and Feedback Priority](planning/onenine-adoption-and-feedback.md) + — owner-activated plan and passed local pilot gate for safe checkout-bound + adoption, atomic Python/Rust ingestion, a Rust parser, agent feedback + artifacts, and controlled evaluation. +- [Development-Session MCP Feedback Loop](planning/development-feedback-mcp.md) + — completed owner-activated plan for explicit session-scoped client feedback + tools, read-only CodeMesh maintainer intake, human-approved changes, and + append-only verified rechecks. - [Graph Intelligence and Agent Memory](planning/graph-intelligence-design.md) — relationship evidence, graph analysis, parser tiers, memory, and portability proposal. @@ -104,30 +139,40 @@ Document labels have these meanings: method-decomposition evaluation. - [Repository Identity and Snapshot Retention](planning/repository-identity-and-snapshot-retention.md) — implemented local multi-checkout retention and future shared-store requirements. -- [Summary Model Qualification](planning/summary-model-qualification.md) — proposed - qualification procedure; no deployment profile is currently qualified. +- [Summary Model Qualification](planning/summary-model-qualification.md) — implemented + runner and operating procedure; no deployment profile is currently qualified. - [SDLC Roadmap](planning/sdlc-roadmap.md) — deferred broader artifact coverage. ### Engineering policy - [`CONTRIBUTING.md`](../CONTRIBUTING.md) — contribution entry point. +- [`COMMANDS.md`](../COMMANDS.md) — canonical supported command index. - [`SECURITY.md`](../SECURITY.md) — vulnerability reporting and safe operation. - [`THIRD_PARTY_NOTICES.md`](../THIRD_PARTY_NOTICES.md) — third-party and distribution boundaries. - [Engineering Standards](engineering/standards.md) — change discipline, verification, documentation, and definition of done. +- [Software Engineering Methodology](engineering/methodology.md) — evidence, + authority, guidance placement, and reusable-control admission boundaries. - [Conventional Commit Validation](engineering/conventional-commits.md) — local commit-message checker. - [GitHub Actions Baseline](engineering/github-actions.md) — exact workflow scope and evidence boundary. - [Markdown Link Checking](engineering/link-checking.md) — deterministic local link validation. +- [Release Preparation](engineering/release-preparation.md) — source version, + candidate contract, compatibility, and reviewable release gates; no release + authorization. +- [Release Notes Template](engineering/release-notes-template.md) — prepared, + tagged, published, deployed, and stable-state separation for a future release. - [Decision Record Template](decisions/0000-decision-record-template.md) — ADR format for implemented architecture decisions. ## Maintenance Rules -- Keep commands in one canonical guide and link to them elsewhere. +- Keep supported contributor and agent invocations in the root `COMMANDS.md`; + keep ordered operational and evaluation procedures in their controlling + guides and link between them. - Keep current implementation and evidence in `current/project-status.md`; keep future execution order in `planning/next-steps.md`. - Mark proposals and historical reviews explicitly near the title. diff --git a/docs/current/agent-access-contracts.md b/docs/current/agent-access-contracts.md index 0c0dae3..6179a84 100644 --- a/docs/current/agent-access-contracts.md +++ b/docs/current/agent-access-contracts.md @@ -2,7 +2,14 @@ Document type: current interface reference -Agent Access exposes the same read surface through REST, MCP tools, and the typed .NET client in `CodeMesh.Control.AgentAccess.AgentAccessClient`. +For call order, algorithms, adapter behavior, and failure interpretation, use +[Retrieval Internals](retrieval-internals.md). This document remains the +canonical interface field and surface reference. + +Agent Access exposes its repository read surface through REST, MCP tools, and +the typed .NET client in `CodeMesh.Control.AgentAccess.AgentAccessClient`. +Session-scoped feedback recording and maintainer intake are MCP/Python-only +local development controls; they are not added to REST or the .NET client. It also serves a local read-only web UI at `/ui` for humans to inspect service health, repositories, repository freshness, ingestion runs, and context search results. The UI reuses the REST read store and is not a separate write surface. @@ -24,6 +31,15 @@ lookup accepts URL-encoded path characters in `repository_id`; generated aliases remain slash-free and are retained with project identity across checkout moves and linked-worktree ingestion. +The normal stdio MCP server can instead start with an exact +`project_id`/`checkout_id`/`repository_root` binding and optional +`source_view_hash`. A complete binding is required for that product profile. +Bound repository tools inject the project id, reject conflicting caller +selectors, resolve only that checkout's current slot, and reject packages when +the project, checkout, root, source view, commit, or dirty-state freshness check +is not accepted. The diagnostic profile may still run unbound for explicit +administrative investigation. + Filters are exact-match string filters. They can match top-level node fields or metadata fields. `kinds` filters by CodeMesh node kind names such as `Method`, `Class`, `File`, `TestFile`, `TestCase`, or `Declaration`. ## REST Endpoints @@ -105,7 +121,16 @@ Search diagnostics: - `selected_project_id` and `selected_snapshot_id`: the project provenance and exact snapshot namespace selected for the response when available. -Agent Access merges vector hits, lexical graph hits, and MongoDB generated-summary hits, then dedupes by `node_id`. +Repository-scoped lexical search filters the initial graph match before +loading and matching searchable properties. Unbound administrative search +retains its cross-repository behavior. The change preserves lexical scoring, +candidate limits, ordering, filters, and parameterized query values. + +Agent Access merges vector hits, lexical graph hits, and MongoDB generated-summary +hits, then dedupes by `node_id`. When final scores tie, raw vector, lexical, and +summary relevance break the tie before kind and source-line ordering. Context +package selection covers distinct files before taking repeated items from the +same file when eligible alternatives remain. ## Summary Coverage @@ -122,7 +147,11 @@ Agent Access merges vector hits, lexical graph hits, and MongoDB generated-summa - `freshness`: local git diagnostics when the repository path is accessible to Agent Access. - `refresh`: derived latest-refresh activity and outcome from the latest ingestion run. -Freshness fields include `indexed_commit`, `current_commit`, `current_branch`, `working_tree_dirty`, `is_stale`, `status`, and `detail` when available. If ingestion has not stored an indexed commit yet, `status` is `unknown` and `is_stale` is `null`; agents should treat the response as a useful warning rather than proof that the index is fresh. +Freshness fields include `indexed_commit`, `current_commit`, `current_branch`, +`working_tree_dirty`, `is_stale`, `status`, and `detail` when available. Bound +responses also include a binding assessment with `accepted` or `rejected` +status and exact issues. Missing provenance remains `unknown` for unbound +administrative reads but is rejected by bound normal-profile package access. Snapshot-aware fields include `selected_snapshot_id`, `indexed_source_view_hash`, `commit_mismatch`, `source_view_mismatch`, and @@ -132,6 +161,17 @@ live checkout freshness is unavailable. Refresh fields include `status`, `is_running`, `changed`, `latest_run_id`, `stage`, `started_at`, `finished_at`, `diagnostic_count`, `error_diagnostic_count`, `warning_diagnostic_count`, `stats`, `detail`, and `recommended_action` when available. `status` is derived from the latest ingestion run and can be `no_run`, `running`, `completed`, `skipped`, `failed`, `dry_run`, or `unknown`. +The MCP status result adds `codemesh_next_action`. A fresh, accepted checkout +binding directs the agent to continue immediately with one agent-format +`codemesh_get_context_package` request using a concise task-shaped query and +default limits. If that package covers every required facet, retrieval stops. +Only a specific unresolved implementation or test facet justifies one focused +follow-up; the task budget is at most three non-overlapping packages. Every +other state directs the agent to stop CodeMesh discovery and fall back to +direct source inspection. Status remains an explicit diagnostic operation +rather than a required first hop. This presentation-only field is not part of +the REST status model. + `NodeSummaryCoverageResponse` fields: - `repository_id`: resolved repository id. @@ -177,6 +217,14 @@ Each `IngestionRunSummary` includes: ## Context Package +Optional `CODEMESH_CONTEXT_PACKAGE_TIMINGS` diagnostics write sanitized stage, +search-operation, and hydration-operation timings to stderr. They preserve +the package response and formatting contracts. Search timings include nested +pipeline totals and per-operation call counts, are isolated per request, and +must not be summed as wall-clock latency. See the +[evaluation timing contract](../evaluation/mcp-effectiveness.md) for capture +and interpretation. + `ContextPackageQuery` fields: - Core controls: `query`, `repository_id`, `limit`, `kinds`, `filters`, `vector`, `context_depth`. @@ -203,9 +251,22 @@ Each package item includes: - `relationship_groups`: grouped summaries for `callers`, `callees`, `reads`, `writes`, `contains`, `defines`, and `uses_types`. - `rationale`: ranking and expansion metadata. -Validation recommendations currently cover the .NET harness, Agent Access Python tests/lint, deployment parser dry-runs, end-to-end smoke checks for Docker/service changes, and Markdown dry-runs. - -MCP `codemesh_get_context_package` also accepts `output_format = "agent"`. In that mode the tool returns the agent brief directly as one text content block and intentionally omits structured content, avoiding MCP's duplicate text-plus-structured serialization. MCP agent output is capped at 12,000 characters even when a larger package budget is requested; the CLI continues to honor its explicit output budget. The brief includes validation recommendations plus each item's file span, node id, stable key, repository id, content hash, score, rank, and rationale details and adds a truncation marker when capped. Use the default `json` format when a caller needs the complete structured package. +Validation recommendations currently cover the .NET harness, Agent Access +Python tests/lint, Cargo formatting/Clippy/tests, deployment parser dry-runs, +end-to-end smoke checks for Docker/service changes, and Markdown dry-runs. + +MCP `codemesh_get_context_package` is the configured normal-host entry action +and also accepts `output_format = "agent"`. The bound package path validates +the exact binding and freshness before search and fails closed when either is +not accepted, so a separate status call is not required before retrieval. In +agent mode the tool returns the brief directly as one text content block and +intentionally omits structured content, avoiding MCP's duplicate text-plus- +structured serialization. MCP agent output is capped at 12,000 characters even +when a larger package budget is requested; the CLI continues to honor its +explicit output budget. The brief includes validation recommendations plus each +item's file span, node id, stable key, repository id, content hash, score, rank, +and rationale details and adds a truncation marker when capped. Use the default +`json` format when a caller needs the complete structured package. ## Symbols, Nodes, And Neighbors @@ -254,25 +315,36 @@ The `diagnostic` profile adds these non-destructive inspection tools: - `codemesh_get_summary_coverage` - `codemesh_list_ingestion_runs` -Select a profile with `mcp --profile normal|diagnostic`. Unknown profile names -fail before the MCP server starts. The manifest records the selected `profile` -and includes only that profile's tools and contracts. Repository deletion is -outside both MCP profiles and remains an explicitly invoked administrative REST -or Python CLI operation. +Two explicitly activated local development-session profiles are also +implemented: + +- `development-feedback` contains the four normal tools plus + `codemesh_get_feedback_session` and `codemesh_record_feedback`. +- `feedback-maintainer` contains the four normal tools plus + `codemesh_list_feedback`, `codemesh_get_feedback`, and + `codemesh_prepare_feedback_resolution`. + +Select a profile with `mcp --profile PROFILE`. Unknown profile names fail before +the MCP server starts. Both feedback profiles additionally require a complete +repository binding, `--feedback-session`, and +`--feedback-session-sha256`; startup and every feedback call revalidate the +immutable manifest, expiry, revocation record, exact binding, and allowlist. +The manifest records the selected `profile` and includes only that profile's +tools and contracts. Repository deletion remains an explicitly invoked +administrative REST or Python CLI operation outside every MCP profile. The MCP initialize result includes canonical server instructions for normal-host onboarding. For implementation discovery, likely change impact, and validation selection, they direct agents to start with an agent-format context package at default limits and use its ranked spans for targeted verification reads. The same text is exposed as the top-level `instructions` field in the stable -manifest. +manifest. A bound server's instructions also name its exact project, checkout, +and root. -Those initialization instructions are the currently implemented server-side -guidance, not proof that every host presents or follows them. The selected v1 -product contract therefore adds explicit repository-owned guidance; the current -manual block is in [MCP Setup](../guides/mcp-setup.md). The reduced tool profile -is implemented, while checkout binding and the live runtime probe remain planned -in [Agent Integration Contract](../planning/agent-integration-contract.md). +Those initialization instructions are defense in depth rather than proof that +every host presents or follows them. The implemented selected contract combines +repository-owned guidance, an exact bound normal profile, hashed installation, +and a live provider-free probe. See [MCP Setup](../guides/mcp-setup.md). `codemesh_get_tool_guidance` accepts optional `task` text and returns ranked workflows with recommended tool steps. Use it when an agent has not yet decided whether to search context, inspect freshness, look up a symbol, fetch neighbors, or build a package. @@ -291,6 +363,145 @@ option prints the default `normal` profile. The manifest is covered by `agent-access/tests/test_mcp_contract.py`. +## Installation, Probe, And Feedback CLI + +`install plan` produces a complete JSON plan containing the exact MCP launch, +binding, target-file hashes, full new content, diffs, and a plan hash without +changing the target. The current `codemesh-installation-plan-v2` adds an +optional pinned session path and hash; it is required for feedback profiles and +rejected for other profiles. The loader remains compatible with reviewed v1 +normal plans. `install apply` requires the exact reviewed hash and rejects +target drift. `--configuration-only` leaves repository guidance to an external +instance manager. + +`mcp-probe --plan PLAN` verifies that the reviewed plan is applied, launches +its exact stdio command with `CODEMESH_MODEL_PROVIDER=none`, checks the normal +four-tool surface, exact one-repository listing, accepted/fresh binding, and a +non-empty context package. `--query` and repeated `--expected-path` options turn +that package into a repository-specific retrieval gate. + +For a feedback-profile plan, the same command checks the exact session pins, +binding, initialization instructions, and profile tool surface. The client +probe calls session discovery without writing a packet; the maintainer probe +performs a bounded allowlisted list operation. Altered, expired, revoked, and +wrong-binding sessions fail closed before a feedback server becomes usable. + +`eval live --capture-context-package-timings` opts into a sanitized sideband +record for each context-package call. Records contain only fixed stage and +operation names, elapsed times, counts, and success state. They exclude queries, +paths, content, credentials, environment values, and repository, snapshot, or +node identifiers. The evaluator aggregates measured calls by case and excludes +warm-ups from the reported p50s. This instrumentation does not alter MCP tool +arguments, JSON responses, agent text, ranking, safety behavior, or character +budgets. + +`eval agent --preflight-only --configured-plan PLAN +--max-reported-tokens CAP` adds the paired-evaluation gate without invoking a +model. A configured suite must pin its repository commit. Before repository +preparation or any model call, the preflight requires the selected runner to +declare a hard reported-token cap using accounting identity +`input-output-reasoning-v1`. Reported tokens add `input_tokens`, +`output_tokens`, and `reasoning_output_tokens`; `cached_input_tokens` is a +subset of `input_tokens` and is not added again. Missing, negative, non-integer, +or internally inconsistent usage fails closed. + +Each scheduled execution receives the campaign's remaining allowance. A runner +claiming the capability must guarantee that the complete invocation, including +any internal provider activity, cannot report more than that allowance. The +evaluator stops before the next execution when the allowance reaches zero, +stops after an incomplete execution, never retries automatically, and records +the ceiling, aggregate, remainder, accounting identity, stop state, and +per-execution limit. A runner that exceeds its declared limit is an +infrastructure failure. + +The default `--runner codex` reports usage only after completion and the current +Codex CLI has no compatible hard-limit option, so both `--preflight-only` and +model-backed agent evaluation refuse it before any model call. Post-run +observation, a fixed repetition count, and operator intent are not enforcement +substitutes. + +Explicit `--runner capped-codex` starts a new loopback Responses proxy for each +agent execution. It counts the complete submitted input through +`/responses/input_tokens`, reserves input plus twice the admitted +generated-token envelope before forwarding, clamps `max_output_tokens`, and +sets Codex custom provider request and SSE retries to zero. The doubled envelope +is required because the committed accounting adds reasoning output to Codex's +already-inclusive output count. Only foreground `/responses` calls +with local function/custom tools are supported. Background calls, compaction, +other or cost-bearing endpoints and tools, insufficient allowance, missing +usage, duplicate or declared retries, and accounting disagreement fail closed. +Interrupted responses retain their reservations. Completion additionally +requires exact agreement between cumulative proxy usage and Codex final usage. +The execution report's `hard_cap_ledger` contains sanitized totals, request +counts, retry settings, and fixed failures; request content, credentials, and +provider response text are excluded. + +The runner also requires the selected model to exist in the chosen Codex +executable's bundled catalog. It writes that one exact entry to an ephemeral +`model_catalog_json` so shell and MCP tool definitions remain model-compatible, +then removes it after the execution. The proxy answers Codex's cost-free model +discovery locally and never forwards it upstream. + +After a compatible runner passes that capability gate, the remaining configured +preflight still requires clean target and CodeMesh checkouts, exact +source/index/plan/binding agreement, committed guidance, the normal four-tool +surface, a passing context probe, and a passing wrong-checkout rejection probe. +Its report retains hashes and stable identities rather than raw guidance or MCP +payloads. + +Configured suites may declare `baseline_mcp_servers`. Those servers must exist +in the reviewed checkout-local configuration and are supplied identically to +control and treatment; treatment then adds CodeMesh. Inline environment values +are rejected for baseline servers, while declared `env_vars` names may be +forwarded. Spontaneous and evaluator-assisted suites retain their historical +isolated diagnostic-profile behavior. + +`feedback record` continues to write `codemesh-feedback/v1` packets only to an ignored +`.codemesh-feedback/` outbox inside the checkout. Linked outboxes are rejected, +and packet files are created exclusively so an existing file or link cannot be +overwritten. Packets contain commit, dirty-state hash, binding, +snapshot, languages, parser profile, categorized paths, validation result, +confidence, and bounded sanitized descriptions; they exclude raw prompts, MCP +payloads, source excerpts, and environment values. `feedback validate` checks +the strict schema and content id. `feedback summarize` groups validated packets, +preserves exact provenance, prioritizes open failures, and recognizes passing +rechecks that explicitly supersede earlier failures. + +`feedback session plan` creates a reviewable +`codemesh-development-feedback-session-plan/v1` artifact for an exact CodeMesh +checkout and one or more client checkouts. The plan pins an immutable manifest, +expiry of no more than seven days, exact profile tool lists, local/provider-free +declarations, reporter identities, and packet, request, text, and path bounds. +It verifies that every client outbox is ignored and unlinked. +`feedback session activate` requires the reviewed plan hash and creates the +manifest exclusively. `inspect` reports bounded status without client roots; +`revoke` requires the manifest hash and creates a separate immutable revocation +record. Neither close nor expiry removes retained packets. + +`codemesh_record_feedback` accepts only bounded observation fields. It fixes +classification to `diagnostic` and derives the session, reporter role, +repository Git state, binding, snapshot/parser availability, and CodeMesh Git +state on the server. Accepted snapshot/parser provenance comes from the most +recent successful status or context-package call in the same stdio process; +the recording endpoint does not issue a new store request. Setup, binding, and freshness failures may record a +bounded unavailable-provenance code. Content, ranking, language, relationship, +budget, and validation reports require an accepted fresh snapshot and parser +provenance. The resulting strict `codemesh-feedback/v2` packet is +content-addressed and append-only. Validation and summarization accept both v1 +and v2. + +The maintainer tools enumerate only roots in the active manifest. By default +they expose only v2 packets from that session; `include_legacy=true` may expose +valid v1 packets as `legacy_unscoped`. Resolution preparation validates an +agent-authored draft and returns a stable hash with +`human_review_required=true` and `human_approved=false`. It never writes the +plan, changes CodeMesh, records approval, invokes a provider, or transmits +feedback. + +A v2 packet may name superseded feedback only when it is a passing recheck. +Maintainer status derives an earlier report as resolved only when the recheck +comes from the same checkout and matches its issue category and task family. + ## .NET Client Use `CodeMesh.Control.AgentAccess.AgentAccessClient` for typed .NET calls: diff --git a/docs/current/architecture.md b/docs/current/architecture.md index 6432d56..258b7d7 100644 --- a/docs/current/architecture.md +++ b/docs/current/architecture.md @@ -4,6 +4,10 @@ Document type: current implementation reference CodeMesh is organized around a write-side ingestion pipeline and a read-side Agent Access surface. +For a progressive introduction and glossary, start with the +[Developer handbook](../guides/developer-handbook.md). This page owns component +boundaries and code navigation; linked chapters explain the algorithms. + ## Product Boundary The architecture is service-capable, but the initial product wedge is narrower: @@ -24,7 +28,8 @@ not source truth. ## High-Level Flow 1. A repository is selected through CLI options or configuration. -2. A parser worker emits code nodes, relationships, diagnostics, and source spans. +2. One parser worker or an exact language-set composite emits code nodes, + relationships, diagnostics, and source spans. 3. Ingestion computes a deterministic snapshot id, stages graph records, source content, vector embeddings, generated summaries, repository registry records, and ingestion run records under that snapshot namespace. 4. The registry atomically publishes checkout-current and eligible clean branch-head slots; failed or stale generations do not replace active slots. 5. Agent Access resolves a project, alias, or snapshot reference to an active snapshot and reads the stores through REST, MCP tools, Python CLI commands, and the .NET client. @@ -43,16 +48,25 @@ not source truth. - Health contracts. - Hashing, fingerprinting, and repository alias utilities. - A shared repository path policy for excluding generated, cache, virtual-environment, smoke, and temporary directories. +- Default known-secret-file exclusion plus repository-relative allow and deny + patterns shared by parsing, snapshot identity, refresh, and watch. +- Fail-closed exclusion of symlink, junction, and reparse-point path components + below the selected repository root, including discovered and explicitly + selected C# solution/project manifests. ### Ingestion `src/CodeMesh.Ingestion` coordinates repository ingestion: - Calls parser workers. +- Canonicalizes selected language sets and merges every parser result before + computing or publishing one snapshot; parser errors abort publication. - Handles dry-run and persistent ingestion modes. - Resolves persistent project, alias, and checkout identity from private Git/local markers. - Computes deterministic source snapshots and stages cross-store generations. +- Includes the effective path-policy version and normalized filter configuration + in snapshot identity. - Publishes checkout-current and clean branch-head slots with compare-and-swap semantics. - Writes graph, content, vector, summary, registry, and run data. - Performs incremental skip logic for unchanged records. @@ -62,13 +76,22 @@ not source truth. - Assigns versioned summary complexity tiers from node kind, source size, truncation, and available parser metadata; the selected token budgets are persisted with each generated summary. +- Runs summary-model qualification through the same prompt, parser, budget, + provider, and redaction path; records immutable suite/profile/source identity, + emits restricted blinded-review artifacts, and compiles sanitized reports + only after compatible live-retrieval evidence is bound. ### Parser Workers `src/CodeMesh.Parser.CSharp` is the C# parser worker: - Exposes `/health`, `/capabilities`, and `/parse`. -- Uses Roslyn to parse solutions and projects. +- Uses Roslyn to parse solutions and projects. Its container runtime includes + the .NET SDK required by `MSBuildWorkspace`; HTTP health alone does not prove + project loading. Standard design-time outputs are isolated in request-owned + temporary directories in the container, preserving compiler metadata and the + read-only source mount. See the [runtime decision](../decisions/0004-parser-container-requires-dotnet-sdk.md) + and [output-isolation decision](../decisions/0005-isolate-parser-design-time-outputs.md). - Emits file, test file, namespace, declaration, type, member, test case, and relationship nodes. - Represents partial types as logical symbols plus declaration nodes. - Detects common .NET test attributes and CodeMesh-style `Test...` methods in test files as `TestCase` nodes. @@ -77,10 +100,25 @@ not source truth. `src/CodeMesh.Parser.Python` is the local Python parser: - Parses `.py` files from a repository root. -- Emits file, class, function, and async function nodes. +- Emits file, class, function, async function, import, and test-case nodes. - Emits containment relationships based on indentation. +- Handles multiline definition headers, repeated guarded imports, local calls, + and calls through imported aliases. - Uses the shared repository path policy to exclude generated/cache directories such as `.venv`, `.tmp`, `__pycache__`, `.pytest_cache`, smoke output, and build output. -- Does not yet emit Python call/import/type relationships. +- Does not provide complete dynamic-import, data-flow, or type resolution. + +`src/CodeMesh.Parser.Rust` is the local Rust parser: + +- Parses `.rs` files from a repository root while excluding Cargo `target`. +- Emits module, struct, enum, trait, impl, function, method, `use`, and test-case + nodes with source spans and content hashes. +- Emits containment, implementation, reference, invocation, and type-use + relationships. +- Records PyO3 module, class, function, and exported-name metadata. Composite + Python/Rust ingestion links unambiguous static Python imports and calls to + those exports. +- Uses source-structural matching rather than rustc semantic analysis; macro, + dynamic-import, and conditional-compilation resolution remain bounded. `src/CodeMesh.Parser.Deployment` is the local deployment artifact parser: @@ -107,6 +145,9 @@ not source truth. - MongoDB for content, repository registry, ingestion runs, and node summaries. - Qdrant for vector embeddings. - In-memory storage for tests. +- Neo4j node and relationship upserts are split into bounded auto-commit batches + so large repositories do not require one graph-sized transaction. Snapshot + slots remain unpublished until all store writes complete. ### Control And CLI @@ -116,7 +157,8 @@ not source truth. - Workspace/configuration helpers. - Status and doctor checks. - Typed Agent Access .NET client. -- Ingestion, repository, run, summary, and embedding verification commands. +- Single- and multi-language ingestion, repository, run, summary, and embedding + verification commands. Refresh and watch preserve the canonical language set. ### Agent Access @@ -125,10 +167,28 @@ not source truth. - FastAPI REST endpoints. - Profile-selected MCP tool manifests and handlers; the default normal-user profile exposes the four package-first read tools, while the diagnostic - profile adds non-destructive inspection tools. + profile adds non-destructive inspection tools. Explicitly activated + development-feedback and feedback-maintainer profiles add bounded local + packet recording or allowlisted read-only intake without changing either + existing profile. - Python CLI commands. - Store read paths for graph, content, summaries, vectors, repositories, and ingestion runs. - Context ranking, formatting, and package generation. +- Exact normal-profile project/checkout/root/source-view binding. Bound tools + inject the selected project and reject caller overrides or stale, dirty, + mismatched, missing, and ambiguous checkout state. +- A hashed installation planner/applier that forwards environment variable + names rather than secret values, plus provider-free stdio probes that check + the exact launch command, profile tool surface, binding, session identity, + freshness, or feedback discovery as applicable. Installation-plan v2 pins + feedback session path and hash while retaining v1 normal-plan loading. +- A versioned local feedback recorder, validator, and prioritizing summarizer, + plus immutable development-session manifests and revocation records. V2 MCP + packets derive repository, binding, snapshot/parser availability, session, + and CodeMesh provenance in the server. Maintainer intake scans only + allowlisted outboxes and deterministically hashes non-authorizing resolution + drafts. Raw prompts, payloads, source excerpts, credentials, and environment + values are not retained. ## Main Data Stores @@ -147,6 +207,141 @@ not source truth. - Add context package enrichers for documentation, deployment, test, and SDLC metadata. - Add model providers for embeddings and summaries. +These are existing seams, not an execution plan. Changes still follow +[Next Steps](../planning/next-steps.md) or explicit owner direction. + +## Processes and dependency direction + +```mermaid +flowchart LR + Operator[Developer or operator] --> CLI[.NET CLI] + CLI --> Ingest[Ingestion orchestrator] + Ingest --> Local[Local parser clients] + Ingest --> HTTP[HTTP parser client] + HTTP --> Worker[CSharp ASP.NET worker] + Local --> Domain[Domain records] + Worker --> Domain + Ingest --> Graph[(Neo4j graph)] + Ingest --> Mongo[(MongoDB content and registry)] + Ingest --> Vector[(Qdrant vectors)] + Ingest -. optional .-> Models[Model providers] + Host[Agent host] --> MCP[Python stdio MCP] + Human[Human or .NET client] --> REST[Python REST and UI] + MCP --> Read[CodeMeshReadStore] + REST --> Read + Read --> Graph + Read --> Mongo + Read --> Vector + Read -. query embeddings .-> Models +``` + +The arrows show calls, not a requirement to start every process. Local parsing +is the default in the CLI. The C# HTTP worker is an alternative parser boundary. +The Python MCP process and REST process each construct their own read facade; +MCP does not relay requests through REST. Both use the same stores. Normal MCP +needs local filesystem access to the bound checkout for freshness. + +The composition root is +[CLI Program.cs](../../src/CodeMesh.Cli/Program.cs): `RunIngestAsync` constructs +parsers, stores, and providers explicitly and injects their contracts into +`IngestionOrchestrator`. There is no ingestion dependency-injection container. +The [C# worker Program.cs](../../src/CodeMesh.Parser.CSharp/Program.cs) registers +`CSharpParseService` as a singleton in ASP.NET and passes request cancellation +to `ParseAsync`. + +At the project-reference level, all parser projects depend on Domain. Storage +depends on Domain; Ingestion depends on Domain and Storage contracts; Control +depends on Domain. CLI references and composes these projects. These boundaries +are visible in the [CLI project](../../src/CodeMesh.Cli/CodeMesh.Cli.csproj), +[Ingestion project](../../src/CodeMesh.Ingestion/CodeMesh.Ingestion.csproj), and +[Control project](../../src/CodeMesh.Control/CodeMesh.Control.csproj). Parser +selection belongs in CLI; persistence decisions belong in Ingestion and Storage. + +Python startup enters +[cli.main](../../agent-access/codemesh_agent_access/cli.py). +`rest` starts Uvicorn with `rest.app`; `mcp` starts the stdio server. +[dependencies.get_settings/get_store](../../agent-access/codemesh_agent_access/dependencies.py) +cache one settings object and facade per process. The REST lifespan closes the +owned stores; `mcp.run` delegates its lifetime to FastMCP's `server.run` and has +no equivalent explicit `close_store` hook. Mongo's synchronous operations +are dispatched through `asyncio.to_thread`, while the graph adapter uses the +async Neo4j driver. This means changing an environment variable in a running +process does not reconstruct its cached settings. + +## Project and package map + +| Location | Responsibility and useful starting symbols | Representative validation | +| --- | --- | --- | +| [CodeMesh.Domain](../../src/CodeMesh.Domain/) | `CodeNode`, `ParseRequest`, `SnapshotPublication`, `RepositoryPathPolicy`; shared contracts and fingerprints. | `TestHash`, path-policy tests, `TestAgentAccessContractsMapRestJson`. | +| [CodeMesh.Cli](../../src/CodeMesh.Cli/Program.cs) | Dispatch, `RunIngestAsync`, `CreateParserClient`, `RunRefreshAsync`, `RunWatchAsync`, provider/store factories. | Workflow argument tests and end-to-end smoke. | +| [CodeMesh.Control](../../src/CodeMesh.Control/) | `EnvironmentFileLoader`, `CodeMeshWorkspace`, refresh/watch helpers, `CodeMeshStatusService`, `AgentAccessClient`. | Environment, workflow, status, and client tests. | +| [CodeMesh.Ingestion](../../src/CodeMesh.Ingestion/) | `IngestionOrchestrator`, `CompositeParserClient`, identities, cleanup, providers and qualification. | Incremental, multi-language, redaction, snapshot, and summary tests. | +| [CodeMesh.Parser.CSharp](../../src/CodeMesh.Parser.CSharp/CSharpParseService.cs) | `LoadProjectContextsAsync`, `ExtractDeclarationNodes`, `ExtractRelationships`; local and HTTP use of Roslyn. | Compile-item, partial-declaration, cross-project-call tests. | +| [Other parsers](ingestion-internals.md#parser-specific-mechanisms) | Python, Rust, Markdown, and deployment `ParseAsync` implementations. | Corresponding synthetic parser fixtures in the .NET harness. | +| [CodeMesh.Storage](../../src/CodeMesh.Storage/) | Adapter implementations of `StorageContracts.cs`; graph schema, Mongo documents, vector payloads, local in-memory model. | Adapter round-trips and lifecycle tests. | +| [Agent Access core](../../agent-access/codemesh_agent_access/store.py) | `CodeMeshReadStore`, lexical scoring, expansion, package assembly, freshness. | `test_context_ranking.py`, `test_store_components.py`. | +| [Agent Access interfaces](agent-access-contracts.md) | `rest.py`, `mcp.py`, `tools.py`, `cli.py`, `models.py`, `formatting.py`, `web.py`. | REST, MCP, formatting, and CLI contract tests. | +| [Onboarding and feedback](enrichment-and-evaluation.md#installation-probing-and-feedback) | `binding.py`, `installer.py`, `probe.py`, `feedback.py`, `feedback_session.py`. | Installer/probe, binding, feedback, session, and provider-free stdio integration tests. | +| [Evaluation](../../agent-access/codemesh_agent_access/evaluation/) | Fixture, live, agent, model, metrics, and capped-runner modules. | Evaluation orchestration and accounting tests. | + +The .NET test symbols above live in +[tests/CodeMesh.Tests/Program.cs](../../tests/CodeMesh.Tests/Program.cs); Python +tests live in [agent-access/tests](../../agent-access/tests/). See the +[test harness map](../evaluation/testing.md#harness-implementation-and-invariant-map) +for what they prove. + +## Configuration and startup ownership + +`Program.cs` reads the command, takes `--root` or the current directory for its +initial environment-file load, and calls +`EnvironmentFileLoader.LoadIfExists`. The loader fills only absent process +variables, ignores comments and malformed assignments, and removes paired +quotes. Explicit store/provider command options normally precede environment +variables, which precede each factory's defaults. This is implemented directly +in factories such as `CreateGraphStore`, not in a general configuration binder. + +[codemesh.yaml](../../codemesh.yaml) is a workspace marker and descriptive +configuration inventory. `CodeMeshWorkspace.Discover` and workflow helpers use +its presence, but the current CLI does not deserialize it as a runtime settings +hierarchy. Changing `ingestion.batch_size` there does not change Neo4j's +`WriteBatchSize`; listing an optional provider there does not implement one. +Source options, environment loading, and +[Python Settings.from_env](../../agent-access/codemesh_agent_access/config.py) +are the operational configuration paths. + +Python `Settings.from_env` reads process environment, without loading `.env`. +The REST and stdio launches therefore need the same intended store names and +endpoints supplied to their own processes. Compose supplies environment to its +containers; the host-side .NET loader does not export values back into the +parent shell. [Development and debugging](../guides/development-and-debugging.md) +explains how to avoid confusing those environments. + +[docker-compose.yml](../../docker-compose.yml) runs Neo4j, MongoDB, Qdrant, the +C# parser, and Agent Access, with Ollama behind an optional profile. Host ports +are loopback-bound; Agent Access's container port is 8080 and its default host +mapping is 8088. A direct Python launch defaults to port 8080. Compose's +`depends_on` supplies startup ordering for stores, without an application +readiness health condition. The C# container sees this checkout read-only at +`/workspace`; arbitrary host paths are not automatically available inside it. + +## Where to change what + +| Intended change | First implementation seam | Consumers that need review | +| --- | --- | --- | +| Select or exclude files | `RepositoryPathPolicy` and parser enumeration | Snapshot scope, C# manifest containment, refresh/watch, security docs. | +| Change C# graph output | `ExtractDeclarationNodes` / `ExtractRelationships` | Stable ids, storage payloads, retrieval expansion, parser tests. | +| Change cross-language links | `CompositeParserClient.AddPyO3Relationships` | Python import metadata, Rust export metadata, relationship ranking. | +| Change snapshot identity or retention | `SnapshotIdentity`, `BuildPublicationAsync`, `MongoRegistryStore` | In-memory model, Python selector/binding reads, cleanup, ADR 0001. | +| Change relevance or file coverage | `_lexical_score`, `_ranked_unique_hits`, diversification helpers | Lexical candidate acquisition, context expansion, frozen evaluation suites. | +| Change package content or budgets | `get_context_package`, `_snippet_for_hit`, formatter | MCP cap, CLI output, Python and .NET response models. | +| Add or change an interface field | `models.py` and the responsible route/tool | MCP manifest and signatures, `tools.py`, CLI, .NET JSON attributes, UI. | +| Change provider generation | Provider contract, summary/embedding adapter | Generation identity, budget policy, skip logic, usage and qualification tests. | +| Change troubleshooting visibility | Status service, freshness helpers, run metadata | REST/MCP response contracts and sanitized diagnostics. | + +The [change guide](../guides/making-changes.md) turns these seams into concrete +walkthroughs. Avoid beginning with a broad refactor: the current source already +provides narrower places to modify most behavior. + ## Planning References - [Project Status](project-status.md) records the current implementation and diff --git a/docs/current/enrichment-and-evaluation.md b/docs/current/enrichment-and-evaluation.md new file mode 100644 index 0000000..39f5bcc --- /dev/null +++ b/docs/current/enrichment-and-evaluation.md @@ -0,0 +1,257 @@ +# Enrichment and evaluation internals + +Document type: current implementation reference + +Embeddings and generated summaries enrich a source index. Installation/probing +connects that index to a particular agent host, feedback records observations, +and evaluation measures specified behavior. These components do not replace +source evidence or turn implementation into demonstrated product benefit. +The [handbook](../guides/developer-handbook.md) explains the core workflow. + +## Embedding generation and query embedding + +Write-side embedding providers implement +[`IEmbeddingProvider`](../../src/CodeMesh.Ingestion/Embedding/IEmbeddingProvider.cs). +The CLI selects +[`OllamaEmbeddingProvider`](../../src/CodeMesh.Ingestion/Embedding/OllamaEmbeddingProvider.cs), +[`LmStudioEmbeddingProvider`](../../src/CodeMesh.Ingestion/Embedding/LmStudioEmbeddingProvider.cs), +or [`NoEmbeddingProvider`](../../src/CodeMesh.Ingestion/Embedding/NoEmbeddingProvider.cs). +The orchestrator gives the provider nodes plus a dictionary of **redacted** +content. Providers create bounded text inputs with node context and source, +batch requests, and attach provider/model/content provenance to returned vectors. + +The embedding id includes provider, model, node id, and content hash; Qdrant's +point key additionally incorporates the storage namespace. The orchestrator's +incremental lookup checks provider/model and node/content identity, and rewrites +embeddings for rewritten nodes. `--force-embeddings` bypasses the unchanged-vector +skip and implies inclusion; it conflicts with `--skip-embeddings`. +[`EmbeddingVerifier.VerifyAsync`](../../src/CodeMesh.Ingestion/Embedding/EmbeddingVerifier.cs) +compares graph and vector state for the selected provider/model. It checks +coverage and stale/missing records, not embedding quality. + +Ollama generation uses its batch embedding endpoint and falls back to the legacy +single-input endpoint only for HTTP 404/405. LM Studio uses the compatible +embedding response shape. A wrong vector count or malformed dimension batch is +an error, not a partially successful embedding result. The .NET tests +`TestOllamaEmbeddingProviderMapsResponses`, +`TestLmStudioEmbeddingProviderMapsResponses`, +`TestForcedEmbeddingIngestionRebuildsUnchangedEmbeddings`, and +`TestEmbeddingVerifierReportsMissingAndStaleVectors` in +[Program.cs](../../tests/CodeMesh.Tests/Program.cs) cover these seams. + +Read-side +[`QueryEmbeddingProvider`](../../agent-access/codemesh_agent_access/embeddings.py) +is a separate Python adapter configured by `CODEMESH_MODEL_PROVIDER` and +`CODEMESH_EMBEDDING_MODEL`. It prefixes query text with `search_query:`, requests +one vector, and supplies it to Qdrant search. Provider health checks whether the +configured model is listed. Setting write-side flags does not configure the +Python process: model and vector dimensions must be compatible across both +sides. `test_query_embedding_provider_prefixes_queries` and provider-health +tests in [test_store_components.py](../../agent-access/tests/test_store_components.py) +cover mapping rather than live model quality. + +## Summary selection, budgets, identity, and persistence + +[`CodeSummaryGenerator.GenerateAsync`](../../src/CodeMesh.Ingestion/Summary/CodeSummaryGenerator.cs) +selects candidates using +[`CodeSummaryIdentity.IsCandidate`](../../src/CodeMesh.Ingestion/Summary/CodeSummaryIdentity.cs). +Eligible kinds include files/test files, types, methods/test cases, constructors, +properties, and events. Declaration nodes are not candidates. In particular, +a C# logical symbol's summary input can be signature-oriented; generation does +not automatically traverse to all its declaration bodies. + +`GenerateAsync` derives a generation-configuration fingerprint, calculates +summary ids, loads existing summary state, and skips only matching **completed** +ids unless forced. An id binds the snapshot namespace, node/content, summary +kind, provider/model, prompt version, and generation fingerprint. Changing +configured generation behavior can regenerate summaries without changing the +core source snapshot. Matching alternate provider/model/prompt summaries can +remain retained for the same current content. + +[`SummaryBudgetPolicy.CreateRequest`](../../src/CodeMesh.Ingestion/Summary/SummaryBudgetPolicy.cs) +chooses the maximum of kind, source-size/span, and parser-metadata tiers. +Truncated or large source promotes the tier. The current tiers are Compact, +Standard, Complex, and Aggregate, with visible summary budgets of 128, 192, 320, +and 448 tokens. The request separately records input-character allowance, +reasoning reserve, and maximum completion budget; these fields serve different +purposes. Input size is a character count, not an exact tokenizer measurement. + +[`SummaryPrompt`](../../src/CodeMesh.Ingestion/Summary/SummaryPrompt.cs) owns +prompt construction, version/hash, structured response parsing, and supported +confidence normalization. Provider adapters implement +[`ICodeSummaryProvider`](../../src/CodeMesh.Ingestion/Summary/ICodeSummaryProvider.cs) +for Ollama, LM Studio, and OpenAI. The latter's implementation is in +[OpenAiCodeSummaryProvider.cs](../../src/CodeMesh.Ingestion/Summary/OpenAiCodeSummaryProvider.cs); +it maps request options and response usage to the shared result, including +requested/effective generation metadata. This is the inspected adapter contract, +not a current external model availability recommendation. + +Generation is sequential per candidate. `CreateSummaryAsync` catches individual +non-cancellation provider exceptions and writes a failed summary with error +metadata. `GenerateAsync` upserts both successful and failed records, cleans +obsolete identities, and reports completed/failed/skipped counts. Cancellation +propagates; a summary-store write failure also escapes to ingestion. + +The [Mongo summary adapter](../../src/CodeMesh.Storage/Mongo/MongoNodeSummaryStore.cs) +migrates the old unique identity index to `node_summary_identity_v2` so generation +fingerprints can coexist correctly. Python +[summary_store.py](../../agent-access/codemesh_agent_access/summary_store.py) +searches completed records and exposes coverage. Node lookup first seeks the +requested content hash, then can fall back to a completed summary for that node +in the namespace; consumers should retain the returned summary's own provenance. + +Tests in the [.NET harness](../../tests/CodeMesh.Tests/Program.cs) cover +`TestSummaryBudgetsScaleWithNodeComplexity`, +`TestSummaryGeneratorPersistsBudgetMetadata`, +`TestSummaryGeneratorRegeneratesWhenConfigurationChanges`, and +`TestMongoSummaryStoreMigratesGenerationIdentity`. Python's +`test_node_summary_lookup_falls_back_when_content_hash_differs` in +[store tests](../../agent-access/tests/test_store_components.py) documents the +read-side fallback explicitly. + +## Disabled and failed capabilities + +| Condition | Implemented consequence | +| --- | --- | +| Ingest without inclusion flags | CLI supplies no-op providers and skips embedding/summary generation; source parsing and persistence proceed. | +| Python provider mode `none` | No query embedding call. Lexical graph search and stored-summary search remain available. | +| No stored summaries | Packages still contain source hits and snippets; `summary` can be null. | +| Query embedding HTTP failure | Python adapter returns an empty vector for caught HTTP errors; text search can continue. Malformed response values are not all caught. | +| Write-side embedding failure | Normally throws through the orchestrator and prevents the new slot publication. | +| Individual summary-provider failure | Failed summary record/counter; source ingestion can complete. A completed source run does not mean complete summaries. | +| Summary-store or core-store write failure | Escapes and fails the staged generation; already completed writes are not rolled back. | + +These branches are implemented in the +[CLI factories](../../src/CodeMesh.Cli/Program.cs), +[orchestrator](../../src/CodeMesh.Ingestion/IngestionOrchestrator.cs), and provider +modules linked above. Provider mode `none` does not erase prior enrichment or +eliminate store dependencies. Use isolated no-summary indexes when an evaluation +requires a genuine no-summary baseline. + +## Summary-model qualification + +The production generation path is also used by +[`SummaryQualificationRunner`](../../src/CodeMesh.Ingestion/Summary/Qualification/SummaryQualification.cs). +The runner validates a frozen suite/profile and clean source identity, redacts +input, performs warm-up and measured repetitions, and emits a private archive +plus model-blinded review packet. The example +[suite](../evaluation/assets/summary-qualification-suite.example.json) is +synthetic wiring evidence, not a sufficient quality corpus. + +The CLI's `summaries qualify bind-retrieval` binds distinct no-summary/candidate +live reports, matching suite/repository/commit, a shared ranking identity, and a +reviewed query-level assessment to that archive. `SummaryQualificationCompiler` +requires complete compatible reviews, retrieval evidence, and resource/cost +evidence before producing a sanitized result. Missing evidence means an invalid +run, not a passing candidate. `TestSummaryQualificationRunnerAndCompiler` in +[Program.cs](../../tests/CodeMesh.Tests/Program.cs) uses controlled providers and +artifacts to exercise these gates. + +[ADR 0002](../decisions/0002-summary-qualification-evidence-boundary.md) records +why generation/compilation stays in .NET and retrieval measurement stays in the +existing Python evaluator. Operating commands remain in +[COMMANDS.md](../../COMMANDS.md#summary-model-qualification), with the full +qualification requirements in +[Summary Model Qualification](../planning/summary-model-qualification.md). +No deployment profile is established as qualified merely by this implementation. + +## Installation, probing, and feedback + +[`create_installation_plan`](../../agent-access/codemesh_agent_access/installer.py) +binds the target checkout to an exact launch and optional onboarding text. A +plan contains complete intended file bytes, previous/new hashes, diffs, and a +plan hash. Application checks the reviewed hash and target drift. Managed blocks +preserve unrelated configuration/guidance. The config names environment +variables to forward rather than embedding their secret values; the generated +launch selects the exact CodeMesh package path and binding. + +[`run_runtime_probe`](../../agent-access/codemesh_agent_access/probe.py) +verifies the plan is applied, starts its stdio command with provider mode `none`, +checks initialization instructions and the normal four-tool manifest, checks +the one bound repository and fresh status, then retrieves a nonempty package. +Expected paths can turn it into a focused retrieval gate. +`run_runtime_rejection_probe` deliberately overrides a binding dimension and +checks rejection. These launch real read processes and use stores; they are +provider-free but not offline unit tests. + +[`record_feedback`](../../agent-access/codemesh_agent_access/feedback.py) +requires an exact Git root and ignored `.codemesh-feedback/` outbox. Its strict +versioned packet records repository and CodeMesh commits, dirty-state hash, +binding/snapshot/languages/parser profile, task family, tool usage, categorized +paths, validation outcome, confidence, and bounded descriptions. Sanitizers +reject unsafe text/path shapes; the content id binds the normalized packet. +`validate_feedback` checks schema and id, and `summarize_feedback` groups evidence, +prioritizes unresolved failures, and recognizes explicit superseding rechecks. + +Feedback does not change ranking, trigger ingestion, transmit itself externally, +or automatically modify the roadmap. Operator-supplied classifications are +observations whose provenance must be reviewed. Tests live in +[test_installer_probe.py](../../agent-access/tests/test_installer_probe.py) and +[test_feedback.py](../../agent-access/tests/test_feedback.py). The canonical +installation and feedback procedure remains [MCP Setup](../guides/mcp-setup.md). + +## Evaluation data flow and evidence classes + +| Harness | Mechanism | What a pass supports | +| --- | --- | --- | +| [fixture.py](../../agent-access/codemesh_agent_access/evaluation/fixture.py) | `run_mcp_evaluations` substitutes `FixtureEvaluationStore`, constructs diagnostic MCP, and calls tools in process. Restores the original store hook in `finally`. | Deterministic tool/format/guidance/redaction-fixture behavior. No network, live-store, or agent-benefit conclusion. | +| [live.py](../../agent-access/codemesh_agent_access/evaluation/live.py) | `McpEvaluationClient` starts stdio; `run_live_evaluation` checks tool-source identity, health, repository and freshness, then warm-up/measured suite cases. | Retrieval, relationships, budgets, timing, and leak checks for that exact suite/index/tool candidate. | +| [agent.py](../../agent-access/codemesh_agent_access/evaluation/agent.py) | `run_agent_evaluation` prepares pinned source, runs randomized control/treatment order per repetition, validates outputs and command policy, and records adoption, correctness, time, and tokens. | A paired outcome in its recorded integration mode and authority envelope. | +| [benchmark.py](../../agent-access/codemesh_agent_access/evaluation/benchmark.py) | `run_model_benchmark` checks clean source/index identity, runs a live gate, then graded query/change tiers with evidence and review artifacts. | Model performance on that benchmark, subject to qualification/review gates. It is a separate runner path from the capped agent campaign. | + +[models.py](../../agent-access/codemesh_agent_access/evaluation/models.py) owns +suite/report schemas. Checked-in [suites](../../agent-access/codemesh_agent_access/evaluation/suites/) +freeze targets, repository commits, thresholds, and tasks. +[`metrics.ranking_metrics`](../../agent-access/codemesh_agent_access/evaluation/metrics.py) +matches normalized target paths/ids/keys, calculates recall and precision at k, +reciprocal rank, and relevance-weighted nDCG, with each target contributing at +most once to ranked gain. A high recall can coexist with poor latency or agent +non-adoption; these are separate measurements. + +Configured agent evaluation verifies reviewed installation, identical relevant +guidance, fresh binding, baseline MCP identities, and positive/rejection probes. +Spontaneous and evaluator-assisted suites retain their own historical diagnostic +profiles and onboarding conditions. Treatment-only assistance changes the +comparison class; it cannot be presented as configured prompt-parity evidence. +Raw traces are optional sensitive artifacts; sanitized retained reports belong +under the existing evidence discipline, not copied into source documentation. + +The actual harness tests are +[test_evaluation.py](../../agent-access/tests/test_evaluation.py), +[test_effectiveness_evaluation.py](../../agent-access/tests/test_effectiveness_evaluation.py), +and [test_model_benchmark.py](../../agent-access/tests/test_model_benchmark.py). +They use test runners and fixtures to validate orchestration without requiring +paid model execution. See [Testing](../evaluation/testing.md) for check selection +and [MCP Effectiveness Evaluation](../evaluation/mcp-effectiveness.md) for live +procedures and verdict interpretation. + +## Hard-capped agent runner + +Before an agent campaign prepares repository work or calls a model, +`require_reported_token_hard_cap` requires a positive integer allowance and a +runner declaring `input-output-reasoning-v1` accounting. Each execution receives +only the remaining allowance. The campaign stops at zero, after incomplete +execution, or on inconsistent/over-limit accounting; it does not automatically +retry. The default `CodexRunner` does not declare compatible enforcement and is +refused. Explicit `CappedCodexRunner` implements the separate bounded path. + +[`capped_codex.py`](../../agent-access/codemesh_agent_access/evaluation/capped_codex.py) +contains `HardCapLedger`, `CappedResponsesProxy`, and `CappedCodexRunner`. The +proxy counts submitted input before generation. `reserve` commits input plus +twice the admitted generated-token allowance because this repository's reported +accounting adds reasoning output to already-inclusive output. `settle` releases +unused allowance only after complete, validated usage agrees with counted input. +Interrupted calls retain their full reservation and stop further admission. + +The runner uses an ephemeral loopback proxy, zero request/stream retries, an +exact bundled model catalog entry, and final agreement between proxy and runner +usage. It rejects unsupported cost-bearing request shapes rather than guessing +their accounting. The sanitized ledger excludes request content and credentials. +Do not assume the separate `eval model` benchmark automatically inherits this +agent-campaign control. + +[ADR 0003](../decisions/0003-hard-capped-codex-evaluation-runner.md) records the +design rationale; [test_capped_codex.py](../../agent-access/tests/test_capped_codex.py) +tests admission, interruption, unsupported activity, retry rejection, and +reconciliation. These are enforcement tests. They do not authorize model spend +or establish that a particular provider-backed campaign completed. diff --git a/docs/current/identity-and-persistence.md b/docs/current/identity-and-persistence.md new file mode 100644 index 0000000..0581c93 --- /dev/null +++ b/docs/current/identity-and-persistence.md @@ -0,0 +1,269 @@ +# Identity and persistence + +Document type: current implementation reference + +Read the [handbook glossary](../guides/developer-handbook.md#vocabulary) first. +This chapter follows +[GraphTypes.cs](../../src/CodeMesh.Domain/Graph/GraphTypes.cs), +[SnapshotContracts.cs](../../src/CodeMesh.Domain/Contracts/SnapshotContracts.cs), +and the [storage interfaces](../../src/CodeMesh.Storage/StorageContracts.cs). + +## From source entities to stored records + +`CodeNode` separates a parser id and stable key from its content hash. Changing +a method body can preserve its logical symbol while changing a declaration's +content. Moving a declaration can change its declaration id and span even if +the logical symbol still exists. A `CodeRelationship` names endpoint node ids, +kind, label, and metadata; these endpoint ids are parser identities, not Mongo +document ids or globally unique Neo4j keys. + +C# makes the separation especially visible. A logical `Method` or `Class` +contains signature-oriented content and an unknown source span. A separate +`Declaration` contains source text and a concrete span. `Defines` points from +declaration to logical symbol. Partial declarations can therefore share one +logical symbol while retaining multiple source locations. Retrieval follows +these links to obtain readable source. See +[`ExtractDeclarationNodes` and `ExtractRelationships`](../../src/CodeMesh.Parser.CSharp/CSharpParseService.cs) +and `TestCSharpParserSeparatesPartialDeclarations` in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs). + +Content is independently addressable. `CodeContent.Hash` identifies text and +`CodeNode.ContentHash` references it. Ingestion redacts text first, remaps node +hashes, and deduplicates content by the resulting hash. Consequently a content +hash is not necessarily the hash of the file on disk: it may describe a +redacted fragment or a logical symbol signature. Preserve file spans as the +navigation coordinates and the hash as stored-content identity. +[ContentRedactor](../../src/CodeMesh.Ingestion/ContentRedaction.cs) and +`TestIngestionRedactsSensitiveContentBeforeStoresAndEmbeddings` cover this +transformation. + +## Project, checkout, and alias + +[RepositoryIdentityService.ResolveAsync](../../src/CodeMesh.Ingestion/RepositoryIdentityService.cs) +uses private local markers. In a Git checkout, `project.json` is under the Git +common directory's `codemesh` directory and `checkout.json` under the +worktree-specific Git directory's `codemesh` directory. Linked worktrees share +the former and get separate checkout identities. Non-Git roots use +`.codemesh/project.json` and `.codemesh/checkout.json` below the selected root. + +An explicit `--project-id` takes precedence over an existing project marker. +Otherwise the service uses the marker or generates a new `prj_` id. A remote +URL is recorded as provenance; it does not automatically attach separate +clones to the same project. Marker files are written through a temporary file +and replacement. This protects an individual marker write, not the entire +identity/registry operation. + +For an existing checkout marker, the service looks up its registered root. If +the old root differs and still exists, it treats the marker as a live copy and +allocates a new checkout id. If the old directory has disappeared, moving the +checkout can retain its identity. The alias is retained from the project marker +or registered project before falling back to a path-derived alias. + +The corresponding tests are `TestCheckoutIdentitySurvivesMovesAndRejectsLiveCopies`, +`TestLinkedWorktreesShareProjectsButNotCheckouts`, and +`TestRepositoryAliasesAreGeneratedAndResolved` in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs). Alias uniqueness is not +enforced by a global unique index; lookup code rejects ambiguity. See +[MongoRegistryStore.GetRepositoryAsync](../../src/CodeMesh.Storage/Mongo/MongoRegistryStore.cs) +and Python's +[find_repository_document](../../agent-access/codemesh_agent_access/content_store.py). + +## Deterministic snapshot calculation + +[`SnapshotIdentity.Compute`](../../src/CodeMesh.Ingestion/SnapshotIdentity.cs) +operates on redacted, unstamped parser output. It sorts canonical node, +relationship, and content descriptions, hashes the sequence into a source-view +hash, and combines it with the other identity dimensions: + +```text +snapshot id = "snp_" + StableId( + identity format, project id, source-view hash, + scope fingerprint, parser-profile fingerprint, graph-schema version) +``` + +This is explanatory notation, not a second implementation. `StableId` is +defined in [CodeMeshHash](../../src/CodeMesh.Domain/Utilities/CodeMeshHash.cs). +Scope includes the canonical language set, repository-relative selected +solution/project paths, sorted options, and normalized path-policy fingerprint. +The parser profile includes parser name, language, redaction-profile version, +and path-policy version. It does **not** automatically hash the parser binary +or CodeMesh commit; a behavioral change may require a deliberate identity +version/profile decision. + +[CodeMeshFingerprint](../../src/CodeMesh.Domain/Utilities/CodeMeshFingerprint.cs) +excludes runtime metadata such as checkout/root, commit, run id, and last-seen +time. These are stamped afterward by `RefreshMetadata`. Embeddings and +summaries are also outside core snapshot identity, so enriching an existing +snapshot does not create a new source snapshot. + +Identical effective input can reuse one snapshot across commits and worktrees. +That does not promise every filesystem move yields identical parser output: +some structural parsers derive `CodeNode.Project` from the root directory name, +and that field participates in the node fingerprint. Distinguish persistence +of the project/checkout ids from stability of all parsed fields. + +## Store ownership and physical keys + +The .NET ingestion adapters own normal index writes. Python owns retrieval and +also exposes an explicit administrative deletion path. The +[in-memory store](../../src/CodeMesh.Storage/Memory/InMemoryCodeMeshStore.cs) +implements the same interfaces for tests, but its lock-based behavior does not +establish production MongoDB race or crash guarantees. + +| Store and adapter | Records and keys | Scope, indexes, and consumers | +| --- | --- | --- | +| [Neo4jGraphStore](../../src/CodeMesh.Storage/Neo4j/Neo4jGraphStore.cs) | `CodeMeshNode` nodes and `CODEMESH_REL` relationships. `storageKey` is `snapshotId:parserId`. | Unique node storage-key constraint; indexes on code id, relationship id/storage key, repository/node id, and repository/run fields. Python [graph_store.py](../../agent-access/codemesh_agent_access/graph_store.py) must use the same key convention. | +| [MongoContentStore](../../src/CodeMesh.Storage/Mongo/MongoContentStore.cs) | Content `_id` is the content hash; text, media type, timestamps. | `repositoryIds`, `repositoryRuns`, and `repositoryLastSeenAtUtc` associate shared content with snapshot namespaces. Removal drops one reference before deleting unreferenced text. Python [content_store.py](../../agent-access/codemesh_agent_access/content_store.py) reads it. | +| [QdrantVectorStore](../../src/CodeMesh.Storage/Qdrant/QdrantVectorStore.cs) | Deterministic UUID point derived from namespace plus embedding identity; vector and searchable payload. | Payload includes node/content/provider/model and snapshot provenance. Batch dimensions must agree. Collection creation uses the first vector width and configured distance, defaulting to cosine. No custom payload indexes are created by this adapter. | +| [MongoNodeSummaryStore](../../src/CodeMesh.Storage/Mongo/MongoNodeSummaryStore.cs) | Summary `_id` includes namespace, node, content, summary kind, provider/model, prompt version, and generation fingerprint. | Unique `node_summary_identity_v2` compound index; repository/node, repository/kind/name, and repository/content-hash indexes. Python [summary_store.py](../../agent-access/codemesh_agent_access/summary_store.py) reads completed summaries and coverage. | +| [MongoRegistryStore](../../src/CodeMesh.Storage/Mongo/MongoRegistryStore.cs) | Project and checkout identity, snapshots, generations, observations, compatibility repository summaries, ingestion runs. | Mongo `_id` keys plus an embedded slot array and revision on each project. The adapter does not create a separate slot collection or additional registry indexes. | + +Graph records retain the parser `id`; their `repositoryId` property is the +**snapshot namespace**, while metadata `projectId` identifies the persistent +project. Agent Access normalizes public hits to `repository_id = projectId` +and `snapshot_id = snapshotId`. Passing a public project id directly to a graph +filter without resolution will miss snapshot records. This mapping is covered +by `test_graph_storage_key_uses_repository_prefix` and +`test_context_package_uses_snapshot_namespace_for_relationships` in +[store tests](../../agent-access/tests/test_store_components.py) and +[ranking tests](../../agent-access/tests/test_context_ranking.py). + +The registry's default collections are: + +| Collection | `_id` and purpose | +| --- | --- | +| `snapshot_projects` | `prj_` id; alias, project revision, and all slots for that project. | +| `snapshot_checkouts` | `chk_` id; owning project, current registered root, timestamps. | +| `snapshots` | `snp_` id; source/scope/profile fingerprints, state, counts, generation and retention time. | +| `ingestion_generations` | `gen_` plus run identity; staging, published, or failed attempt. | +| `snapshot_observations` | Observation identity; checkout/snapshot/commit/branch/dirty-state association. | +| `repositories` | Persistent project id; compatibility summary and default selected-snapshot metadata. | +| `ingestion_runs` | Run id; successful completion metadata, parser/provider identity, diagnostics counts and incremental statistics. | + +`MongoRegistryStoreOptions` exposes collection-name construction options, but +Python uses fixed names for the snapshot collections. Changing those defaults +requires a coordinated read/write contract change. Do not confuse constructor +customizability with an end-to-end configurable deployment surface. + +## Publication order and atomicity + +`IngestionOrchestrator.BuildPublicationAsync` captures expected slot snapshot +ids and versions, creates checkout-current, and adds branch-head only for a +clean named branch. Detached or dirty source can update checkout-current without +moving a clean branch-head. A generation is staged before graph/content/model +writes. All requested store operations and namespace cleanup must finish before +`PublishSnapshotAsync` is called. + +```mermaid +sequenceDiagram + participant I as Ingestion + participant S as Data stores + participant R as Mongo registry + I->>R: Read slot expectations and stage generation + I->>S: Upsert graph, content, optional enrichments + I->>S: Remove obsolete records in selected namespace + I->>R: PublishSnapshotAsync with expected slots + R->>R: Check expectations + opt Expectations match + R->>R: Write observation and snapshot/generation metadata + R->>R: Replace project document if revision matches + end + alt Publication succeeds + R-->>I: Published + I->>R: Write compatibility repository and completed run + else Slot or revision conflict + R-->>I: Conflict result + I->>R: Attempt MarkGenerationFailedAsync + I->>I: Throw without writing completed run + end +``` + +The atomic operation is the final revision-checked replacement of one project +document containing its slot array. It prevents that replacement from applying +against a changed project revision and updates the requested slots together. +Neo4j writes, Mongo content updates, Qdrant writes, generation metadata, and +compatibility registry records are **not** in that atomic operation. + +`MongoRegistryStore.PublishSnapshotAsync` writes observation, generation, active +snapshot, and collectible metadata before the project compare-and-swap. A +conflict can therefore leave those records ahead of the actual slot selection. +The orchestrator attempts `MarkGenerationFailedAsync` using +`CancellationToken.None`, then rethrows. Active slot selection is the primary +publication fact; a snapshot or generation state alone is insufficient. +Likewise, `WriteRegistryAsync` runs after publication, so a later registry/run +write failure can occur after slots have already changed. + +The normal bound reader follows checkout-current, whereas unbound project +lookup first uses the compatibility repository summary. An explicit snapshot +selector can resolve a recorded namespace without testing whether it is active. +Thus staging is a convention enforced by the supported selection path, not an +access-control boundary around the raw stores. See +[get_bound_repository and resolve_snapshot_id](../../agent-access/codemesh_agent_access/content_store.py). + +An identical snapshot can also receive optional enrichment updates in place. +There is no request-wide transaction spanning sequential reads during package +construction. Do not describe snapshots as immutable across all enrichment and +administrative operations or promise a fully atomic multi-store read. + +Representative tests in the [.NET harness](../../tests/CodeMesh.Tests/Program.cs) +are `TestMultiLanguageIngestionPublishesOneDeterministicSnapshot`, +`TestSnapshotSlotsRetainCleanHeadsAndBoundDirtyHistory`, +`TestFailedSnapshotGenerationPreservesActiveSlots`, and +`TestMongoRegistryStoreRoundTripsRepositoriesAndRuns`. They protect the stated +cases, not every interleaving or process-crash point. + +## Retention, pins, and deletion + +The conceptual lifecycle is: + +```mermaid +stateDiagram-v2 + direction LR + [*] --> Staging + Staging --> Active: published + Staging --> Failed: failed + Active --> Collectible: no references + Collectible --> Active: retained again + Collectible --> Deleted: grace and cleanup + Failed --> Deleted: cleanup + Failed --> Staging: later attempt + Deleted --> [*] +``` + +This is the intended lifecycle, with the intermediate-write limits described +above. A referenced or reused active snapshot stays active. `BuildPublicationAsync` +defaults the grace period through the CLI to +86,400 seconds and clamps negative requested values to zero. +`UpsertRetainedSlotAsync` supports `ExplicitPin` and `Evidence`; `DeleteSlotAsync` +uses an expected version and recomputes collectibility. These are registry APIs, +with tests, rather than a documented top-level pin command. A pin preserves a +snapshot; it does not verify the quality or acceptance of the evidence attached +to it. + +[SnapshotGarbageCollector.CollectAsync](../../src/CodeMesh.Ingestion/SnapshotGarbageCollector.cs) +lists eligible unreferenced snapshots, then removes graph, content references, +vectors, and summaries, and finally deletes the snapshot registry record. It +collects per-snapshot non-cancellation failures so a later call can retry. +`DeleteSnapshotRecordAsync` checks for slot references again. There is no +background collector started by `RunIngestAsync`, and there is no lock covering +the whole cross-store deletion against concurrent new pins. Serialize local +cleanup with retention/publication work; a shared coordinator remains deferred. + +[RepositoryCleanupService.DeleteRepositoryAsync](../../src/CodeMesh.Ingestion/RepositoryCleanupService.cs) +is broader: it enumerates every retained snapshot plus the legacy namespace, +cleans all configured stores, then removes registry state. Python's +`CodeMeshReadStore.delete_repository` implements the administrative surface +described in [Repository Delete](agent-access-contracts.md#repository-delete). +Partial cleanup is possible and exceptions do not roll back earlier deletions. +This is why deleting a repository is not a routine cure for a missing search hit. + +Migration from an old absolute-path id occurs only after successful ingestion +under the new identity. The old namespace is registered as collectible with +unknown legacy fingerprints and migrated registry history; its data is not +deleted immediately. See the migration tail of `IngestAsync` and +`MongoRegistryStore.MigrateLegacyRepositoryAsync`. + +For documented rationale, read +[ADR 0001](../decisions/0001-local-repository-identity-and-snapshot-publication.md). +For concrete failure diagnosis, continue to +[Development and debugging](../guides/development-and-debugging.md). diff --git a/docs/current/ingestion-internals.md b/docs/current/ingestion-internals.md new file mode 100644 index 0000000..01cd4f6 --- /dev/null +++ b/docs/current/ingestion-internals.md @@ -0,0 +1,315 @@ +# Ingestion internals + +Document type: current implementation reference + +Ingestion transforms selected source into one index generation. Its main +collaborators are `IParserClient`, graph/content/vector store contracts, +optional summary contracts, and an optional snapshot-aware registry. +[Architecture](architecture.md) locates the projects; +[Identity and persistence](identity-and-persistence.md) owns the identity and +publication details. This chapter explains the work between those boundaries. + +## Request construction and repository selection + +[CLI `RunIngestAsync`](../../src/CodeMesh.Cli/Program.cs) resolves `--root` or +`--repository-root`, canonicalizes languages, resolves explicit solution/project +paths, validates the path filter, then constructs adapters and an +[`IngestionRequest`](../../src/CodeMesh.Domain/Contracts/IngestionContracts.cs). +`CodeMeshIngestPathResolver.ResolveFile` tries an existing repository-relative +path first and then an existing caller-relative path. Prefer absolute +manifest paths when the calling directory differs from the target checkout. +Missing explicit files fail before parsing. + +`CreateParserClient` selects local C#, Python, Rust, Markdown, or deployment +clients. `--parser-url` or `CODEMESH_CSHARP_PARSER_URL` selects +[`HttpParserClient`](../../src/CodeMesh.Ingestion/HttpParserClient.cs), whose +`ParseAsync` posts the same domain request to `/parse`, checks HTTP success, +and deserializes `ParseResult`. The CLI gives that HTTP client a ten-minute +timeout. It rejects HTTP parser configuration with multiple languages because +the implemented composite uses local clients for one combined generation. + +The request defaults to C#. Embeddings and summaries require explicit inclusion +flags; selecting an endpoint or model alone does not enable their write paths. +The [enrichment chapter](enrichment-and-evaluation.md) explains force/skip +behavior. Tests for selection and argument normalization are +`TestIngestPathResolverAcceptsRelativeFiles`, +`TestIngestPathResolverRejectsMissingFiles`, and the refresh/watch workflow +tests in [Program.cs](../../tests/CodeMesh.Tests/Program.cs). + +## Shared path filtering + +[`RepositoryPathPolicy`](../../src/CodeMesh.Domain/Utilities/RepositoryPathPolicy.cs) +is applied before supported parser enumeration. It excludes generated/cache +directories and linked path components, then applies known-secret-file and +configured allow/deny rules. Allow patterns restrict eligible repository-relative +paths; deny patterns win. Pattern normalization and the policy version become +part of the snapshot scope, so changing the filter can change identity even +when the surviving source looks similar. + +The C# parser applies containment to solution/project manifests as well as source +documents. Its `ManifestPathFilter` allows manifest discovery without requiring +every manifest to match a source allow pattern, while preserving the other +policy protections. It rejects an explicitly selected manifest outside the +policy rather than handing it to MSBuild. The full supported glob and secret +policy remains in [Security and Redaction](security-and-redaction.md). + +Test the shared rule and the actual caller. For example, +`TestRepositoryPathPolicyRejectsLinkedPaths` covers the reusable policy, while +`TestCSharpParserAppliesRepositoryPathFilters` verifies parser integration. +`TestWatchWorkflowDetectsFileChanges` covers the watcher using that policy. +All are in the [.NET harness](../../tests/CodeMesh.Tests/Program.cs). + +## Orchestrator call order + +[`IngestionOrchestrator.IngestAsync`](../../src/CodeMesh.Ingestion/IngestionOrchestrator.cs) +performs these phases in order: + +1. Canonicalize language and validate the effective path filter. Capture local + Git provenance. For persistent snapshot-aware operation, resolve project and + checkout markers and register the checkout. +2. Construct `ParseRequest` and await the selected parser. Any diagnostic whose + severity is `error`, case-insensitively, aborts before generation staging and + publication. Warnings remain in the result. +3. Redact content, remap node content hashes, deduplicate content, and compute + deterministic snapshot identity. Stamp storage namespace and provenance onto + nodes and relationships afterward. +4. Read existing state in that namespace from incremental stores. Compare node + content/fingerprints, relationship fingerprints, and content hashes to select + writes and calculate skipped counts. +5. Capture slot expectations, build `SnapshotPublication`, and stage its + generation. Write graph first, then content, optional embeddings, and + optional summaries. +6. Remove obsolete records within the selected namespace using retained id/hash + sets. Publish the slots after these operations succeed. +7. Construct a completed `IngestionResult`, write compatibility repository and + successful run records, then register any legacy namespace for later cleanup. + +This order is visible in the +[ingestion walkthrough](../guides/developer-walkthroughs.md#walkthrough-1-ingest-and-publish-a-small-repository). +It is important that relationships reach storage after their endpoint nodes, +redaction precedes provider calls, and slots move after the data writes. + +## Incremental behavior and failure boundaries + +Incremental here means avoiding unnecessary **writes** after parsing, not +incrementally parsing only changed syntax trees. `IsUnchangedNode` compares +content hash and `CodeMeshFingerprint.ForNode`; relationships use their own +fingerprints. Runtime timestamps and run/checkout metadata are excluded from +those fingerprints. An unchanged record may therefore retain an earlier +per-record ingestion-run stamp; use registry observations and slots to determine +current checkout selection. + +The normal snapshot-aware path compares records in the newly calculated +snapshot namespace. A changed source view usually creates a new namespace, so +its first ingest writes that snapshot's graph. Reingesting identical source can +skip those writes. Globally content-addressed Mongo text is shared through +references, but graph identity is still snapshot-scoped. The alternate path +without `ISnapshotRegistryStore` uses a mutable repository namespace and is +useful in tests; its refresh cleanup is not historical snapshot retention. + +`IIncremental*` stores delete records outside the current sets. Older +`IRefreshable*` interfaces delete stale run stamps instead. Summary cleanup also +runs when summary generation is disabled, retaining only summaries matching +current candidate node/content identities. Skipping embeddings does not run the +embedding-generation/cleanup block. See `TestIncrementalIngestionSkipsUnchangedRecords`, +`TestRepositoryRefreshRemovesStaleRecords`, and +`TestIncrementalIngestionSkipsUnchangedSummaries` in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs). + +Dry-run still parses, redacts, and calculates source identity; it does not +resolve persistent identity markers, stage generations, write stores, or call +model providers through the orchestrator. The CLI constructs configured adapter +objects even for dry-run. Dry-run counts and ids are not proof of a published +snapshot. `TestDryRunSkipsStores` asserts the store boundary. + +There is no whole-ingestion retry loop. A parser exception escapes before +staging. Once staged writes begin, a failure triggers best-effort generation +failure recording and rethrows. Cleanup of the failed namespace is separate. +Provider summary failures have a different boundary: individual non-cancellation +generation errors become failed summary records and counters, allowing the +source snapshot to complete. Embedding failures normally escape and fail the +generation. See [optional capability failures](enrichment-and-evaluation.md#disabled-and-failed-capabilities). + +Cancellation is passed through parser, storage, and provider interfaces. Parsers +check it at file/declaration boundaries; generation failure recording deliberately +uses a non-cancelled token. CLI calls do not install a linked Ctrl+C cancellation +pipeline for all this work. Watch's delay also has no cancellation token, so do +not promise graceful transactional cleanup on process termination. + +## Parser-specific mechanisms + +### C#: semantic symbols plus source declarations + +[`CSharpParseService.ParseAsync`](../../src/CodeMesh.Parser.CSharp/CSharpParseService.cs) +first loads project contexts. `LoadProjectContextsAsync` tries an explicit +project, an explicit solution, a discovered top-level solution, and a unique +discovered project. If none loads, it emits `CMSHARP010` and falls back to +filesystem C# parsing with a constructed compilation. `TryLoadProjectAsync` +and `TryLoadSolutionAsync` record load/workspace diagnostics; a fallback warning +must not be mistaken for full MSBuild semantics. + +MSBuild loading respects project compile items and referenced compilations. +`CreateProjectParseContextsAsync` obtains syntax and compilation contexts; +`CreateWorkspace` registers MSBuild once under a lock. This makes restoring the +target's dependencies relevant to relationship fidelity. The fallback can index +source when project loading is unavailable, but cannot recreate all the target's +build configuration, references, or conditional compilation choices. + +Parsing then makes three passes across the project contexts: emit files, emit +declarations and register symbol lookup keys, and extract relationships. Waiting +until all declarations are registered is what permits a call in one loaded +project to resolve to a local symbol in another. `ExtractDeclarationNodes` +emits one logical node per symbol and a separate source declaration per syntax +location; `ExtractRelationships` emits containment and declaration-to-symbol +`Defines` links before semantic edges. + +`AddBodyReferenceRelationships` uses Roslyn symbol information for invocation and +constructor expressions and field/property/event access. Assignment context +distinguishes writes from reads, with compound access needing both. Type, +return, parameter, inheritance, implementation, and override relationships are +added when their target resolves to an indexed symbol. External or unresolved +targets are not evidence that no dependency exists. Calls through runtime +reflection or dynamic dispatch are not a complete runtime graph. + +Test attributes such as `Fact`, `Theory`, and `TestMethod` identify test cases; +the parser also recognizes CodeMesh's `Test...` methods in test files. This +classification affects retrieval ranking and summary candidate selection. +Representative tests in [Program.cs](../../tests/CodeMesh.Tests/Program.cs) are: + +- `TestCSharpParserHonorsProjectCompileItems` for project-aware source selection. +- `TestCSharpParserEmitsInvocationAndMemberAccessRelationships` for call/read/write + edges, and `TestCSharpParserEmitsCrossProjectInvocationRelationships` for pass + ordering across projects. +- `TestCSharpParserSeparatesPartialDeclarations` for logical/source separation. +- `TestCSharpParserEmitsTestFilesAndCases` for test recognition. + +### Python: indentation, definitions, imports, and bounded calls + +[`PythonParseService`](../../src/CodeMesh.Parser.Python/PythonParseService.cs) +is implemented in C#, without executing Python or using Python's AST runtime. +`ParseFile` emits whole-file content, gathers imports, then walks definition +headers while maintaining an indentation stack. `MatchDefinition` can combine +multiline headers using delimiter/header termination logic. `ExtractBlock` +selects a definition's content; classes become `Class`, functions become +`Method`, and test functions become `TestCase`. Async state, decorators, module, +qualified name, and parent are metadata. + +`ParseImports` retains module, imported-name, and alias metadata on declaration +nodes, including repeated guarded imports. `AddSemanticRelationships` then +matches calls against uniquely named definitions or imported aliases within the +file. A call through an imported alias can point to its import declaration +rather than a resolved foreign implementation. Despite the helper's name, this +is structural matching, not a Python type checker, complete cross-module +resolver, or dynamic-import analysis. Ambiguous names are left unresolved. + +Read `TestPythonParserEmitsClassesFunctionsAndContainment` and +`TestPythonParserEmitsImportsCallsAndTestCases` in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs). `TestPythonParserParsesAgentAccessProject` +uses this repository as an additional parser-shape check, not proof that every +Python construct is supported. + +### Rust: declaration scopes and PyO3 metadata + +[`RustParseService.CollectDeclarations`](../../src/CodeMesh.Parser.Rust/RustParseService.cs) +recognizes `mod`, `struct`, `enum`, `trait`, `impl`, `fn`, and `use` shapes using +regular expressions and accumulated attributes. `FindDeclarationEnd` and brace +tracking delimit blocks; enclosing declarations determine containment. `NodeKind` +maps Rust constructs onto the shared graph vocabulary, with Rust-specific +details retained as metadata rather than adding an enum for every syntax form. + +`AddSemanticRelationships` resolves unambiguous names within the collected file: +impl-to-trait `Implements`, impl/use references, calls to uniquely named +functions, and signature type uses. PyO3 attributes record exported names and +module/class/function roles for the composite parser. This does not run rustc, +expand macros, evaluate every `cfg` branch, or resolve a Cargo dependency graph. +`CMRS002` indicates a nonempty file with no recognized declarations. + +`TestRustParserEmitsDeclarationsRelationshipsTestsAndPyO3Exports` in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs) provides a representative +embedded Rust fixture, including tests and native export metadata. + +### Markdown: heading structure + +[`MarkdownParseService.ParseFile`](../../src/CodeMesh.Parser.Markdown/MarkdownParseService.cs) +creates a `Document` with whole-file content and `Section` nodes for recognized +hash-prefixed headings. It maintains a heading-level stack for containment, but +a section's own text ends at the next heading of any level. Descendant sections +remain separate graph nodes rather than being concatenated into the parent's +snippet. Metadata stores heading level, anchor, and document kind. + +`FindHeadings` is a line regex, not a complete Markdown parser. It does not track +fenced code blocks, and anchor generation does not add duplicate-heading +suffixes. Repeated titles can therefore collide in section identity. Links to +code, commands, or other documents are not modeled. These inspection findings +are recorded in the [debugging guide](../guides/development-and-debugging.md#inspection-findings). +`TestMarkdownParserEmitsDocumentsSectionsAndContainment` covers the supported +document/heading path. + +### Deployment: Dockerfile and Compose structure + +[`DeploymentParseService`](../../src/CodeMesh.Parser.Deployment/DeploymentParseService.cs) +selects recognized Dockerfile and Compose filenames. `ParseDockerfile` creates +a file plus stage nodes from `FROM` boundaries, preserving base image, alias, +and selected runtime instruction metadata. `ParseComposeFile` finds service +blocks, emits one service node per name, and extracts build/image, ports, +environment keys, volumes, and dependencies. `depends_on` becomes a +`References` relationship labelled `depends_on` between services in that file. + +The implementation is based on lines, indentation, and regular expressions, +not full Compose evaluation. It does not merge override files, resolve all YAML +features or substitutions, or prove a running deployment. It does not parse +Helm, Kubernetes, or CI/CD workflows merely because those files use YAML. +`TestDeploymentParserEmitsDockerfileAndComposeNodes` and +`TestDeploymentParserParsesRepositoryRuntimeArtifacts` in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs) show the expected shapes. + +The structural parsers catch file IO/access failures as warning diagnostics and +continue with other files. A completed run can therefore have partial file +coverage. Inspect diagnostics as well as counts. + +## Multi-language composition + +[`CompositeParserClient.ParseLanguages`](../../src/CodeMesh.Ingestion/CompositeParserClient.cs) +normalizes aliases, removes duplicates, and ordinal-sorts languages. +`ParseAsync` checks that the request selects exactly the configured language set, +then awaits all selected clients with `Task.WhenAll`. It combines nodes, +diagnostics, and content; content hashes are deduplicated. Duplicate node ids +fail; duplicate relationships can coalesce only when endpoint, kind, and label +agree. Any combined error diagnostic causes the orchestrator to abort before +publication. One language cannot publish independently inside this call. + +`AddPyO3Relationships` uses Rust `pythonExportName` metadata and Python static +import/call metadata. It adds references and invocations only when an export +name has exactly one candidate. The edges carry `boundary=pyo3` and source/target +languages. Name matching is a bounded cross-language lead, not proof of Python +module loading or a verified native ABI. + +`TestCompositeParserLinksPythonCallsToRustPyO3Exports` covers the boundary; +`TestMultiLanguageIngestionPublishesOneDeterministicSnapshot` covers canonical +language order and failure without publication. Both are in +[Program.cs](../../tests/CodeMesh.Tests/Program.cs). Use one combined ingest for +the desired language set: separate ingests select separate snapshots rather +than appending another language to checkout-current. + +## Refresh and watch + +[`CodeMeshRefreshWorkflow`](../../src/CodeMesh.Control/Configuration/CodeMeshRefreshWorkflow.cs) +finds a repository root, resolves the selected solution, and builds `ingest` +arguments. It preserves explicit language/filter/provider options; it does not +load a prior run's settings from MongoDB. Repeat the intended `--languages` and +path filters when starting a new refresh/watch command. + +[`CodeMeshWatchWorkflow`](../../src/CodeMesh.Control/Configuration/CodeMeshWatchWorkflow.cs) +enumerates eligible files and maps relative paths to `length:last-write-ticks`. +`HasChanges` compares the file count, paths, and those values. The CLI ingests +once immediately, then polls (default two seconds, minimum one) before each +subsequent full ingest. It processes one ingestion at a time. `--run-once` and +`--max-runs` bound the loop. + +This is polling, not a filesystem event subscription or content-hash watcher. +Preserving a file's length and timestamp can evade detection, and Git-only +metadata changes need an explicit refresh because `.git` is excluded. A thrown +ingestion exception leaves the watch command through its outer error handler; +there is no automatic retry/backoff loop. See the +[changed-source walkthrough](../guides/developer-walkthroughs.md#walkthrough-3-refresh-changed-source-and-handle-stale-access) +for the implications for snapshot selection and bound access. diff --git a/docs/current/project-status.md b/docs/current/project-status.md index 5a894e0..1023992 100644 --- a/docs/current/project-status.md +++ b/docs/current/project-status.md @@ -2,9 +2,16 @@ Document type: current status snapshot -Reviewed: 2026-08-13 +Reviewed: 2026-09-08 -Implementation reviewed through: `57d8d5c`. +Implementation status is reviewed against the current inspected worktree. +Current search-scope correction and provider-free retrieval evidence is bound +to candidate `4fb9e81`. Capped-runner and incomplete campaign evidence remains +bound to candidate `0f18071`. Earlier configured/onboarded task-adaptive evidence +remains bound to `fbc1433`; other configured agent evidence remains bound +separately to one-step one|nine candidate `8844c21`, Config.Net candidates +`b16eee7` and `3804028`, and YoutubeDownloader candidate `64d4052`. Earlier paid +campaigns remain bound to candidates `0bde606` and `0378740`. Retained evidence below remains bound to the exact commits named in each evidence section; it does not inherit this implementation baseline. @@ -15,6 +22,55 @@ set execution order; use tests or evaluation harnesses do not by themselves establish generalized agent benefit, release readiness, deployment acceptance, or production fitness. +## Implementation Update: 2026-09-10 + +The owner-activated development-session MCP feedback loop is implemented in the +current local source. An immutable, hash-pinned, expiring, separately revocable +session authorizes exact client and CodeMesh bindings. The +`development-feedback` profile adds bounded local v2 packet recording with +server-derived provenance; the `feedback-maintainer` profile adds allowlisted +list/get intake and deterministic non-authorizing resolution-plan hashes. +Installation-plan v2 pins the session path and hash while retaining v1 normal +plan loading. The existing normal four-tool and diagnostic surfaces remain +separate. + +The full 257-test Agent Access suite, including a real-stdio provider-free +integration test, passes locally. The integration records a failed client report, permits +maintainer intake and resolution planning without a CodeMesh edit, records a +passing superseding recheck, derives the original failure as resolved, rejects +calls after revocation, and retains both append-only packets. This proves the +implemented same-machine workflow and safety contracts only. It does not prove +that agents provide useful feedback, that a proposed fix is correct, that a +human approved an edit, or that CodeMesh improves agent outcomes. + +## Review Checkpoint: 2026-09-08 + +The review began on clean local `main` at `b2c053c`, 65 commits ahead of +`origin/main`. Only documentation and retained evidence changed between the +packaged source candidate `a158bc0` and that review baseline; application source, +tests, build files, and dependency locks are unchanged. Historical checks remain +bound to their recorded candidates and were not rerun as current-HEAD tests. + +| Reviewed area | Evidence checked in this review | Conclusion | +| --- | --- | --- | +| Recent implementation | Parser design-time output isolation, explicit HTML attribute escaping, exclusive feedback packet creation, and their regression coverage in source/tests | Recent corrections are present; no implementation drift requiring a contract or architecture update was identified. | +| Local distribution and runtime records | SHA-256 recheck of all four package artifacts, eight configured-runtime evidence files, and twelve dependency locks against the retained manifests | All 24 hashes match. Package/runtime execution remains the retained `a158bc0` local evidence, not a new execution or release acceptance. | +| Product progress | Configured campaign records and the review packet's outcome gate | One bounded Config.Net improvement coexists with neutral, regressed, and incomplete campaigns; repeatable configured-agent benefit remains unproven. | +| Public state | Read-only GitHub main, workflow, alert, release, and tag queries on 2026-09-08 | Remote main remains `8e9c0da`; hosted successes cover that older source. There are 39 open CodeQL alerts, zero open Dependabot/secret-scanning alerts, and no releases or tags. | +| Remaining work | Seven-gate [source review packet](../evaluation/review-packet-a158bc0.md) and [execution order](../planning/next-steps.md) | Local preparation is reviewable. Current-source hosted review, supported environment/resource and upgrade acceptance, repeatable product evidence, and owner release/publication decisions remain open. | + +This checkpoint refreshes source and evidence inspection only. It does not +activate another model campaign, change a pilot binding, start services, or +supply new application, dependency-advisory, or environment-acceptance results. + +Documentation verification passed Markdown lint, version consistency, +publication safety, workflow policy, and whitespace checks. The canonical +Markdown-link check failed on two missing `docs/experimental.md` targets inside +ignored downloaded CodeQL C# and Python query packs under `.codemesh-evals/`; +it reported no project-documentation link failures. Those retained packs were +left unchanged. Application suites and smoke were skipped for this +documentation-only review. + ## Product Position CodeMesh is a local-first context and likely change-impact service for coding @@ -32,23 +88,153 @@ usability evidence, but is no longer the v1 acceptance contract. | Area | Current status | Important boundary | | --- | --- | --- | | C# parsing | Implemented with Roslyn semantic relationships, declarations, test files, and test cases. | Strongest parser and current product wedge. | -| Python parsing | Files, classes, functions, async functions, and containment implemented. | Imports, calls, routes, tool handlers, and test links remain limited. | +| Python parsing | Files, classes, functions, async and multiline functions, imports, local/imported calls, test cases, and containment implemented. | This is a lightweight structural parser rather than a complete Python semantic model; dynamic imports and type resolution remain limited. | +| Rust parsing | Files, modules, structs, enums, traits, impl blocks, functions, methods, `use` declarations, test cases, call/type relationships, and PyO3 exports implemented in `CodeMesh.Parser.Rust`. | The parser is heuristic and source-structural rather than rustc-semantic; broader macro and conditional-compilation resolution remains open. | | Markdown parsing | Documents, sections, anchors, source spans, and containment implemented. | Cross-links to code, commands, endpoints, and projects are not modeled. | | Deployment parsing | Initial Dockerfile and Compose nodes, metadata, containment, and service dependencies implemented. | Helm, Kubernetes, CI/CD, overrides, and code-to-runtime links remain planned. | -| Ingestion | Dry-run, incremental writes, staged deterministic snapshots, compare-and-swap slot publication, reference-aware cleanup, refresh, watch, redaction, optional embeddings, and optional summaries implemented. | Watch is polling-based; provider-backed paths remain optional. | -| Storage | Snapshot-namespaced Neo4j graph, MongoDB content/registry/runs/summaries, Qdrant vectors, and in-memory test adapters implemented. | Publication is atomic at the registry slot boundary; normal service-backed operation requires configured stores. | -| Agent Access | REST, stdio MCP, Python CLI, typed .NET client, repository status, context search/packages, symbols, nodes, neighbors, and deletion implemented. The default four-tool normal MCP profile, expanded non-destructive diagnostic profile, profile-specific manifest, and canonical manual repository guidance are implemented. | Checkout binding, fail-closed package freshness, live runtime probe, and reviewable installer are not implemented. Repository deletion remains outside both MCP profiles. Returned relationships and rankings are navigation evidence, not runtime proof. | -| Context packaging | Agent-format packages, 12,000-character MCP cap, source spans, rationale, relationships, and validation recommendations implemented. | Source verification remains required for consequential conclusions. | +| Ingestion | Dry-run, incremental writes, staged deterministic snapshots, atomic multi-language parser composition, compare-and-swap slot publication, reference-aware cleanup, refresh, watch, redaction, optional embeddings, and optional summaries implemented. | One selected language set publishes one snapshot or fails before slot activation; watch remains polling-based. | +| Storage | Snapshot-namespaced Neo4j graph, MongoDB content/registry/runs/summaries, Qdrant vectors, and in-memory test adapters implemented. Neo4j graph publication uses bounded write batches. | Publication is atomic at the registry slot boundary rather than one cross-store transaction; normal service-backed operation requires configured stores. | +| Agent Access | REST, stdio MCP, Python CLI, typed .NET client, repository status, context search/packages, symbols, nodes, neighbors, and deletion implemented. The normal four-tool profile retains exact binding, fail-closed freshness, reviewed installation, provider-free probing, and package guidance. Explicit development-feedback and feedback-maintainer profiles add immutable session authority, server-derived v2 packet recording, allowlisted intake, and non-authorizing resolution hashes. V1 packets and installation plans remain readable. | Repository deletion remains outside every MCP profile. Feedback is local diagnostic input, not trusted truth, approval, outcome memory, or product-benefit evidence. Returned relationships and rankings remain navigation evidence, not runtime proof. | +| Context packaging | Agent-format packages, 12,000-character MCP cap, source spans, rationale, relationships, and .NET, Python, and Rust validation recommendations implemented. | Source verification remains required for consequential conclusions. | | Local UI | Read-only health, repository, run, freshness, and search views implemented. | It is a local inspection surface, not established product-benefit evidence. | -| Evaluation | Deterministic fixture, live MCP, paired agent, and model benchmark harnesses implemented. | Paid/model-backed runs require explicit authorization; fixtures prove contracts only. | -| Generated summaries | Complexity-aware prompts and local/online providers implemented. | No summary deployment profile is CodeMesh-qualified. | -| Security | Pre-persistence value redaction and generated/cache path exclusions implemented. | Known secret-file exclusion and configurable allow/deny policy remain open. | +| Evaluation | Deterministic fixture, live MCP, spontaneous, evaluator-assisted, configured/onboarded paired-agent, and model benchmark harnesses implemented. Agent campaigns require explicit input-plus-output-plus-reasoning reported-token accounting, a runner-declared hard-cap capability, remaining-budget limits on every invocation, boundary stopping, and no automatic retry. Configured mode also verifies the reviewed normal-profile plan, committed guidance, baseline MCP identities, fresh binding, positive runtime probe, and fail-closed rejection probe. | Configured outcomes include neutral one\|nine, regressed and improved Config.Net, and two duration-regressed YoutubeDownloader campaigns. None establishes repeatable benefit. Default `codex` capability preflight refuses because it cannot guarantee the hard cap. Explicit `capped-codex` failed closed in its first authorized campaign when provider credits were unavailable; no pair completed and the campaign is consumed. | +| Generated summaries | Complexity-aware prompts, local/online providers, and a provider-neutral qualification runner/report compiler are implemented. The runner uses the production prompt/parser, requires calibration and held-out qualification partitions, redacts inputs, records immutable suite/profile/source identity, creates typed blinded review packets, and binds isolated live-retrieval plus resource/cost evidence before applying quality, safety, retrieval, and deployment gates. | No summary deployment profile is CodeMesh-qualified. Provider-backed execution, a complete reviewed corpus, two blinded reviews, isolated indexes, query-level retrieval assessment, deployment measurements, and model/source/spend authority remain separate requirements. | +| Security | Pre-persistence value redaction, immutable generated/cache exclusions, default known-secret-file exclusion, repository-relative allow/deny globs, fail-fast pattern validation, and path-policy-bound snapshot identity implemented. | Explicitly including known secret files requires a separate CLI flag and does not weaken redaction or repository authorization requirements. | | Repository identity | Stable project/checkout markers, deterministic source snapshots, checkout-current and clean branch-head slots, explicit retained slots, legacy migration, and richer local freshness implemented. | Separate clones attach with explicit `--project-id`; shared-service tenant/authorization/privacy controls remain deferred. | Detailed current interfaces are in [Agent Access Contracts](agent-access-contracts.md), and component boundaries are in [Architecture](architecture.md). +## Activated one|nine Pilot + +The bounded pilot is active only for the clean Primary checkout at +`fe2f761583bf78601961ff17934185c4b6a632f9`. It is bound to project +`prj_a570a726917d4d0ca596085aaf07af50`, checkout +`chk_70c50c5494de4837ab0627aca021eab2`, and source view +`fea1951d1eeb93ac265112aef7000dcac1a9832b000ea81b360b0851df56e69b`. +Provider-free ingestion run +`98b439c4b0d484297852dae7723be555e75bf8fcf7c565c96a9b0bad43b66062` +published snapshot +`snp_bf026d3f2e2b5a40f3c9fa112ae36b7e026b0f5a0f0483a23796b0a6adc9243e` +with 45,650 nodes, 95,218 relationships, 12,873 content items, zero +embeddings, and zero summaries. It redacted 324 content items before +persistence. + +The reviewed plan at hash +`fcadac37ae4df261e8b746e14f9191e22a7b6399b991925d4d1e12a5947daf34` +changed only Primary's ignored `.codex/config.toml`. The normal-profile probe +accepted the exact fresh binding and the configured preflight preserved prompt +parity. Its installed ignored `AGENTS.override.md` guidance was retained by +source classification and hash, not raw content. The positive gate returned +both `app/feed_hub/native_replay.py` and +`native/feed_hub_replay/src/lib.rs`; the wrong-checkout gate failed closed with +the live binding, freshness, and provenance issue codes. + +An execution-readiness recheck on 2026-09-01 found the detached CodeMesh +candidate still clean at `0bde606` and confirmed that the reviewed plan, +installed configuration, installed guidance, and retained preflight still +matched their recorded hashes. Primary had advanced to `011f562` on `main`; its +branch tip was preserved while the checkout was temporarily restored to exact +`fe2f761`. The exact configured preflight passed again without repinning or +re-indexing, and Primary was returned to clean `main` at `011f562` after the +campaign. + +Clean detached CodeMesh candidate `0bde606` passed all eight frozen one|nine +provider-free gates, comprising four positive retrieval gates and four +rejection gates, and validated all three retained feedback packets. The same +candidate passed the pinned six-case, 18-call YoutubeDownloader summary-free +live gate with 1.0 tool success and recall, 0.9 MRR, 0.926186 nDCG, and zero +secret leaks. Its final local verification comprised 124 Python tests, Ruff +check and format, and all five deterministic MCP fixtures. After the missing +ASP.NET Core targeting pack and runtime were installed, attended deterministic +.NET verification at documentation commit `9c8aca3` also passed: locked +restore, build, 64 tests, and format verification completed with zero failures. +The build retained one `CS8625` nullable warning, and the YoutubeDownloader +sample test remained skipped because `CODEMESH_SAMPLE_CSHARP_ROOT` was not +configured. No solution, project, or C# source differed from evaluated +candidate `0bde606` at that verification commit. + +The attended correction path remains evidence: `4eccd80` missed the Rust +expected path, `a6ebaa5` expected the wrong live rejection diagnostic, and +`854d678` failed the YoutubeDownloader MRR threshold at 0.766667. No model was +invoked in those attempts or the final pass. The sanitized exact-candidate +record is [configured one|nine provider-free evidence](../evaluation/evidence/configured-onenine-provider-free-0bde606.json). + +The status-to-context correction is now frozen in clean detached candidate +`0378740`. At that exact commit, locked restore, build, 69 .NET tests, format +verification, 125 Python tests, Ruff check and format, all five deterministic +MCP fixtures, and all repository standards checks passed; one nullable build +warning and the unconfigured YoutubeDownloader sample skip remain. Against the +same exact Primary source identity, the new normal-profile configured preflight, +all eight frozen one|nine gates, all three feedback packets, and the pinned +YoutubeDownloader live gate passed with provider mode `none`. Primary's newer +`main` tip `0297190` was preserved and restored after the gates. The sanitized +record is [corrected candidate provider-free evidence](../evaluation/evidence/configured-onenine-provider-free-0378740.json). + +The subsequent one-step entry correction is frozen in clean detached candidate +`8844c21`. MCP instructions, tool descriptions, and generated onboarding now +make the fail-closed context package the first normal-host action while keeping +repository status as an explicit diagnostic and retaining its compatibility +next-action field. At that exact commit, locked restore, build, 69 .NET tests, +format verification, 125 Python tests, Ruff check and format, all five +deterministic MCP fixtures, and all repository standards checks passed. The +final service-backed .NET rerun passed after an earlier Neo4j cold-start +cancellation; the existing nullable warning and unconfigured YoutubeDownloader +sample skip remain. + +one|nine instance configuration commit `f9f199d` repinned only Primary to the +new candidate and regenerated all managed instance instructions successfully; +it did not activate another checkout. Against exact Primary source identity +`fe2f761`, the reviewed configured preflight, all eight frozen one|nine gates, +all three feedback packets, and the pinned YoutubeDownloader live gate passed +with provider mode `none`. Primary's newer `main` tip `162fa515` was preserved +and restored after the gates. The sanitized record is +[one-step candidate provider-free evidence](../evaluation/evidence/configured-onenine-provider-free-8844c21.json). + +The authorized `gpt-5.6-sol` medium-reasoning configured campaign then completed +three paired repetitions. All six executions were correct and safe, with no +failed MCP calls or command-policy violations. Two of three treatments called +`codemesh_get_repository_status`, but no treatment called +`codemesh_get_context_package`; the third treatment made no CodeMesh call. +The verdict was therefore `insufficient`, with zero treatment wins and zero +CodeMesh-attributable wins. On the two status-adopting pairs, treatment medians +were 2.22% slower and used 6.14% more tokens; overall medians were 30.29% slower +and used 46.61% more tokens. Those differences are diagnostic because no +context retrieval occurred. The sanitized record is +[configured one|nine agent evidence](../evaluation/evidence/configured-onenine-agent-0bde606.json). + +The separately authorized rerun through exact corrected candidate `0378740` +also completed all three pairs correctly and safely, with no failed MCP calls, +command-policy violations, or treatment regressions. One treatment called +`codemesh_get_repository_status`; no treatment called +`codemesh_get_context_package`. The verdict remained `insufficient`, with zero +treatment wins and zero CodeMesh-attributable wins. Overall treatment medians +were 6.07% slower and used 4.17% more tokens. The single status-adopting pair +was 44.22% slower and used 21.54% more tokens, but those differences remain +diagnostic because task context was never retrieved. The sanitized record is +[corrected configured campaign evidence](../evaluation/evidence/configured-onenine-agent-0378740.json). + +The separately authorized `gpt-5.6-sol` medium-reasoning campaign at exact +candidate `8844c21` completed all three configured pairs. All six executions +were correct and safe, with no failed MCP calls or command-policy violations. +Every treatment called `codemesh_get_context_package` directly, so the one-step +entry correction produced full measured adoption. The verdict was `neutral`: +there were no treatment wins, attributable wins, or matched regressions. +Median treatment duration was 1.39% lower and median tokens were 4.13% higher, +both inside the 10% neutrality band. One treatment used 110 shell commands +while the other two used 19; that outlier remains diagnostic. Primary was +restored to clean `main` at `162fa515` after the consumed campaign. The +sanitized record is +[one-step configured campaign evidence](../evaluation/evidence/configured-onenine-agent-8844c21.json). + +Together, the provider-free and model-backed records prove the bounded +configuration, runtime identity, safety, and adoption observed across the +campaigns. They do not establish attributable correctness or efficiency +benefit, repeatability, release readiness, deployment acceptance, provider +authorization, or production fitness. Secondary, Tertiary, Release, and Hotfix +are not activated; Run-only remains excluded. + ## Retained Product Evidence ### Summary-free retrieval @@ -105,56 +291,410 @@ assisted CodeMesh workflow or a broad product verdict. The sanitized transcription is [`docs/evaluation/evidence/config-net-prompt-parity-647d83d.json`](../evaluation/evidence/config-net-prompt-parity-647d83d.json). +### Configured replication readiness + +- Exact clean CodeMesh candidate `b16eee7` adds a configured/onboarded suite + while preserving the frozen Config.Net prompt, targets, thresholds, and + source commit. +- The reviewed local normal-profile installation and ignored repository + guidance bind exact Config.Net commit `ae0af7d`, project `prj_e5d8c8`, + checkout `chk_6d1d16`, and source view `b4981658`. +- The configured preflight passed with prompt parity, fresh accepted binding, + the four-tool normal profile, a positive context package, and fail-closed + rejection. The six-case live gate then passed all 18 calls with recall, MRR, + and nDCG of 1.0 and zero secret leaks. + +This is provider-free configured-integration readiness, not agent adoption or +product-benefit evidence. The exact sanitized record is +[configured Config.Net provider-free evidence](../evaluation/evidence/configured-config-net-provider-free-b16eee7.json). + +The separately authorized configured campaign then completed all three pairs +with context-package adoption in every treatment and no safety, MCP transport, +or command-policy failure. All three controls passed, while two treatments +missed required implementation or test citations; the treatment success rate +was therefore one of three and the verdict was `regressed`. On the only pair +where both conditions passed, treatment was 21.31% slower and used 117.18% +more tokens. This is valid negative configured-product evidence, not replicated +benefit. Preserve the consumed result and do not retry it automatically. The +sanitized record is +[configured Config.Net agent evidence](../evaluation/evidence/configured-config-net-agent-b16eee7.json). + +### Independent configured replication + +Exact clean candidate `64d4052` adds a configured/onboarded version of the +frozen YoutubeDownloader audio-container impact task without changing its +prompt, targets, thresholds, source commit, or model settings. Its 125 Python +tests, Ruff checks, five deterministic fixtures, reviewed normal-profile plan, +configured preflight, positive and wrong-checkout probes, two-case impact live +gate, and unchanged six-case live gate all passed. The impact gate covered all +six required targets with recall and MRR 1.0, nDCG 0.891323, and zero leaks; +the unchanged live gate retained recall 1.0, MRR 0.9, nDCG 0.926186, and zero +leaks. Exact identities and hashes are retained in the +[configured YoutubeDownloader provider-free evidence](../evaluation/evidence/configured-youtube-downloader-provider-free-64d4052.json). + +The separately authorized configured campaign completed all three pairs with +100% control and treatment correctness, context-package adoption in every +treatment, and no safety, MCP, or command-policy failure. Treatments attempted +the context package two, two, and three times. With no correctness win or +regression, treatment median duration was 49.79% higher and median tokens were +41.00% higher across all three matched pairs, so the valid verdict is +`regressed`. This replicates configured adoption and correctness on another +C# repository, not repeatable product benefit. Raw query arguments were not +retained, so focused-query guidance is a diagnosis lead rather than an +established cause. The sanitized result is +[configured YoutubeDownloader campaign evidence](../evaluation/evidence/configured-youtube-downloader-agent-64d4052.json). + +### Task-adaptive query correction + +The provider-free YoutubeDownloader diagnosis preserved its six impact targets, +source commit, repository identity, and retrieval thresholds. One frozen +task-shaped package covered all six targets in one call with primary rank two. +Three proactive focused packages also covered all six, but produced 35,991 +agent-formatted characters instead of 12,000 and repeated five of 24 ranked +item occurrences. The diagnostic comparison is retained separately because its +CodeMesh worktree contained the new suite and guidance edits: +[task-adaptive query diagnosis](../evaluation/evidence/youtube-downloader-task-adaptive-query-diagnosis-e2e1bca.json). + +Exact clean detached candidate `fbc1433` contains the smallest correction. +Agents now start with one concise task-shaped package, stop when it resolves +every required facet, and issue a focused follow-up only for a specific +unresolved implementation or test facet. The guidance caps each task at three +non-overlapping packages. At that commit, 125 Python tests, Ruff checks, five +deterministic MCP fixtures, the reviewed normal-profile preflight, positive +probe, and wrong-checkout rejection all passed. The adaptive five-case gate +passed all 15 calls with recall 1.0, MRR 0.875, nDCG 0.862253, and zero leaks. +The unchanged impact gate retained recall and MRR 1.0 and nDCG 0.891323; the +unchanged canonical gate retained recall 1.0, MRR 0.9, and nDCG 0.926186. +No model provider was invoked. Exact hashes and identities are in the +[task-adaptive provider-free evidence](../evaluation/evidence/youtube-downloader-task-adaptive-provider-free-fbc1433.json). + +The separately authorized `fbc1433` configured campaign then completed all +three pairs with 100% control and treatment correctness, one successful context +package in every treatment, and no safety, MCP, or command-policy failure. It +had no correctness win or regression. Median treatment tokens were 6.63% lower, +but median duration was 10.0192% higher, crossing the frozen 10% threshold by +0.0192 percentage points and producing a `regressed` verdict. The campaign also +reported 2,596,056 aggregate tokens, including 2,219,264 cached-input tokens, +and therefore exceeded its authorized 1.6M reported-token ceiling. The +evaluator at that candidate had no hard aggregate-token stop; this is a control +failure, not a reason to discard or retry the consumed result. Raw traces and +query arguments were not retained. The sanitized record is +[task-adaptive configured campaign evidence](../evaluation/evidence/configured-youtube-downloader-agent-fbc1433.json). + ## Current Gap -Explicit repository onboarding is now the selected v1 product contract. -CodeMesh publishes MCP initialization instructions and documents a canonical -manual repository guidance block. Its default normal MCP profile now exposes -only context package, repository status, repository listing, and node lookup; -the diagnostic profile adds non-destructive inspection tools. The selected -profile is recorded in the static manifest and unknown profiles fail before -startup. +The next search diagnosis now has request-isolated operation timing in the +source. It separates source-store fetches, candidate selection, context +expansion, declaration and content reads, optional relationships, and final +ranking, while retaining the existing package-stage timing and unchanged +responses. Injected-delay tests cover attribution, result parity, overlapping +requests, and failure cleanup. This instrumentation does not establish a new +agent-benefit result. + +Clean instrumentation candidate `2879e7f` passed the unchanged adaptive live +suite in a fresh isolated store with recall 1.0, MRR 0.875, nDCG 0.862253, +and zero leaks. The task-shaped package p50 was 451.070 ms. Adding 50,000 +synthetic unrelated graph nodes left retrieval unchanged but increased package +p50 to 1,504.467 ms, including 1,227.914 ms in lexical fetch. The query plan +loaded lexical properties for all 51,685 nodes before repository filtering. +An isolated query experiment moved that filter before property projection, +loaded properties for only the 1,685 target nodes, and preserved the exact +ordered candidate hash. Clean correction candidate `4fb9e81` passed the frozen +adaptive, impact, and canonical live suites with unchanged quality and zero +leaks. Under the same synthetic store load, task-shaped package p50 fell to +391.808 ms (73.96% lower) and lexical fetch p50 to 124.113 ms (89.89% lower). +All adaptive cases retained identical ordered candidates. The +[correction evidence](../evaluation/evidence/search-scope-correction-4fb9e81.json) +records exact candidates, checks, hashes, and claim limits. This controlled mechanism +does not establish the cause of the historical 16.7-second result or an agent +benefit. The [search-scope diagnosis](../evaluation/evidence/search-scope-diagnosis-2879e7f.json) +retains timings, plans, fixture identity, report hashes, and limitations. -Unambiguous checkout binding, fail-closed freshness before returning trusted -packages, the live runtime probe, and reviewable installation flow remain open. -Current repository status reports can detect stale or dirty local checkouts, but -that advisory behavior does not yet enforce the selected host checkout as a -package-return acceptance condition. +Explicit repository onboarding is implemented for the bounded one|nine pilot: +the normal profile exposes only context package, repository status, repository +listing, and node lookup; exact binding and freshness are enforced; and +installation plus probing are reviewable. Clean one-step candidate `8844c21`, +exact Primary commit `fe2f761`, the configured preflight, the eight-gate frozen +one|nine pilot, and the YoutubeDownloader live gate all pass provider-free. The +configured normal-host campaigns at historical candidate `0bde606` and +corrected candidate `0378740` were both valid but insufficient. Across them, +three treatments called repository status and none retrieved task context. The +subsequent `8844c21` campaign was valid and neutral: all three treatments +retrieved a context package, all six executions passed safely, and there were +no attributable wins or regressions. The prompt-parity Config.Net campaign recorded zero MCP calls and zero MCP call attempts. Preserve it as negative spontaneous-discovery evidence; do not change -the frozen task prompt, grading, or targets to manufacture adoption. The next -product-proof campaign should instead evaluate the exact configured/onboarded -integration shipped to normal users, while retaining the same task prompt for -control and treatment. - -The evaluated required MCP configuration completed without a harness failure, -and the CodeMesh manifest publishes the intended instructions and tool -contracts. Current Codex documentation says the host reads MCP server -instructions, but its prompt-input diagnostic does not expose the separate base -instructions and tool schemas. The retained campaign therefore cannot attest -the exact selection context presented to the model, and no CodeMesh transport -or contract defect has been confirmed. - -Implement the selected contract in the order defined by -[Agent Integration Contract](../planning/agent-integration-contract.md). After -material integration or retrieval changes, repeat the summary-free live gate. -The configured/onboarded agent campaign remains model-cost-bearing and requires -explicit authorization, clean committed checkouts, a fresh matching index, and -report metadata that identifies the shipped guidance and tool profile. Unit and -contract tests for instructions do not substitute for normal-host evidence. +the frozen task prompt, grading, or targets to manufacture adoption. The +first two configured one|nine campaigns evaluated the status-first onboarding +path and showed that the status-result action did not produce context adoption. +The third campaign preserved the same task and comparison conditions and showed +that direct context-package entry did produce adoption in every treatment. + +The evaluated MCP configuration completed without a harness failure, and the +CodeMesh manifest published the intended instructions and tool contracts. The +measured transition failure was narrower: a fresh accepted status response did +not itself state the next tool action. The current implementation adds an MCP-only +`codemesh_next_action` status field that directs a fresh accepted binding to +continue immediately to an agent-format context package and directs every +other state to fall back. This correction is committed, covered by +deterministic contract tests, and verified through the full provider-free +exact-candidate gates at `0378740`. Its paid campaign still observed one +status-only treatment and no context-package call, so the action field did not +produce the required normal-host handoff. + +The current implementation removes that two-step dependency. MCP server +instructions, context-package and status tool descriptions, and generated +repository onboarding now use the fail-closed context package as the single +normal-host entry action. Repository status remains available for explicit +ingestion, count, and freshness diagnostics, and its compatibility next-action +field remains intact. Exact clean candidate `8844c21` passed the full +deterministic suite, configured preflight, eight one|nine pilot gates, feedback +validation, and pinned YoutubeDownloader live gate with no model provider. The +authorized paired campaign then observed context-package adoption in every +treatment and a neutral outcome. This proves the measured handoff for the +selected task; it does not prove attributable product benefit or +cross-repository repeatability. + +The configured/onboarded evaluator now preserves prompt parity, uses the exact +reviewed checkout-local `normal` installation, and retains integration mode, +guidance, baseline MCP, tool-profile, binding, installation-plan, and runtime- +probe identities. It rejects a dirty or stale target, a dirty CodeMesh source +checkout, commit or binding mismatch, changed plan files, unexpected tool +surface, and a failed rejection probe. Historical spontaneous and assisted +modes retain their prior diagnostic behavior. + +The provider-free task-shaped context-package coverage diagnosis is now +complete at exact baseline candidate `b16eee7`. The frozen 58-term +task query returned eight test paths and only one of seven required targets. +Four concise task-derived facets covered all seven targets in a separate +five-case/15-call provider-free suite with recall 1.0, MRR 0.660714, nDCG +0.76259, and zero secret leaks. The unchanged canonical live suite also passed +all 18 calls with recall, MRR, and nDCG of 1.0. This isolates a bounded guidance +and query-decomposition gap without proving which query the paid treatments +used. Exact details are retained in the +[focused-query diagnosis](../evaluation/evidence/config-net-focused-query-diagnosis-b16eee7.json). + +Exact clean detached candidate `3804028` contains the smallest correction: the +context package remains the single normal-host entry tool, while MCP +instructions, tool and status guidance, and generated onboarding direct +multi-part work to a small set of focused implementation and test queries +instead of one long full-task query. All 125 Python tests, Ruff checks, and five +deterministic MCP fixtures pass at that candidate. + +The reviewed configuration-only Config.Net plan, separately managed ignored +guidance, configured preflight, positive probe, and wrong-checkout rejection +all passed with provider mode `none`. The committed focused suite then passed +all five cases and 15 calls with recall 1.0, MRR 0.660714, nDCG 0.76259, and +zero secret leaks; `LazyVar.cs` remains at the accepted rank-eight boundary. +The unchanged canonical suite also passed all six cases and 18 calls with +recall, MRR, and nDCG of 1.0 and zero secret leaks. This establishes an exact +provider-free correction candidate, not normal-agent compliance or product +benefit. Exact identities and report hashes are retained in the +[focused-query provider-free evidence](../evaluation/evidence/config-net-focused-query-provider-free-3804028.json). +The separately authorized exact-candidate campaign then completed all three +pairs and received an `improved` verdict. Every treatment used the context +package, made two or three package attempts, passed the citation grader, and +completed without safety, MCP, or command-policy failure. Controls passed two +of three; the second control missed `ConfigurationBuilder.cs`, producing one +CodeMesh-attributable win and no treatment regressions. Across the two jointly +successful pairs, treatment median duration was 13.69% lower and median tokens +were 3.59% lower. This is positive configured evidence for one task, repository, +model, and three-pair campaign; it does not establish that the wording change +caused the result or that product benefit is repeatable. The sanitized record is +[focused-query configured campaign evidence](../evaluation/evidence/configured-config-net-agent-3804028.json). +Preserve all one|nine and Config.Net campaigns in their original evidence +classes. This retest is consumed; no further model spend is authorized. + +Both configured YoutubeDownloader campaigns are consumed. Candidate `64d4052` +preserved full adoption and correctness but regressed duration by 49.79% and +tokens by 41.00%. Candidate `fbc1433` reduced each treatment to one context +package and lowered median tokens by 6.63%, but its 10.0192% duration regression +still failed the frozen threshold. It also exceeded its authorized aggregate +reported-token ceiling because that historical evaluator did not enforce a hard +stop. The current evaluator closes that control gap with explicit +reported-token accounting, a mandatory compatible runner capability, +remaining-budget limits, boundary stopping, and no automatic retry. The +default Codex runner does not claim that capability and is refused before any +model call. The explicit `capped-codex` runner now counts complete request input, +reserves worst-case output-plus-reasoning charge, enforces zero provider +request/stream retries, retains ambiguous reservations, and reconciles its +sanitized ledger against Codex final usage. These records establish configured +adoption and correct/safe completion, not repeatable benefit or efficiency +causality. Neither consumed campaign may be retried. + +Provider-free attribution at exact instrumentation candidate `66033ad` measured +the task-shaped package at 16,830.931 ms total p50: search accounted for +16,725.370 ms (99.37%) and item hydration for 102.668 ms (0.61%). Hydration did +not qualify as dominant, so the proposed concurrency optimization was rejected. +Any fresh paid evaluation still requires exact-candidate deterministic, +configured, retrieval, safety, and capped-runner gates plus explicit spend +authority. + +Exact clean candidate `0f18071` passed 149 Python tests, Ruff check and format, +all five deterministic MCP fixtures, repository standards, the configured +positive and fail-closed rejection preflight, and the frozen adaptive, impact, +and canonical provider-free suites. A synthetic provider-free run through the +installed Codex executable completed one counted request with zero retries and +exact proxy/final-usage agreement. The one separately authorized +`gpt-5.6-sol` medium-reasoning campaign then stopped after its first control +execution because the provider reported no available credits. No pair +completed. The proxy retained the interrupted call's full 267,528-token +reservation, admitted no later request, and recorded zero request or stream +retries. The campaign is `insufficient`, incomplete, and consumed; it is hard-cap +failure evidence only, not agent adoption, correctness, efficiency, or spend +evidence. The sanitized record is +[capped YoutubeDownloader campaign evidence](../evaluation/evidence/configured-youtube-downloader-agent-0f18071.json). + +Expand onboarding to another one|nine checkout only with separate authority, +exact binding, fresh ingestion, and a passing checkout-specific probe. The +earlier `de239b7` stale state and `9f3216c` stale-candidate rejection remain +useful historical diagnostics; they are not the current candidate state. + +## Public Repository And Hosted Evidence + +At the 2026-09-08 read-only GitHub recheck, the public `main` branch remains at +`8e9c0da`, while the local configured-evaluator candidate and its documentation remain unpublished. +The latest hosted baseline-verification success applies to remote `8e9c0da`; +there is no exact-HEAD hosted workflow evidence for the unpublished local +commits. + +The latest hosted CodeQL workflow completed successfully for remote `8e9c0da`, +which proves that the scan ran, not that the repository is security-clean. +GitHub still reports 39 open CodeQL alerts: 3 critical command-line-injection +alerts and 36 high alerts, comprising 35 path-injection findings and one +reflected-XSS finding. Dependabot and secret scanning report zero open alerts. +These hosted findings apply to the published remote state and do not establish +the security state of the unpublished local commits. + +The original [CodeQL triage](../evaluation/evidence/codeql-triage-8e9c0da.json) +is preserved. Local security hardening now validates evaluation suite names and +identifiers, contains setup patches and generated artifact paths, and rejects +invalid suites before preparation or model execution. Literal argv and all five +UI routes have adversarial regression coverage. The full Python suite passes +227 tests with two existing dependency warnings. The +[local boundary review](../evaluation/evidence/security-boundary-review-20260905.json) +accounts for all 39 findings, including explicit operator-selected paths and +trusted executable inputs. This is local verification, without an updated +hosted scan or alert dismissal. + +The subsequent UI correction explicitly HTML-escapes dynamic link attributes +and the numeric search limit at their render points. All 231 Python tests pass, +including hostile attribute values and invalid or out-of-range limit rejection +before store access; Ruff lint and formatting pass. Two existing dependency +warnings remain. This verification does not dismiss the hosted XSS finding. + +The local-input security review then reproduced overwrites of existing packet +destinations, including symlinks and hardlinks. Feedback recording now creates +packet files exclusively and explicitly rejects linked outboxes before the +existing Git ignore check (which already rejected Linux outbox symlinks). The +three overwrite regressions fail before correction and pass afterward; an +additional test checks explicit outbox rejection. The full +Python suite now passes 235 tests with two existing dependency warnings. + +The [local CodeQL review](../evaluation/evidence/local-codeql-review-b73e19a.json) +now binds complete extraction and pinned suites to exact local source. Python +`b73e19a` has zero default-threat findings and 61 local-input findings; unchanged +C# source at `e0beba9` has zero default findings and 192 local-input findings. +All 253 local-input results have retained locations and boundary review. The +HTML finding is absent after explicit attribute escaping. Cached first attempts +were rejected for local-threat claims and rerun, with original results retained. +This is local static-analysis evidence, without hosted dismissal or a blanket +security-clean claim. + +## Local Release Preparation + +The source version is now declared in `VERSION` and consumed by .NET builds. +Python package metadata, lockfile, and version export are checked for agreement; +REST OpenAPI uses that export. Both CLIs expose a provider-free version command. +The retained source value is `0.1.0`, preserving the existing Python value and +creating no published release. The full local verification passes 70 .NET tests +with no skips, 228 Python tests, and five deterministic MCP cases. The .NET +build retains its existing nullable warning and Python retains two dependency +warnings. The retained 2026-09-05 .NET and locked Python dependency advisory +checks report no known vulnerabilities for CodeMesh's own dependency sets; this does not clear +the frozen external sample's separate advisory or the hosted CodeQL findings. + +[Release Preparation](../engineering/release-preparation.md) defines a reviewable +candidate contract, compatibility policy, environment evidence, verification, +recovery, and publication gates. Local [package verification](../evaluation/evidence/local-package-verification-96b4ebf.json) +now binds Python wheel/source artifacts, a .NET publish archive, and their +checksums to clean `96b4ebf`. Installed-wheel CLI, REST OpenAPI, normal manifest, +bundled suite/patch loading, and all five fixture cases pass. Full smoke passes +at that candidate. Parser packaging correction `1ef5ca9` adds the required SDK +to its runtime and the version files to the Docker allow-list; its project +probe loads through MSBuild without filesystem fallback, while retaining a +read-only generated-editor-config warning at that historical candidate. +The subsequent C# capability `0.2.3` output-isolation correction now passes two +concurrent read-only container parses without warnings, preserves generated +compiler metadata and cross-project relationships, and cleans temporary output +after completion and observed project failures. Its deterministic verification +passes 71 .NET tests with no skips and 228 Python tests. See the +[output-isolation decision](../decisions/0005-isolate-parser-design-time-outputs.md). +Hosted security review, Windows and +other environment acceptance, a supported upgrade/rollback rehearsal, +appropriate product evidence, and owner release approval remain separate gates. +A [disposable Linux recovery rehearsal](../evaluation/evidence/local-recovery-rehearsal-f314043.json) +now passes at `f314043`: the complete stopped four-volume set restored into new +volumes with identical image IDs and matching graph, MongoDB, vector, registry, +snapshot, and context-item digests. This closes the local same-version recovery +mechanism check; it does not establish a supported upgrade or production recovery. + +A [development-baseline upgrade and rollback rehearsal](../evaluation/evidence/local-development-upgrade-ce23f2e.json) +now verifies isolated `8e9c0da` to `ce23f2e` reader compatibility, re-ingestion, +identity preservation, eight source citations, scoped deletion, and exact +whole-set rollback. The initial six-case retrieval gate failed its relationship +check without sample dependency assets. Locked sample restore failed on the +frozen AngleSharp advisory; the assets it produced enabled a subsequent +unchanged six-case/18-call gate to pass, with advisory warnings retained. The +local mechanism passes, while sample dependency and supported-environment +acceptance remain open. All task containers are down and backups are preserved. + +The [Linux Python runtime matrix](../evaluation/evidence/python-runtime-verification-b73e19a.json) +now passes all 235 tests and five deterministic fixture cases on Python 3.12.14, +3.13.15, and 3.14.7 at `b73e19a`. Each retains two existing dependency warnings. +This extends local source verification; Windows and released-artifact acceptance +remain separate. + +[Refreshed local artifacts](../evaluation/evidence/local-package-verification-a158bc0.json) +now bind the completed parser, UI, and feedback corrections to clean `a158bc0`. +The installed wheel passes version/OpenAPI checks, the four-tool normal manifest, +all 16 packaged suites and patches, and five fixture cases. The .NET publish +contains the design-time targets and reports the exact build SHA. Full smoke and +two concurrent read-only parser-container parses pass; the parser emits no +warnings and leaves no scratch directories. The retained 2026-09-05 dependency +advisory checks report no known vulnerabilities in CodeMesh's 11 .NET projects and 48 Python +packages. All task containers are down. These are local artifacts and checks, +with the separate sample advisory and release/acceptance gates still open. + +The [source review packet](../evaluation/review-packet-a158bc0.md) now supplies +draft release notes and a seven-gate evidence ledger. +Artifact bytes and all 12 dependency-lock identities were reverified on +2026-09-08. The packet records the 2026-09-05 requirement for a PR and `verify` +for main; no release or remote tag exists at the 2026-09-08 recheck. +Owner review, publication authority, supported profiles, and representative +resource acceptance remain open. + +The [fresh configured runtime probe](../evaluation/evidence/configured-runtime-verification-a158bc0.json) +now verifies normal-profile installation and retrieval, rejection of incorrect +source hashes, unknown checkouts, and dirty source, and acceptance after source +restoration. The installed wheel drove +the unchanged source-backed launch; no model campaign was run. ## Current Boundaries - Generated summaries remain disabled in the core product-proof baseline. - No summary model is qualified. -- Python, Markdown, and deployment parsers do not match Roslyn's semantic depth. +- Python, Rust, Markdown, and deployment parsers do not match Roslyn's semantic depth. +- The activated one|nine pilot is pinned to one Primary snapshot and must not be + silently expanded or repointed when either checkout advances. - Shared multi-user stores, portable snapshot artifacts, outcome memory, Kubernetes/Helm ingestion, and broader SDLC intelligence are not implemented. -- Current source redaction does not replace repository authorization or a - configurable sensitive-file policy. -- The historical publication review is not evidence for commits after its - recorded snapshot. +- Current source redaction and path filtering do not replace repository + authorization or review of an explicitly broadened ingestion scope. +- Hosted workflow success for remote `8e9c0da` is not evidence for unpublished + local commits, and an automated scan completing successfully does not resolve + or justify its open findings. ## Status Maintenance diff --git a/docs/current/retrieval-internals.md b/docs/current/retrieval-internals.md new file mode 100644 index 0000000..0223d19 --- /dev/null +++ b/docs/current/retrieval-internals.md @@ -0,0 +1,287 @@ +# Retrieval and Agent Access internals + +Document type: current implementation reference + +The central read facade is +[`CodeMeshReadStore`](../../agent-access/codemesh_agent_access/store.py). +It owns graph, content/registry, summary, query-embedding, and vector adapters. +Its methods are shared by the REST, MCP, and Python CLI surfaces. Exact request +and response fields remain in +[Agent Access Contracts](agent-access-contracts.md). + +## Entry points and shared behavior + +| Surface | Entry and call chain | Distinct behavior | +| --- | --- | --- | +| REST | [rest.py](../../agent-access/codemesh_agent_access/rest.py) → Pydantic request model → `CodeMeshReadStore`. | Typed HTTP JSON, FastAPI validation, unbound administrative selection; no automatic normal-MCP binding. | +| MCP | [mcp.py](../../agent-access/codemesh_agent_access/mcp.py) → [tools.py](../../agent-access/codemesh_agent_access/tools.py) → request model → read facade. | Profile allowlist, binding injection, `ToolResult` JSON envelope or text-only agent brief. | +| Python CLI | [cli.py](../../agent-access/codemesh_agent_access/cli.py) → tool/facade functions. | Local process reading stores directly; CLI formatting and explicit output budgets. | +| .NET client | [AgentAccessClient](../../src/CodeMesh.Control/AgentAccess/AgentAccessClient.cs) → REST HTTP endpoints. | Typed C# records, explicit JSON property names, URL-escaped selectors, owned or injected `HttpClient`. | +| Local UI | [web.py](../../agent-access/codemesh_agent_access/web.py) → shared read facade. | Server-generated HTML at `/ui`; read-only health, repository, run, and search views. | + +The stdio MCP runtime requires a complete binding for the normal profile. +`build_server` can also be used directly by tests; the runtime requirement is +enforced by `mcp.run`. Normal exposes four tools: context package, repository +status, repository listing, and node lookup. Diagnostic adds inspection tools; +neither profile contains repository deletion. REST and the Python administrative +CLI retain deletion as a separately invoked operation. + +[models.py](../../agent-access/codemesh_agent_access/models.py) defines request +types and defaults. These are ordinary typed Pydantic fields, not universally +strict bounded integers: several limits are clamped in downstream functions +rather than rejected at model construction. Do not assume all surfaces return +the same error for every malformed value. `test_rest_contract.py`, +`test_mcp_contract.py`, and `test_cli_output.py` under +[tests](../../agent-access/tests/) protect surface-specific mapping. + +## Repository and snapshot resolution + +An unbound query may supply a project id, alias, explicit snapshot id, or slot +key. `_resolve_repository_id` first reads a compatibility repository summary +and uses its selected snapshot metadata. If that fails to identify a repository, +`MongoContentStore.resolve_snapshot_id` tries an explicit snapshot record, then +matching slot keys. Ambiguous aliases and slot matches raise errors. Certain +Mongo errors return the original selector or no match, so an empty result can +also be a store availability symptom. + +The bound path is intentionally more specific. +[`RepositoryBinding`](../../agent-access/codemesh_agent_access/binding.py) +requires project id, checkout id, and an absolute root; it optionally includes +source-view hash. `bound_repository_reference` permits omitted selection or the +configured project/checkout id and rejects conflicting overrides. It does not +accept arbitrary aliases as equivalent to a bound identity. + +[`MongoContentStore.get_bound_repository`](../../agent-access/codemesh_agent_access/content_store.py) +reads the registered checkout, verifies its project, selects exactly its +`CheckoutCurrent` slot from `snapshot_projects`, reads that snapshot, and finds +the latest observation for that project/checkout/snapshot. `bound_repository_document` +builds a summary from these records rather than trusting the compatibility +summary's default branch selection. Missing or ambiguous state cannot produce +an accepted bound package. + +```mermaid +sequenceDiagram + participant H as Agent host + participant M as Bound MCP tool + participant R as Read facade + participant DB as Registry + participant G as Local Git + participant S as Search and hydration + H->>M: get_context_package(query) + M->>M: Check caller selector and inject binding + M->>R: get_context_package(query, binding) + R->>DB: Resolve exact checkout-current and observation + R->>G: Inspect commit and working-tree state + alt Binding rejected + R-->>H: Tool error before search + else Binding accepted + R->>S: Search selected snapshot + S-->>R: Ranked source hits + R->>S: Hydrate content, summaries, relationships + R-->>M: Package with citations and rationale + M-->>H: JSON or bounded agent text + end +``` + +`test_bound_repository_document_uses_exact_checkout_slot`, +`test_binding_assessment_requires_exact_identity_root_and_freshness`, and +`test_context_package_rejects_unaccepted_binding_before_search` in +[test_store_components.py](../../agent-access/tests/test_store_components.py) +protect the critical order. The latter asserts that rejection happens before +the search path is used. + +## What freshness actually checks + +[`_repository_freshness`](../../agent-access/codemesh_agent_access/store.py) +captures local Git state in a worker thread. It compares indexed and current +commits, selected versus observed snapshot ids, and indexed versus current dirty +state. Any current or indexed dirty view is unverifiable for normal access and +therefore stale. If Git/root/commit provenance is unavailable, freshness is +unknown rather than accepted. + +`assess_binding` additionally checks project, checkout, normalized root, optional +configured source-view hash, fresh status, `is_stale is False`, and known +provenance. `require_accepted_binding` raises `RepositoryBindingError` for a +rejected assessment. Node reads through a bound MCP server apply the same +freshness gate; administrative unbound reads do not inherit that gate. + +This does not reparse source or recompute `SnapshotIdentity.SourceViewHash` on +every request. The source-view comparison uses recorded identities and Git +observations. Ignored source changes and edits concurrent with the read are not +covered by a full filesystem read transaction. Freshness acceptance is the +implemented local check at that moment, not a perpetual guarantee about bytes +an agent reads later. See [inspection findings](../guides/development-and-debugging.md#inspection-findings). + +`get_repository_status` also returns stored counts, the latest available run, +summary coverage, and `_refresh_status`'s interpretation of that run. It does +not continuously observe the running .NET process. In particular, the +orchestrator writes its completed run record after successful work; parser +exceptions can occur without a new failed run appearing in this API. + +## Search candidate acquisition + +`search_context` resolves scope first, then optionally embeds the text query if +an explicit vector was not supplied and the query provider is enabled. It +starts vector, lexical, and summary tasks concurrently when their inputs exist. +That is concurrency across search sources, not parallel hydration of every +candidate. Model-free mode skips query embedding, but still allows lexical +search and lookup of previously stored summaries. + +The graph adapter's +[`search_context_nodes`](../../agent-access/codemesh_agent_access/graph_store.py) +uses query terms against stored symbol/path/metadata fields. This lexical search +is not full-text search over Mongo source bodies. `_search_lexical_context` +requests up to `min(max(limit * 16, 64), 500)` raw nodes, applies exact filters, +scores them, and diversifies to `min(max(limit * 4, 16), 200)` candidates before +constructing context hits. This bounded pool means a relevant node outside the +acquired candidates cannot be rescued by changing the final sort alone. + +Vector search uses +[`QdrantSearchClient`](../../agent-access/codemesh_agent_access/vector_store.py) +and exact payload filters, then looks up the corresponding graph nodes and +builds expanded hits. Summary search uses +[`MongoNodeSummaryStore`](../../agent-access/codemesh_agent_access/summary_store.py) +to find completed generated summaries and turn their relevance into candidates. +`_candidate_limit` bounds those retrieval pools. Diagnostics distinguish raw +candidate counts from counts contributing to the final returned hits. + +## Ranking, expansion, and file coverage + +`_lexical_score` compares the full normalized query to id, stable key, name, +path, language, signature/display, and native export/attribute metadata. Exact, +prefix, and substring matches take explicit early scores. Otherwise +`lexical_terms` removes common grammatical noise and splits language-specific +identifiers, and scoring combines overall term coverage, best-field coverage, +symbol/filename coverage, and semantic-metadata coverage. Small inflection +variants help queries such as a plural noun match singular identifiers. + +These are hand-written relevance heuristics, not calibrated probabilities. +`_adjust_score` multiplies raw relevance by a node-kind weight, adds an expansion +bonus, and clamps to `[0, 1]`. `_ranked_unique_hits` retains the best hit per +`node_id` and merges source score components; it does not sum all source scores +into one universal relevance estimate. It sorts by adjusted score, highest raw +source relevance, kind weight, and earlier source line. The raw tie-break is +important when adjusted scores saturate at 1. + +Low-signal nodes such as parameters can expand to useful containing context. +`_select_context_node` obtains graph candidates, `_choose_context_candidate` +prefers relevant kinds with a distance penalty, and `_build_context_hit` +attaches a primary source declaration. A hit can consequently retain a logical +method id while taking the path/span/content from its declaration. Its +`expanded_context` explains the source, selected node, declaration, and reason. +Without that distinction, a debugger may wrongly treat a method's zero span as +missing source. + +After deduplication, `_diversify_ranked_hits` groups hits by normalized file path +and takes one item per file per round. This can put a lower-scoring hit from a +new file ahead of the second hit in a high-scoring file. The behavior applies to +context search as well as packages; symbol search is a separate path with its +own scoring and optional declaration attachment. + +Representative evidence in +[test_context_ranking.py](../../agent-access/tests/test_context_ranking.py): + +- `test_vector_search_expands_parameter_declaration_to_method_context` and + `test_vector_search_can_disable_context_expansion` isolate expansion. +- `test_lexical_score_ranks_multi_term_coverage` and + `test_lexical_score_uses_language_signature_and_inflection_for_native_boundary` + protect task and native-boundary matching. +- `test_ranked_hits_break_saturated_scores_with_raw_relevance` protects the + saturated-score tie-break. +- `test_ranked_hits_cover_distinct_files_before_repeating` and + `test_lexical_candidates_cover_distinct_files_before_repeating` protect both + stages of file diversity. + +Unscoped searches deduplicate by parser node id, not `(snapshot_id, node_id)`. +Scope retrieval to the intended repository/snapshot when inspecting repeated +symbols across multiple indexes. Normal bound MCP supplies that scope. + +## Package hydration and budget allocation + +`get_context_package` calls context search with expansion enabled and search-stage +relationship inclusion disabled. It optionally loads repository/run metadata, +then hydrates each returned hit **sequentially** through +`_hydrate_context_package_item`. For each item it reads node/content, falls back +to the hit's content hash when necessary, looks up a completed summary, and +collects grouped relationship summaries if requested. Every graph lookup uses +the hit snapshot or resolved query namespace. + +Hydration completes before snippet allocation. The assembly loop divides the +remaining character allowance by the remaining item count, extracts/trims a +snippet, and carries unused allowance forward. Content metadata deliberately +omits the full stored text; `snippet` is the budgeted source output. The +structured `max_characters` budget counts emitted snippets, not JSON overhead, +relationship rows, summaries, or metadata. `total_snippet_characters` reports +that sum. + +`_snippet_for_hit` conditionally slices by line numbers when they fit within the +loaded content, then `_trim_text` applies a marker within the budget. Source +fragments can have file-absolute spans, so this conditional slicing has a known +edge case described in [Inspection findings](../guides/development-and-debugging.md#inspection-findings). +For a suspect snippet, inspect the declared path/span and content hash instead +of assuming trimming proves the source is absent. + +MCP agent mode formats the package using +[`format_context_package_for_agent`](../../agent-access/codemesh_agent_access/formatting.py) +and caps the **whole formatted brief** at 12,000 characters or the caller's +smaller allowance. It returns one `TextContent` in `CallToolResult`, with no +structured duplicate. JSON mode returns the complete structured package through +the tool envelope. Python CLI agent output honors its explicit budget without +the MCP-only ceiling. Character limits are not token-count guarantees. + +`test_context_package_includes_snippet_relationships_and_registry_metadata` in +[ranking tests](../../agent-access/tests/test_context_ranking.py), plus +[formatting tests](../../agent-access/tests/test_formatting.py) and +[CLI tests](../../agent-access/tests/test_cli_output.py), protect these separate +budgets and output forms. + +## Relationship evidence and validation suggestions + +`_get_context_relationship_summaries` reads local graph relationships and +normalizes the opposite node's identity, kind, name, and source span. +`_prioritize_context_relationships` orders useful edges and applies group/total +limits. Incoming `Invokes` populate callers; outgoing `Invokes` populate callees; +other groups expose reads, writes, containment, definitions, and type uses. +These directional groups support likely impact navigation, but limits can omit +edges and heuristic parsers can omit or misresolve dependencies. + +The recommendation builder, `_validation_recommendations`, inspects packaged +file paths and whether the repository alias identifies CodeMesh. It suggests +known .NET, Agent Access Python, Rust, deployment, smoke, and Markdown checks, +with reasons and applicable paths. It does not run commands, inspect all target +build configurations, or infer a complete transitive test-coverage graph. Use +the target repository's own instructions to choose and authorize validation. + +`test_context_relationship_prioritization_prefers_agent_useful_edges` and +`test_context_relationship_prioritization_applies_group_limits` in +[ranking tests](../../agent-access/tests/test_context_ranking.py) cover selection. +The canonical command policy remains [COMMANDS.md](../../COMMANDS.md) and +[Testing](../evaluation/testing.md). + +## Failure interpretation and instrumentation + +| Observation | Code path and interpretation | +| --- | --- | +| MCP binding error | `bound_repository_reference`, `assess_binding`, or `require_accepted_binding`; search may not have started. Use status for the precise issues. | +| HTTP validation error | FastAPI/Pydantic could not construct a request; examine the surface schema before debugging ranking. | +| Empty hits | Scope, path/kind filters, bounded candidate acquisition, parser omissions, or adapter fallback. Check store health and diagnostics. | +| `lexical_fallback=true` | A vector existed but contributed no returned hits; this is narrower than every possible query-provider failure. | +| Empty snippet with metadata | Content excluded, absent, hash fallback missed, or character allowance exhausted. | +| Unknown freshness | Local Git/root or indexed provenance unavailable. A healthy container alone cannot verify a host path. | +| Raised adapter/normalization exception | Not all exceptions become empty responses. REST has no uniform custom error envelope for every facade failure. | + +Mongo and graph adapters catch selected dependency errors and can yield no data; +query embedding catches HTTP errors and returns no vector. Such behavior allows +some degraded reads but does not make all stores optional or guarantee every +failure becomes a graceful lexical fallback. The model-free development profile +still uses graph/content/registry stores for live retrieval. + +The opt-in `_ContextPackageTiming` records repository resolution, search, +metadata, item hydration, assembly, and individual hydration operations through +a sanitized sideband. It emits even on failure and does not alter the response +budget or ranking. `test_context_package_emits_sanitized_stage_timings_with_injected_delays` +in [ranking tests](../../agent-access/tests/test_context_ranking.py) uses a +controlled clock to verify stage attribution. Live evaluation aggregates +measured calls separately from warm-ups. Follow +[the evaluation guide](../evaluation/mcp-effectiveness.md) before collecting +live timings; a timing result is not retrieval-quality evidence. diff --git a/docs/current/security-and-redaction.md b/docs/current/security-and-redaction.md index fb89589..266c0ae 100644 --- a/docs/current/security-and-redaction.md +++ b/docs/current/security-and-redaction.md @@ -31,6 +31,30 @@ The current local-first ingestion path redacts sensitive-looking values before c Current behavior: - A shared repository path policy excludes generated, cache, virtual-environment, smoke, and temporary directories before supported parsers enumerate files. +- Files and directories reached through repository symlinks, junctions, or + other reparse points are excluded fail closed, including links that would + traverse to content outside the selected repository root. +- C# solution and project discovery uses the same containment policy. An + explicitly selected solution or project file is rejected when it is outside + that policy rather than being opened by MSBuild. +- Known secret files are excluded by default, including `.env` variants, + credential and secret data files, common private-key and keystore formats, + Terraform state, and selected local cloud or container credential files. + Safe environment templates such as `.env.example`, `.env.sample`, and + `.env.template` remain eligible. +- Repeated `--allow-path ` options restrict ingestion to matching + repository-relative paths. Repeated `--deny-path ` options add + exclusions and take precedence over allow patterns. +- Path patterns support `*`, `?`, and `**`, match case-insensitively, and reject + absolute paths or parent traversal. They are anchored at the repository root: + use `README.md` for only the root file and `**/README.md` for that basename at + any depth. Generated/cache exclusions cannot be overridden by an allow + pattern, and allow patterns cannot opt linked paths back into scope. +- `--include-known-secret-files` explicitly disables only the default known- + secret-file exclusions. It does not bypass generated/cache exclusions, + configured deny patterns, value redaction, or repository authorization. +- The normalized filter configuration and path-policy version participate in + snapshot identity, and watch uses the same effective filter as ingestion. - Private key blocks are replaced with ``. - Environment-style assignments with names containing `SECRET`, `PASSWORD`, `PASSWD`, `TOKEN`, `API_KEY`, `PRIVATE_KEY`, or `CONNECTION_STRING` are redacted. - JSON string values with sensitive-looking key names are redacted. @@ -39,17 +63,165 @@ Current behavior: - Redacted content receives a new content hash, and graph nodes are remapped to that redacted content hash before store writes, embeddings, and summaries. - Ingestion emits diagnostic `CMSEC001` when redaction changes one or more content items. -The following behavior remains planned in [Next Steps](../planning/next-steps.md): +Repository-specific sensitive-file conventions still require reviewed deny +patterns; CodeMesh does not infer every proprietary credential or artifact +name safely. -- Configurable allowlists and denylists for ingestion. -- Default exclusion for known secret files in addition to the current generated-directory exclusions. -- Broader tests for additional secret formats and false-positive control. +Example restricted ingestion: + +```powershell +dotnet run --project src/CodeMesh.Cli -- ingest ` + --root C:\path\to\repository ` + --allow-path "src/**" ` + --allow-path "tests/**" ` + --deny-path "src/generated/**" ` + --skip-embeddings +``` Shared multi-user storage adds tenant isolation, path privacy, authorization, retention, and verified-deletion requirements. Those requirements are proposed in [Repository Identity and Snapshot Retention](../planning/repository-identity-and-snapshot-retention.md); they are not current shared-service behavior. +### Development feedback sessions + +The MCP feedback write surface is disabled unless its process is started with +an exact `development-feedback` profile, repository binding, immutable session +manifest path, and reviewed manifest SHA-256. Startup and every feedback call +recheck the hash, expiry, revocation record, and exact participant binding. +Session activation and revocation use exclusive creation. The manifest allows +at most seven days and applies bounded packet, per-process request, text, and +path limits. The request limit is a runaway-output control, not a cross-process +security quota. + +Client packets are created exclusively under the exact bound checkout's +ignored `.codemesh-feedback/` directory. Linked or redirected outboxes, +absolute or traversing paths, secret-looking text, unsupported fields, and +client-supplied session, repository, binding, snapshot, CodeMesh, or +classification identities are rejected. The in-process tool trace stores only +tool names, sequence, success/failure state, and bounded diagnostic codes; it +does not retain arguments or responses. +Accepted snapshot and parser fields are derived from the most recent successful +status or context-package result already handled by that process. Recording a +packet does not initiate another store, provider, or network request. + +The `feedback-maintainer` profile reads only outboxes named by the active +manifest. Invalid packets produce bounded diagnostics rather than unsafe raw +content. Its resolution endpoint validates and hashes an in-memory draft; it +cannot write CodeMesh or an approval record. MCP annotations describe these +properties to capable hosts but are not the enforcement boundary. Direct IDE +filesystem access remains governed by host approval policy, repository +instructions, Git controls, and the human decision tied to exact feedback ids +and a resolution-plan hash. + +## Local Evaluation And Inspection Boundaries + +Evaluation suites are trusted local executable inputs: validation commands run +with the operator's privileges, and the selected Codex executable must be +trusted. Commands use argument arrays without shell interpolation. An explicit +shell executable inside a suite remains an explicit command, so argv handling +does not make an untrusted suite safe to run. + +Agent and model suite names and artifact identifiers accept ASCII letters, +digits, underscores, hyphens, and internal dots. The first character must be a +letter, digit, or underscore; trailing dots and Windows device names are +rejected. Identifiers must be unique ignoring case within a suite, including +across the model benchmark's question and change tiers. These restrictions +protect generated artifact paths on both supported path conventions. + +Setup patches use forward-slash paths relative to the suite directory. Absolute +paths, parent traversal, Windows drive or stream syntax, missing files, and +symlinks that resolve outside that directory are rejected. The evaluator checks +all suite artifacts before repository preparation or model execution, then +rechecks task destinations when executing them. Generated descendant paths +must resolve inside the selected artifact directory. Raw-trace directories are +created exclusively. These checks do not provide a sandbox against another +process concurrently changing the operator's filesystem. + +Explicit CLI input/output paths remain operator-selected. Report finalization +also trusts the selected report's recorded suite path. Internal response and +policy-audit destinations are supplied by the evaluator, never by the model's +answer tool arguments. Raw traces and retained workspaces remain sensitive. + +The five local inspection routes escape request and stored text and attributes +and encode repository identifiers in local URLs. Every dynamic link attribute +is HTML-escaped after URL encoding, and the numeric search limit is validated +before store access and escaped when rendered. Adversarial route tests cover +script and element injection, attribute breakout, and JavaScript-like values. +The local UI remains an administrative inspection surface, without a new +shared-service authentication or authorization claim. + +Feedback recording rejects symlinked, junction, or redirected outboxes and +creates each packet exclusively. Existing regular files, symlinks, and hardlinks +are preserved. These checks do not claim protection against a hostile process +concurrently replacing directories in the operator's checkout. + +## Hosted CodeQL Triage + +The latest hosted CodeQL run succeeded for published commit `8e9c0da`, but 39 +alerts remain open on that commit. A read-only review groups them as: + +- 13 path findings at the trusted evaluation-suite boundary that warrant safe + identifier validation and setup-patch containment; +- 3 command findings at intentional local argv execution boundaries, with no + shell interpolation but an explicit suite/executable trust requirement; +- 1 reflective-XSS finding that appears escaped in current call sites but needs + adversarial route tests before dismissal; and +- 22 explicit operator-selected or internally controlled path findings that + need contract-by-contract review rather than blanket dismissal. + +Local hardening now validates suite names and identifiers, contains setup +patches and generated descendants, tests literal argv execution, and tests all +five UI routes with adversarial values. The remaining explicit path boundaries +have been reviewed individually in the +[local security review](../evaluation/evidence/security-boundary-review-20260905.json). +The original triage above records the hosted findings before this correction. + +No alerts were dismissed and no security-clean claim is supported. The hosted +scan covers remote `8e9c0da`, not unpublished local changes. Exact alert numbers, +classification, and recommended order are retained in the +[CodeQL triage record](../evaluation/evidence/codeql-triage-8e9c0da.json). + +## Local CodeQL Verification + +The [retained local review](../evaluation/evidence/local-codeql-review-b73e19a.json) +uses CodeQL CLI `2.26.4`, Python queries `1.8.9`, and C# queries `1.9.2`, with +pinned `security-extended` suites. Python candidate `b73e19a` has zero findings +under the default threat model and 61 under default plus local inputs (58 path, +3 command). The C# source, unchanged since scanned `e0beba9`, has zero default +findings and 192 with local inputs (189 path, 3 command). All reported C# flows +originate in test-harness temporary or configured sample paths. Every local +result is retained with its location and reviewed boundary; none is dismissed. + +The same local-input configuration reproduces 39 Python results on published +`8e9c0da`. Before the HTML correction, local `e0beba9` has 62; afterward the +reflective-XSS finding is absent. The larger current count also includes newer +local CLI, installation, feedback, and path-validation surfaces. Counts alone +do not measure exploitability or establish a security improvement. + +Use local scans when unpublished security changes need review or an exact +release candidate lacks current hosted evidence. Scan a clean isolated +checkout without private archives or environment files. Pin the official CLI +and query packs, retain checksums, and use absolute suite paths when packs were +downloaded into a private custom directory. C# extraction must trace the full +solution build after locked restore. The exercised command shapes are: + +```powershell +codeql database create --language=python --build-mode=none --source-root= --threads=2 --ram=2048 +codeql database create --language=csharp --source-root= --command="dotnet build CodeMesh.sln --no-restore --no-incremental -p:UseSharedCompilation=false -nodeReuse:false" --threads=2 --ram=2048 +codeql database analyze --format=sarifv2.1.0 --output= --no-sarif-add-file-contents --threads=2 --ram=2048 --rerun +codeql database analyze --format=sarifv2.1.0 --output= --no-sarif-add-file-contents --threads=2 --ram=2048 --threat-model=local --rerun +``` + +Keep default and local-input results separate. The first attempt to add the +local threat model reused default-model BQRS results; those two SARIF files are +retained but invalid for local-threat claims. `--rerun` forces evaluation when +changing the model. Check extraction coverage, query identities, invocation +errors, and findings before accepting a result. Private SARIF and databases +may contain source paths and data-flow context; do not upload them implicitly. +Revisit this local procedure when equivalent exact-candidate hosted evidence +is available, and reverify cache behavior after tool changes. + ## Redaction Principles - Preserve enough structure for useful context. diff --git a/docs/decisions/0002-summary-qualification-evidence-boundary.md b/docs/decisions/0002-summary-qualification-evidence-boundary.md new file mode 100644 index 0000000..392ef60 --- /dev/null +++ b/docs/decisions/0002-summary-qualification-evidence-boundary.md @@ -0,0 +1,69 @@ +# Decision: Bind summary qualification across production generation and live retrieval + +- Status: accepted +- Date: 2026-09-01 +- Owners: CodeMesh maintainers + +## Context + +Generated node summaries become searchable navigation evidence. Qualification +therefore has to test the production prompt, parser, redaction, budget, and +provider path while also measuring downstream retrieval against an isolated +no-summary index. The existing live Agent Access evaluator already owns Recall +at k, precision, MRR, nDCG, reliability, and secret-leak measurements. Replacing +that lane would create two ranking contracts, while moving production summary +generation into Python would create a second prompt/parser implementation. + +Qualification artifacts also have different disclosure boundaries. Corpus +source, generated text, and review packets may be sensitive; a publishable +result must not need those fields. Human factual review and provider-backed +execution require authority and evidence that deterministic tests cannot grant. + +## Decision + +- Implement summary generation and qualification compilation in the .NET + ingestion/CLI surface so the runner calls the production summary prompt, + parser, redactor, budget policy, and provider adapters directly. +- Record clean CodeMesh source, frozen suite/corpus, deployment profile, prompt, + provider generation configuration, and measured repetition identities in a + private archive. +- Run a warm-up before measured repetitions and stop measured execution when + warm-up fails. +- Emit a model-blinded review packet separately from the private candidate + identity. Require two complete blinded reviews and aggregate disagreements + conservatively. +- Keep isolated-index creation and ranking measurement in the live Agent Access + evaluation lane. Bind distinct no-summary and candidate live reports to the + qualification run, shared ranking identity, and corpus identity before + compilation. +- Emit a sanitized report without source or generated response text. Missing, + dirty, mismatched, or incomplete evidence produces `invalid-run`; failed + declared gates produce `not-qualified`. +- Treat online-source authorization, model spend, human review, index isolation, + and publication as separate operator authorities. A CLI flag or runner pass + does not grant any of them. + +## Consequences + +- Qualification reuses production generation behavior and the existing live + retrieval metric contract instead of maintaining parallel implementations. +- Private archives and review packets require restricted evidence storage and + are not publication-safe by default. +- The runner does not silently start stores or invoke a model during normal + verification. A real qualification remains an attended, separately + authorized workflow. +- The checked-in synthetic suite verifies wiring only. A reviewed, stratified, + held-out corpus and exact deployment budgets are still required for a real + qualification claim. +- Cross-language live-report schema changes require both .NET and Python + verification because the .NET binder consumes the Agent Access report. + +## Verification + +- The deterministic .NET harness exercises input redaction, warm-up and measured + repetitions, immutable identity, provider usage aggregation, two-reviewer + compilation, mandatory retrieval binding, gates, and incompatible comparison + rejection. +- The Python Agent Access tests and deterministic MCP fixture suite protect the + live-report contract consumed by the binder. +- Repository standards checks validate the operating documentation and links. diff --git a/docs/decisions/0003-hard-capped-codex-evaluation-runner.md b/docs/decisions/0003-hard-capped-codex-evaluation-runner.md new file mode 100644 index 0000000..fe012a2 --- /dev/null +++ b/docs/decisions/0003-hard-capped-codex-evaluation-runner.md @@ -0,0 +1,79 @@ +# Decision: Enforce agent campaign tokens through a loopback Responses proxy + +- Status: accepted +- Date: 2026-09-01 +- Owners: CodeMesh maintainers + +## Context + +Paired agent campaigns need a hard reported-token ceiling across every model +request and any internal retry activity. Codex JSONL reports cumulative usage +only after a turn completes, while the Responses `max_output_tokens` parameter +limits generated visible and reasoning tokens but not submitted input. A +post-run usage audit can detect an overrun but cannot prevent one. + +Codex supports custom Responses providers and independent request and SSE retry +limits. The Responses API also exposes an input-token endpoint that can count a +submitted request before generation. These interfaces allow enforcement without +changing prompts, task ordering, MCP configuration, or grading. + +The controlling external interfaces are the +[Responses create method](https://developers.openai.com/api/reference/cli/resources/responses/methods/create), +[input-token count method](https://developers.openai.com/api/reference/typescript/resources/responses/subresources/input_tokens), +and [Codex configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference). + +## Decision + +- Keep `--runner codex` as the default, truthful fail-closed runner for agent + campaigns. +- Add explicit `--runner capped-codex`, which starts an ephemeral proxy bound to + a random loopback port for each agent execution. +- Give Codex a random per-execution proxy bearer token. Keep the upstream OpenAI + credential inside the proxy process path and exclude it from logs and reports. +- Bind the selected model to its exact bundled entry from the selected Codex + executable through an ephemeral `model_catalog_json`, preserving its shell and + MCP tool contract without forwarding remote model discovery. +- Before each `/responses` request, call `/responses/input_tokens` with every + supported input-bearing field, reserve counted input plus the admitted + worst-case reported charge for the generated-token envelope, and clamp + `max_output_tokens` to the smaller of the remaining allowance and the + selected model's documented 128,000-token output maximum. Because Codex + reports `output_tokens` inclusive of reasoning while the committed accounting + adds `reasoning_output_tokens`, reserve twice the admitted generated envelope. +- Configure the custom Codex provider with request and SSE retries set to zero. + Reject duplicate or declared retry activity as defense in depth. +- Support only foreground Responses calls with local function/custom tools. + Reject background operation, automatic compaction, prompt templates or + unreviewed request parameters outside the input-counting contract, other + provider endpoints, and provider-hosted cost-bearing tools. +- Settle a reservation only from a completed response with valid usage that + agrees with the prior input count. Retain the complete reservation after an + interrupted, missing-usage, or otherwise unknown response. +- Require cumulative proxy usage to match Codex final usage before accepting + completion. Emit only a sanitized hard-cap ledger in evaluation reports. + +## Consequences + +- A complete invocation cannot admit requests whose worst-case reported-token + charge exceeds its assigned allowance, including multiple internal model + turns. +- Interrupted or ambiguous calls can conservatively consume unused allowance + and make a campaign incomplete. The evaluator does not top up or retry them. +- The capped runner requires the OpenAI Responses and input-token endpoints and + does not support local model providers. +- Input-token counting adds one non-generating API request before each model + request. Its latency is runner overhead and must not be attributed to CodeMesh + retrieval. +- The runner enforces token accounting only. It does not grant model-spend, + release, deployment, publication, or external-system authority. + +## Verification + +- Deterministic tests cover exact-boundary admission, multiple requests, + generated-token clamping, accounting disagreement, missing usage, interrupted + streams with retained reservations, backend failure, retry rejection, zero + configured retries, unsupported cost activity, and secret non-disclosure. +- Configured provider-free preflight verifies runner capability before model + access while preserving the existing positive and wrong-checkout probes. +- A model-backed campaign is accepted only when every execution ledger is + complete and agrees with Codex final cumulative usage. diff --git a/docs/decisions/0004-parser-container-requires-dotnet-sdk.md b/docs/decisions/0004-parser-container-requires-dotnet-sdk.md new file mode 100644 index 0000000..72b356b --- /dev/null +++ b/docs/decisions/0004-parser-container-requires-dotnet-sdk.md @@ -0,0 +1,50 @@ +# Decision: Include the .NET SDK in the C# parser runtime + +- Status: implemented locally +- Date: 2026-09-05 +- Owners: CodeMesh maintainers + +## Context + +The parser's container used the ASP.NET runtime image. Its service started and +reported healthy, but a project parse through `MSBuildWorkspace` reported +`CMSHARP012` because no .NET SDK was installed, followed by `CMSHARP010` +filesystem fallback. A healthy HTTP endpoint therefore did not establish the +container's advertised project/solution parsing capability. + +## Decision + +Use the .NET 10 SDK image for the final parser container as well as its build +stage. Keep the existing read-only repository mount and loopback service +binding. Include the canonical version props and file in the restricted Docker +build context. Use an explicit `PublishDir` property for the publish output. + +The application source version and `ParserCapability.Version` serve different +purposes. The parser capability version identifies extractor behavior and is +not replaced by the application release version. + +## Consequences + +- Roslyn project loading has the SDK and MSBuild tooling it requires. +- The runtime image is larger and includes build tools; release evidence must + retain the exact image digest and dependency assessment. +- Referenced projects and assets still need compatible SDKs and dependencies. + This change does not guarantee that every repository loads or authorize + network restores, arbitrary build execution, or broader repository access. +- Health checks remain service-liveness evidence. Acceptance must exercise + parsing and inspect diagnostics for project-loading failure and fallback. + +## Verification + +Build the parser image, start it in the disposable Compose project, and request +`/parse` for the mounted CodeMesh.Domain project. Require nonempty nodes and +relationships, no error diagnostics, and neither `CMSHARP012` nor `CMSHARP010`. +Retain the initial runtime-only failure and the corrected image's result in the +local package verification evidence. Both application suites and the full +provider-free end-to-end smoke remain part of the surrounding validation. + +The corrected probe loaded CodeMesh.Domain with 1,229 nodes and 1,906 +relationships. It retained a `CMSHARP015` warning for generated editor-config +output on the read-only mount and an informational `CMSHARP011` project-loaded +diagnostic. No project-loading failure, filesystem fallback, or error +was reported. The warning remains an environment-acceptance limitation. diff --git a/docs/decisions/0005-isolate-parser-design-time-outputs.md b/docs/decisions/0005-isolate-parser-design-time-outputs.md new file mode 100644 index 0000000..29c517f --- /dev/null +++ b/docs/decisions/0005-isolate-parser-design-time-outputs.md @@ -0,0 +1,68 @@ +# Decision: Isolate parser container design-time outputs + +- Status: implemented locally +- Date: 2026-09-05 +- Owners: CodeMesh maintainers + +## Context + +The SDK runtime added in [decision 0004](0004-parser-container-requires-dotnet-sdk.md) +loads projects, but MSBuild attempts to write generated editor configuration and +assembly-info caches into a read-only repository. Suppressing generation would +remove compiler-visible metadata and could change the compilation context. + +## Decision + +The parser container installs `CodeMesh.DesignTime.targets` as an additive SDK +`Microsoft.Common.targets/ImportBefore` extension. It acts only for design-time +builds carrying the parser's private `CodeMeshDesignTimeOutputRoot` property. +Each workspace receives a request-owned temporary directory. A SHA-256 key of +project path, configuration, platform, target framework, and runtime identifier +separates projects and evaluation variants within that directory. + +The extension redirects standard intermediate and build output directories +before common targets calculate generated file paths. It preserves generation, +compiler-visible values, project references, and the original project extension +directory used to read existing NuGet assets. It does not disable generators, +replace repository imports, or change source paths. The output scope outlives +workspace loading and compilation creation and is deleted after workspace +disposal, including project failure paths. + +This is a parser-container integration. Host SDK installations are not modified. +The target file is packaged with the parser; deterministic tests import it into +explicit fixtures. The C# extractor capability advances to `0.2.3`, independently +of application version `0.1.0`, so fresh ingestion can identify the changed +project-loading behavior. Existing indexed snapshots are not silently rebound. + +## Consequences + +- Standard SDK design-time output can be generated while the repository mount + remains read-only. +- Concurrent requests and same-named projects have isolated output paths. +- NuGet assets and source continue to resolve from their original locations. +- Custom project targets with independent write paths can still fail on a + read-only mount. The extension is not a sandbox for untrusted MSBuild code, + an implicit restore, or a guarantee for every SDK/project configuration. +- Abrupt process termination can leave temporary files until the disposable + container is removed; normal completion and observed failure paths clean up. + +## Verification + +The deterministic suite checks compiler-visible metadata, same-named concurrent +projects, deliberate failure cleanup, and cross-project invocation edges with +the extension active. All 71 .NET tests pass with no skips; 228 Python tests and +five MCP fixture cases pass. The parser container builds and loads +CodeMesh.Domain through a verified read-only mount twice concurrently, with +1,229 nodes, 1,906 relationships, no warning/error diagnostics, and zero +remaining request directories. The two results are identical after excluding +only content observation timestamps. + +The implementation follows the SDK's additive import hooks and supported +[MSBuild SHA-256 property function](https://learn.microsoft.com/en-us/visualstudio/msbuild/property-functions#msbuild-stablestringhash). +The earlier warning and intermediate editor-config-only failure remain +historical diagnostics; the current probe covers standard design-time outputs. + +Revisit this extension when an SDK supplies an equivalent native read-only +workspace mode. Expand it only for a reproduced design-time output failure with +metadata and relationship parity checks; remove it if the supported SDK no +longer needs the redirection. diff --git a/docs/engineering/github-actions.md b/docs/engineering/github-actions.md index 489ee78..f5abd7b 100644 --- a/docs/engineering/github-actions.md +++ b/docs/engineering/github-actions.md @@ -6,9 +6,12 @@ The read-only workflow validates: - locked Python restore, tests, Ruff, and the deterministic in-process MCP fixture suite; - Git whitespace; +- synchronized application-version declarations; - Markdown with the repository-pinned `markdownlint-cli2` version; - local Markdown links; -- publication-safety patterns in Git-visible files; and +- publication-safety patterns in Git-visible files; +- read-only workflow policy, immutable action pins, and disabled checkout + credential persistence; and - Conventional Commit messages in the pushed range, or the pull-request title. Normal pushes validate the complete pushed range. Pull requests use the @@ -33,3 +36,5 @@ surfaces. Review action-version and permission changes as supply-chain changes. Do not add publication or deployment permissions to this verification workflow. +After a workflow change, run the repository-local policy check listed in +[`COMMANDS.md`](../../COMMANDS.md#repository-standards). diff --git a/docs/engineering/methodology.md b/docs/engineering/methodology.md new file mode 100644 index 0000000..1af1ee3 --- /dev/null +++ b/docs/engineering/methodology.md @@ -0,0 +1,117 @@ +# Software engineering methodology + +## Purpose + +This methodology records CodeMesh decisions and evidence boundaries that a +capable coding agent cannot infer reliably from general software-engineering +knowledge. It assumes the agent already knows how to inspect repositories, +design and edit code, debug failures, write tests, review diffs, and document +ordinary changes. + +Do not add guidance merely to restate those built-in capabilities. Add a +reusable control only when it changes a material decision because it encodes: + +- a project-specific engineering choice; +- a non-obvious safety, authority, privacy, or external-side-effect boundary; +- a fragile procedure whose order or stopping condition is essential; +- a contract with an external system or tool; or +- a repeated, evidenced failure that narrower repository guidance cannot fix. + +Generic language or framework advice, routine coding and testing practices, +speculative edge cases, and universal planner/coder/reviewer role chains do not +belong in durable CodeMesh guidance. + +## Evidence and claims + +Use the narrowest truthful state when describing work: + +1. **Proposed**: designed or planned but not implemented. +2. **Implemented**: present in the inspected source or artifact. +3. **Automatically verified**: supported by named passing automated checks. +4. **Inspected**: supported by a bounded manual review of identified evidence. +5. **Locally accepted**: exercised successfully in the declared local setting. +6. **Externally accepted**: accepted by the named host, attended process, or + external environment. +7. **Released**: published as an immutable identified version or artifact. +8. **Deployed**: installed in the identified target environment. +9. **Production-authorized**: separately approved for production or live use. + +One state never implies a later state. Record the exact commit, artifact, +environment, command, result, and limitation needed to support consequential +claims. A workflow definition, configured tool, fixture, or historical result +is not evidence that the current candidate passed. + +CodeMesh product-benefit claims also require the evidence classifications and +comparability rules in +[MCP Effectiveness Evaluation](../evaluation/mcp-effectiveness.md). A fixture, +assisted run, stale index, dirty checkout, non-adopted treatment, or historical +report supports only its explicitly bounded claim. + +## Authority boundaries + +Treat authority as scoped to the requested action: + +- analysis, explanation, diagnosis, and review are read-only unless the request + also authorizes a change; +- implementation authorizes scoped repository-local edits and proportionate + local verification, not unrelated cleanup or external mutation; +- a requested commit authorizes focused local staging and commit creation, not + history rewriting or publication; and +- push, tag, release, publication, deployment, migration, external spend, + production-data mutation, and live side effects each require their own + authority and controlling procedure. + +Use dry-run, read-only, isolated, or disposable inputs before a higher-risk +operation when the repository provides that path. Stop at the first unresolved +authority or evidence gate rather than silently lowering the standard. + +## Where guidance belongs + +Place each control at the narrowest durable layer: + +- **Principle**: a stable project decision in this methodology or the shared + engineering standards. +- **Skill**: a focused, reusable procedure with a discriminating trigger, + explicit outcome, and non-obvious guidance. +- **Workflow or runbook**: an ordered project goal with prerequisites, + checkpoints, stopping conditions, and authority gates. +- **Agent role**: an optional context, permission, or responsibility boundary + for genuinely independent delegated work. +- **Command index**: supported invocations in [`COMMANDS.md`](../../COMMANDS.md); + presence never grants authority to run a high-risk command. +- **Project `AGENTS.md` and canonical documentation**: repository facts, + architecture, domain invariants, read order, safety rules, and release gates. + +Keep normative detail in one canonical location and link to it elsewhere. + +## Reusable-control admission + +Before adding or expanding a principle, skill, agent, hook, plugin, or optional +tool, record answers to these questions: + +1. What material decision will the control change? +2. What repository evidence, repeated failure, or external contract requires + it? +3. Why is existing agent capability or narrower project guidance insufficient? +4. What is the narrow trigger and observable success criterion? +5. What overlap, context cost, permissions, and failure modes does it add? +6. When should it be revised, disabled, or removed? + +If the answers do not demonstrate incremental value, do not add the control. +Prefer instructions over scripts unless deterministic execution is necessary, +and prefer a project-local rule over a cross-project rule when the evidence is +project-specific. + +This test is why CodeMesh does not automatically adopt overlapping code-graph +tools, generic agent roles, hooks, model providers, or release automation from +an external baseline. Each needs independent value and the appropriate +authority before activation. + +## Applying the methodology + +Classify the request before selecting checks or changing state. Read-only +analysis, diagnosis, implementation, acceptance, release, and live operations +have different authority and evidence requirements. Use +[`COMMANDS.md`](../../COMMANDS.md), [Testing](../evaluation/testing.md), and the +task's canonical workflow for the actual checks; this document defines shared +decision boundaries rather than duplicating procedures. diff --git a/docs/engineering/release-notes-template.md b/docs/engineering/release-notes-template.md new file mode 100644 index 0000000..028ec02 --- /dev/null +++ b/docs/engineering/release-notes-template.md @@ -0,0 +1,53 @@ +# CodeMesh TODO version + +- Status: TODO prepared, tagged, published, deployed, and stable state +- Release date: TODO YYYY-MM-DD, or not yet released +- Tag: TODO `vMAJOR.MINOR.PATCH`, or not created +- Release commit: TODO full SHA +- Previous stable release: TODO version, or none + +These fields record separate states. Preparing notes or creating a commit does +not imply that a tag, publication, deployment, or stable-channel promotion +occurred. + +## Release highlights + +- TODO user-visible capability or operational improvement. + +## Fixes + +- TODO important backward-compatible fix, or `None`. + +## Breaking changes + +- TODO incompatible contract and migration path, or `None`. + +## Upgrade + +1. TODO stop or quiesce the application if required. +2. TODO back up persistent data. +3. TODO install or update the immutable release. +4. TODO run migrations. +5. TODO start and verify the application. + +## Migrations + +- TODO list migrations, expected duration, backup behavior, and failure gates, + or `None`. + +## Required data or configuration + +- TODO list required data, secrets/configuration changes, checksums, or + `No change`. + +## Verification + +```text +TODO exact clean-install, version, health, and smoke commands. +``` + +Expected version: TODO version. + +## Known limitations + +- TODO release-specific limitation, or `None known`. diff --git a/docs/engineering/release-preparation.md b/docs/engineering/release-preparation.md new file mode 100644 index 0000000..bf9ced7 --- /dev/null +++ b/docs/engineering/release-preparation.md @@ -0,0 +1,137 @@ +# Release Preparation + +Document type: candidate release policy and reviewable operating procedure + +Status: prepared locally; no release candidate, tag, publication, deployment, +or supported released-version line is declared. Owner acceptance of this +procedure and the completed exact-candidate evidence packet is still required. + +The [local review packet](../evaluation/review-packet-a158bc0.md) contains draft +notes, verified artifact and dependency identities, and a gate-by-gate ledger +for source `a158bc0`. It leaves unavailable acceptance and owner decisions open. + +## Version Identity + +[`VERSION`](../../VERSION) contains the canonical numeric source version, +currently `0.1.0`. This preserves the existing Python package version; it does +not retroactively create a release. [`Directory.Build.props`](../../Directory.Build.props) +sets the .NET assembly/package version from that file. The .NET informational +version can include a Git commit suffix. Python package metadata, its checked-in +export, and the lockfile must agree with `VERSION`; the repository check enforces +that agreement. REST OpenAPI reads the Python export. + +Use the [version commands](../../COMMANDS.md#application-version) without +opening stores. Record full Git SHA, clean status, build environment, dependency +locks, and artifact checksums alongside the version. A matching version alone +does not establish identical source, dependencies, or accepted behavior. + +Version changes update `VERSION`, Python metadata/export, and the regenerated +Python lockfile together. Run the version checker, both application suites, and +package verification. Schema versions for bindings, feedback, snapshots, and +evaluation reports remain independent and need explicit migration review. +`ParserCapability.Version` independently identifies extractor behavior. + +## Candidate Public Contract + +The initial candidate contract consists of the documented .NET CLI commands, +Python CLI commands, REST request/response models, typed .NET Agent Access +client, normal and diagnostic MCP profiles, installation plans, binding and +feedback formats, and persisted local identity/retention behavior. The exact +shapes and authorization differences are in +[Agent Access Contracts](../current/agent-access-contracts.md), +[Identity and Persistence](../current/identity-and-persistence.md), and +[`COMMANDS.md`](../../COMMANDS.md). + +Python implementation modules, graph query plans, ranking weights, HTML layout, +raw traces, private evaluation archives, and undocumented store fields are +internal. Generated summaries and model qualification remain opt-in and +unqualified. No shared-service, cross-tenant, hosted, or production support is +implied. Source citations and freshness checks remain required; rankings and +relationships are navigation evidence. + +During `0.x`, an intentional incompatible public change requires a minor version +increment, explicit migration or re-ingestion instructions, and release notes. +Compatible fixes may use patch increments. Before `1.0`, freeze and review the +public contract and establish a supported upgrade path. Published artifacts +are immutable; a correction receives a new version. + +## Environment And Support Evidence + +There are currently no supported published versions. The source targets .NET +10 and Python 3.12 or newer. The retained [Linux Python verification](../evaluation/evidence/python-runtime-verification-b73e19a.json) +passes all 235 tests and five fixture cases on 3.12.14, 3.13.15, and 3.14.7. +This source-checkout matrix does not establish every future runtime version. +The repository's Windows/PowerShell workflow and Linux Docker profile are +separate acceptance targets. A successful Linux check does not establish +Windows acceptance. Release notes must state exactly which OS/runtime profiles +were exercised and exclude unverified profiles from support claims. + +A candidate packet must include SDK/runtime and dependency-lock identities, +store image digests, enabled providers, loopback bindings, and resource needs. +Mutable image tags and loose source dependency constraints do not substitute +for those retained build identities. A source build needs the documented .NET, +Python/uv, PowerShell, Git, and optional Docker toolchain. + +## Candidate Verification Gates + +1. Select an immutable clean candidate and compare it with the previous released + version, or explicitly record that no previous release exists. Preserve + consumed evaluation failures and existing user state. +2. Run all deterministic and standards checks in [`COMMANDS.md`](../../COMMANDS.md), + including synchronized versions and Conventional Commits. Record passed, + failed, skipped, and unavailable checks separately. +3. Run the full end-to-end smoke on disposable stores. Exercise clean install, + CLI version, REST health/OpenAPI, normal MCP binding/freshness, and cleanup. + Build and inspect local Python and .NET distributable artifacts. Retain their + checksums and verify the installed package, rather than only editable source. +4. Run current dependency advisories and obtain an exact-candidate CodeQL scan. + Resolve or explicitly review remaining findings; successful scan execution + does not establish security cleanliness. The older remote `8e9c0da` scan is + insufficient for the current candidate. +5. Review persisted-state and configuration differences. If an old format is + unsupported, require a documented backup and re-ingestion path; do not infer + backward compatibility from matching schema labels. +6. Retain configured/onboarded outcome evidence appropriate to the release's + claims. Current mixed and consumed campaigns do not prove repeatable agent + benefit. Any fresh campaign needs separate model, source, and spend authority. +7. Complete the [release notes](release-notes-template.md), exact artifact and + environment manifest, known limitations, and owner review. Unavailable gates + remain open and cannot be converted to acceptance by documentation. + +## Upgrade And Recovery Rehearsal + +The [local recovery guide](../guides/backup-and-recovery.md) records the +successfully rehearsed stopped-volume procedure and its precise same-version +Linux boundary. It supplies local recovery evidence; upgrade and supported +environment acceptance remain separate. + +The separate [development upgrade rehearsal](../evaluation/evidence/local-development-upgrade-ce23f2e.json) +exercises `8e9c0da` to `ce23f2e` and exact whole-set rollback. Its local mechanism +passes, but the frozen sample dependency advisory prevents treating the packet +as complete supported-upgrade acceptance. + +Before updating an existing indexed environment, record its exact source and +store identities and stop writers for the selected instance. Back up the +Neo4j, MongoDB, and Qdrant state as one quiesced set, preserve private local +configuration securely, and prove restoration in isolated storage. Do not +combine one store's old backup with newer graph, content, or vector state. + +Restore an isolated copy, install the candidate, apply only documented +migrations or re-ingest the authorized repositories, and validate identity, +freshness, representative citations, counts, and deletion. On failure, stop the +candidate and restore the previous software and the complete consistent store +set. The absence of a previous released version means the first-release +upgrade rehearsal must name the actual development baseline being tested. + +Backup and rollback commands depend on the selected store versions and +operating environment. This procedure does not authorize mutations of existing +user stores; the concrete environment-specific commands and restoration result +must be attached to the candidate packet before approval. + +## Publication And Deployment + +After the complete packet is reviewable, obtain explicit owner authorization +naming the version, source SHA, artifact checksums, publication destination, +and permitted action. Tagging, package publication, GitHub release creation, +deployment, and stable promotion are separate recorded actions. A local commit, +prepared runbook, or hosted standards pass authorizes none of them. diff --git a/docs/engineering/standards.md b/docs/engineering/standards.md index 3bcc552..63b4f6a 100644 --- a/docs/engineering/standards.md +++ b/docs/engineering/standards.md @@ -10,6 +10,12 @@ - Follow the product priorities and deferred-work boundary in [`docs/planning/next-steps.md`](../planning/next-steps.md). +The [Software Engineering Methodology](methodology.md) defines the evidence +states, authority boundaries, guidance placement, and admission test for new +reusable controls. Do not add a hook, skill, agent role, plugin, workflow, or +overlapping repository tool without evidence that it changes a material +CodeMesh decision. + ## Commits Use [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/): @@ -22,19 +28,16 @@ Common types are `feat`, `fix`, `docs`, `test`, `refactor`, `perf`, `build`, `ci`, and `chore`. Use `!` and a `BREAKING CHANGE:` footer together for an incompatible public-contract change. Commit only task-related files or hunks. -The repository includes a non-mutating checker: - -```bash -python tools/check_conventional_commit.py --range origin/main..HEAD -``` - -It does not install Git hooks or alter Git configuration. See +The repository includes the non-mutating checker listed in +[`COMMANDS.md`](../../COMMANDS.md#repository-standards). It does not install +Git hooks or alter Git configuration. See [Conventional Commit validation](conventional-commits.md). ## Verification -Use risk-scaled verification. The canonical commands and end-to-end selection -rules are in [`docs/evaluation/testing.md`](../evaluation/testing.md). +Use the supported invocations in the repository-level +[`COMMANDS.md`](../../COMMANDS.md) and the risk-scaled selection rules in +[`docs/evaluation/testing.md`](../evaluation/testing.md). - Run both .NET and Python suites after shared-contract, integration, serialization, lifecycle, or end-to-end changes. @@ -45,27 +48,13 @@ rules are in [`docs/evaluation/testing.md`](../evaluation/testing.md). without explicit authorization and appropriate gates. - Report passed, failed, skipped, and unavailable checks separately. -For every change, run: - -```bash -git diff --check -``` - -For Git-owned Markdown changes, run: - -```bash -npx --yes markdownlint-cli2@0.23.1 "**/*.md" -python tools/check_markdown_links.py -``` - -The link checker is local-only and performs no network requests. - ## Documentation and decisions - Update documentation with behavior, commands, contracts, setup, architecture, or verified status changes. -- Distinguish proposed, implemented, automatically tested, manually inspected, - accepted, released, and deployed states. +- Distinguish proposed, implemented, automatically verified, inspected, locally + accepted, externally accepted, released, deployed, and production-authorized + states. - Keep normative guidance in one canonical location and link to it elsewhere. - Use the [Documentation Hub](../README.md) to preserve the boundary between current references, status, execution order, proposals, and historical @@ -74,6 +63,23 @@ The link checker is local-only and performs no network requests. [`docs/decisions/0000-decision-record-template.md`](../decisions/0000-decision-record-template.md). - Keep dates, versions, links, measurements, and test claims truthful. +## Releases and versioning + +CodeMesh has no published release or supported released-version line. +`VERSION` is the canonical source version; .NET consumes it during build, +Python declarations are checked for agreement, and REST uses the Python export. +Configuration and evidence schemas have independent versions. + +[Release Preparation](release-preparation.md) defines the candidate API scope, +compatibility rules, verification and migration gates, and a reviewable release +runbook. The runbook still needs owner acceptance and completed release evidence +before use. Apply [Semantic Versioning 2.0.0](https://semver.org/) to published +versions. Released tags and artifacts must be immutable. + +Use the [Release Notes Template](release-notes-template.md) to keep preparation, +tagging, publication, deployment, and stable promotion visibly separate. The +template and a completed implementation do not authorize any of those actions. + ## Definition of done A change is done when: diff --git a/docs/evaluation/assets/summary-qualification-deployment-evidence.example.json b/docs/evaluation/assets/summary-qualification-deployment-evidence.example.json new file mode 100644 index 0000000..c51bdc2 --- /dev/null +++ b/docs/evaluation/assets/summary-qualification-deployment-evidence.example.json @@ -0,0 +1,12 @@ +{ + "schemaVersion": "1.0", + "kind": "summary-qualification-deployment-evidence", + "runId": "replace-with-run-id", + "profileHash": "replace-with-profile-hash-from-private-archive", + "measurementMethod": "replace-with-resource-or-provider-cost-measurement-method", + "peakHostMemoryMb": 0, + "peakAcceleratorMemoryMb": null, + "providerReportedCost": null, + "pricingSnapshotHash": null, + "complete": false +} diff --git a/docs/evaluation/assets/summary-qualification-profile.example.json b/docs/evaluation/assets/summary-qualification-profile.example.json new file mode 100644 index 0000000..fc250c0 --- /dev/null +++ b/docs/evaluation/assets/summary-qualification-profile.example.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": "1.0", + "kind": "summary-qualification-profile", + "name": "replace-with-profile-name", + "intendedUse": "local-offline", + "endpointClass": "local", + "provider": "ollama", + "model": "replace-with-exact-model-id", + "modelRevision": "replace-with-artifact-or-build-hash", + "quantization": "replace-with-quantization", + "numericPrecision": "replace-with-precision", + "chatTemplateHash": "replace-with-sha256", + "contextLength": 32768, + "maxInputCharacters": 4000, + "reasoningTokenReserve": 768, + "maxCompletionTokens": 1536, + "repetitions": 3, + "generationSettings": { + "providerGenerationConfigurationFingerprint": "temperature:0.1;format:json", + "temperature": "0.1", + "topP": "provider-default", + "topK": "provider-default", + "seed": "provider-default", + "stopSequences": "provider-default", + "requestedReasoningEffort": "none", + "effectiveReasoningEffort": "none", + "timeout": "provider-default", + "retryPolicy": "none", + "concurrency": "1", + "batching": "1", + "responseFormat": "json" + }, + "host": { + "operatingSystem": "replace-with-os-and-version", + "cpu": "replace-with-cpu", + "memory": "replace-with-memory", + "accelerator": "none", + "acceleratorMemory": "none", + "servingRuntime": "replace-with-runtime-and-version" + }, + "conditions": [] +} diff --git a/docs/evaluation/assets/summary-qualification-retrieval-assessment.example.json b/docs/evaluation/assets/summary-qualification-retrieval-assessment.example.json new file mode 100644 index 0000000..56ee3e8 --- /dev/null +++ b/docs/evaluation/assets/summary-qualification-retrieval-assessment.example.json @@ -0,0 +1,18 @@ +{ + "schemaVersion": "1.0", + "kind": "summary-qualification-retrieval-assessment", + "runId": "replace-with-run-id", + "baselineReportHash": "replace-with-no-summary-live-report-sha256", + "candidateReportHash": "replace-with-candidate-summary-live-report-sha256", + "correctSummaryHitContributions": 0, + "falsePositiveSummaryHits": 0, + "queriesImproved": 0, + "queriesUnchanged": 1, + "queriesDegraded": 0, + "baselineContextRelevance": 0, + "candidateContextRelevance": 0, + "baselineContextTokenDensity": 0, + "candidateContextTokenDensity": 0, + "unresolvedMaterialRegressions": 0, + "reviewed": true +} diff --git a/docs/evaluation/assets/summary-qualification-review.example.json b/docs/evaluation/assets/summary-qualification-review.example.json new file mode 100644 index 0000000..be52c0c --- /dev/null +++ b/docs/evaluation/assets/summary-qualification-review.example.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": "1.0", + "kind": "summary-qualification-review", + "runId": "replace-with-run-id", + "reviewerId": "replace-with-pseudonymous-reviewer-id", + "blinded": true, + "resolutionRecordHash": "replace-with-shared-resolution-record-sha256", + "samples": [ + { + "sampleId": "replace-with-sample-id-from-review-packet", + "itemId": "replace-with-corpus-item-id", + "supportedFactCount": 0, + "generatedFactCount": 0, + "requiredFactCount": 0, + "requiredFactPresentCount": 0, + "criticalHallucination": false, + "contradiction": false, + "purposeAccurate": false, + "behaviorFactCount": 0, + "behaviorFactPresentCount": 0, + "usefulSupportedFactCount": 0, + "overallFactuallyCorrect": false, + "confidenceAppropriate": false, + "tagTruePositiveCount": 0, + "tagFalsePositiveCount": 0, + "tagFalseNegativeCount": 0, + "safetyFailureCount": 0, + "limitation": null + } + ] +} diff --git a/docs/evaluation/assets/summary-qualification-suite.example.json b/docs/evaluation/assets/summary-qualification-suite.example.json new file mode 100644 index 0000000..2c2a881 --- /dev/null +++ b/docs/evaluation/assets/summary-qualification-suite.example.json @@ -0,0 +1,119 @@ +{ + "schemaVersion": "1.0", + "kind": "summary-qualification-suite", + "name": "codemesh-summary-model-example", + "version": "1.0.0", + "items": [ + { + "id": "pure-addition", + "repositoryId": "synthetic:summary-qualification", + "repositoryCommit": "synthetic-v1", + "nodeId": "synthetic:method:Add", + "contentHash": "4b6c18d2fa709103387ae6d4608f723315df36913eb20b56bc7347d08a48680f", + "language": "csharp", + "nodeKind": "method", + "name": "Add", + "project": "Synthetic", + "filePath": "Calculator.cs", + "startLine": 1, + "endLine": 1, + "source": "public int Add(int left, int right) => left + right;", + "inputSizeBand": "small", + "criticality": "normal", + "strata": ["pure", "small", "return-value"], + "requiredFacts": ["Returns the sum of left and right."], + "optionalFacts": ["The method is side-effect free."], + "forbiddenClaims": ["Subtracts either input.", "Mutates state."], + "requiredTags": ["addition"], + "acceptableTags": ["math", "pure"], + "partition": "calibration", + "expectedResponsibility": "Adds two integer inputs.", + "importantInputs": ["left and right integer values"], + "importantOutputs": ["the integer sum"], + "sideEffects": [], + "failureBehavior": [], + "referenceSummary": "Returns the sum of two integer inputs.", + "secretCanaries": [] + }, + { + "id": "save-with-audit", + "repositoryId": "synthetic:summary-qualification", + "repositoryCommit": "synthetic-v1", + "nodeId": "synthetic:method:Save", + "contentHash": "671d9f562cda3a333952324716b34020b7a981fecefe003df97493093cf6f366", + "language": "csharp", + "nodeKind": "method", + "name": "Save", + "project": "Synthetic", + "filePath": "OrderService.cs", + "startLine": 1, + "endLine": 1, + "source": "public void Save(Order order) { repository.Save(order); audit.Write(order.Id); }", + "inputSizeBand": "small", + "criticality": "high", + "strata": ["side-effect", "orchestration", "small"], + "requiredFacts": ["Persists the order.", "Writes an audit record for the order identifier."], + "optionalFacts": ["Performs persistence before auditing."], + "forbiddenClaims": ["Returns a saved order.", "Suppresses repository failures."], + "requiredTags": ["persistence", "audit"], + "acceptableTags": ["order", "side-effect"], + "partition": "qualification", + "expectedResponsibility": "Persists an order and records the persistence in an audit sink.", + "importantInputs": ["the order to persist"], + "importantOutputs": [], + "sideEffects": ["persists the order", "writes the order identifier to the audit sink"], + "failureBehavior": ["repository or audit failures propagate"], + "referenceSummary": "Persists an order and then records its identifier in the audit sink.", + "secretCanaries": [] + }, + { + "id": "load-with-missing-file-path", + "repositoryId": "synthetic:summary-qualification", + "repositoryCommit": "synthetic-v1", + "nodeId": "synthetic:method:TryLoad", + "contentHash": "1bc85af6a11ae880ad6f787fc8b1b55c605ee6c602ee13d35f28e3b04bbb8ff2", + "language": "csharp", + "nodeKind": "method", + "name": "TryLoad", + "project": "Synthetic", + "filePath": "ConfigLoader.cs", + "startLine": 1, + "endLine": 1, + "source": "public bool TryLoad(string path, out Config? config) { if (!File.Exists(path)) { config = null; return false; } config = Parse(File.ReadAllText(path)); return true; }", + "inputSizeBand": "small", + "criticality": "high", + "strata": ["failure-path", "file-io", "small"], + "requiredFacts": ["Returns false and a null config when the file is absent.", "Reads and parses the file when it exists.", "Returns true after parsing an existing file."], + "optionalFacts": ["Uses an out parameter for the loaded config."], + "forbiddenClaims": ["Creates a missing file.", "Catches parse failures."], + "requiredTags": ["configuration", "file-io"], + "acceptableTags": ["parse", "failure-path"], + "partition": "qualification", + "expectedResponsibility": "Loads configuration from an existing file while reporting a missing path without loading.", + "importantInputs": ["configuration file path"], + "importantOutputs": ["success flag", "parsed configuration or null"], + "sideEffects": ["reads an existing file"], + "failureBehavior": ["returns false with null config when the file does not exist", "parse failures propagate"], + "referenceSummary": "Loads and parses configuration from an existing file; reports false with a null result when the path is missing.", + "secretCanaries": [] + } + ], + "gates": { + "minimumFirstAttemptSchemaRate": 0.995, + "minimumEventualCompletionRate": 0.999, + "maximumSecretCanaryLeaks": 0, + "maximumCriticalHallucinations": 0, + "minimumAtomicFactualPrecision": 0.95, + "minimumRequiredFactRecall": 0.8, + "minimumStratumRequiredFactRecall": 0.7, + "maximumAbsoluteRetrievalRegression": 0.01, + "maximumSafetyFailures": 0, + "maximumP95LatencyMs": 10000, + "minimumNodesPerMinute": 1, + "maximumTotalTokens": 100000, + "maximumTokensPerSuccessfulNode": 10000, + "maximumTotalDurationMs": 120000, + "maximumRetryTokenOverheadRatio": 0, + "maximumPeakHostMemoryMb": 16384 + } +} diff --git a/docs/evaluation/evidence/codeql-triage-8e9c0da.json b/docs/evaluation/evidence/codeql-triage-8e9c0da.json new file mode 100644 index 0000000..358ebe2 --- /dev/null +++ b/docs/evaluation/evidence/codeql-triage-8e9c0da.json @@ -0,0 +1,64 @@ +{ + "schema_version": "codemesh-codeql-triage-v1", + "evidence_class": "hosted-remote-read-only-triage", + "recorded_at": "2026-08-31T19:42:35Z", + "repository": "Oneiros667/CodeMesh", + "scanned_commit": "8e9c0da7b497fe51126e3ef66509a8587cfe8d83", + "latest_codeql_run": { + "database_id": 33318677130, + "conclusion": "success", + "created_at": "2026-08-30T15:05:55Z" + }, + "open_alerts": { + "total": 39, + "critical": 3, + "high": 36, + "rules": { + "py/command-line-injection": 3, + "py/path-injection": 35, + "py/reflective-xss": 1 + } + }, + "triage": [ + { + "priority": "P0", + "classification": "trusted-suite-boundary-hardening", + "alert_numbers": [7, 8, 9, 10, 11, 12, 13, 14, 22, 23, 24, 25, 26], + "count": 13, + "assessment": "Suite-provided setup-patch paths and task, question, or scenario identifiers reach artifact and retained-workspace paths. Current evaluation suites are explicit local operator inputs and may already declare validation commands, but containment and safe-segment validation should be enforced before accepting untrusted suites." + }, + { + "priority": "P1", + "classification": "explicit-local-command-boundary", + "alert_numbers": [1, 2, 3], + "count": 3, + "assessment": "The evaluator executes argv arrays without a shell. One sink runs suite-declared validation and Git commands; two invoke the explicitly selected Codex executable. This is intentional opt-in local execution, not a remote command-injection surface, but suite and executable trust must remain explicit and should be hardened before broader distribution." + }, + { + "priority": "P2", + "classification": "likely-false-positive-needs-regression-test", + "alert_numbers": [39], + "count": 1, + "assessment": "The HTML sink receives composed markup. Current request and store values are escaped for text or attributes, and repository identifiers are URL-quoted. Add adversarial route tests before dismissing the alert; the hosted success alone does not prove XSS safety." + }, + { + "priority": "P3", + "classification": "explicit-operator-or-internal-path-boundary", + "alert_numbers": [4, 5, 6, 15, 16, 17, 18, 19, 20, 21, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38], + "count": 22, + "assessment": "These findings cover explicit CLI input/output paths, custom suite and report paths, internally supplied temporary response paths, repository roots, and a commit-message file. They are local operator-selected boundaries; add containment only where the command contract promises a managed destination." + } + ], + "recommended_order": [ + "Validate suite identifiers as safe path segments and enforce suite-relative setup-patch containment.", + "Document and test the trusted-suite and explicit executable boundary while preserving shell-free argv execution.", + "Add adversarial UI route tests for request and stored values, then reassess alert 39.", + "Review remaining operator-selected paths against each CLI command's documented destination contract." + ], + "boundaries": { + "alert_dismissals": "not_performed", + "code_changes": "not_performed", + "local_unpublished_commits_scanned": false, + "security_clean_claim": false + } +} diff --git a/docs/evaluation/evidence/config-net-focused-query-diagnosis-b16eee7.json b/docs/evaluation/evidence/config-net-focused-query-diagnosis-b16eee7.json new file mode 100644 index 0000000..46cf17b --- /dev/null +++ b/docs/evaluation/evidence/config-net-focused-query-diagnosis-b16eee7.json @@ -0,0 +1,121 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-provider-free-focused-query-diagnosis", + "generated_at": "2026-09-01T13:56:27.476741+00:00", + "baseline": { + "codemesh_commit": "b16eee7c82ee1c43e5dd73e9c92ff4c7451edae2", + "codemesh_clean_detached_checkout": true, + "repository_commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "repository_working_tree_dirty": false, + "project_id": "prj_e5d8c89734dc41958892d6f8f9878b89", + "checkout_id": "chk_6d1d16158afa4964bd19c9bf4d3e9193", + "snapshot_id": "snp_165b1edc656765c92df79fa4369c7f49b0a99b1c9a6c884c96c2ab55b1cb0ec5", + "source_view_hash": "b4981658b9c7ae39100f3b92d871727234e4b6ec4adaee4e9918789187173e57", + "provider": "none" + }, + "full_task_probe": { + "passed": false, + "query_term_count": 58, + "limit": 8, + "required_target_count": 7, + "required_targets_returned": [ + "src/Config.Net.Tests/LogicTest.cs" + ], + "required_targets_missed": [ + "src/Config.Net/Core/IoHandler.cs", + "src/Config.Net/Core/LazyVar.cs", + "src/Config.Net/Core/DynamicWriter.cs", + "src/Config.Net/Core/InterfaceInterceptor.cs", + "src/Config.Net/ConfigurationBuilder.cs", + "src/Config.Net.Tests/ConfigurableMethodsTest.cs" + ], + "returned_paths": [ + "src/Config.Net.Tests/LogicTest.cs", + "src/Config.Net.Tests/MultipleConfigurationFilesTest.cs", + "src/Config.Net.Tests/NotifyPropertyChangedTest.cs", + "src/Config.Net.Tests/Stores/Formats/IniKeyValueTest.cs", + "src/Config.Net.Tests/Stores/IniFileConfigStoreTest.cs", + "src/Config.Net.Tests/Stores/JsonFileCOnfigStoreTest.cs", + "src/Config.Net.Tests/Virtual/VirtualStoreTest.Basics.cs", + "src/Config.Net.Tests/Virtual/VirtualStoreTest.cs" + ] + }, + "focused_query_suite": { + "suite": "config-net-cache-focused-live", + "suite_definition_status": "uncommitted_diagnostic_definition", + "suite_definition_sha256": "6ef9626b937194eda130aeb5ab32d9d61bdcaa4b968c163a6475444997a42c8d", + "passed": true, + "comparable_retrieval_report": true, + "case_count": 5, + "tool_call_count": 15, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.660714, + "mean_ndcg_at_k": 0.76259, + "secret_leaks": 0, + "maximum_primary_rank": 8, + "covered_required_targets": [ + "src/Config.Net/Core/IoHandler.cs", + "src/Config.Net/Core/LazyVar.cs", + "src/Config.Net/Core/DynamicWriter.cs", + "src/Config.Net/Core/InterfaceInterceptor.cs", + "src/Config.Net/ConfigurationBuilder.cs", + "src/Config.Net.Tests/LogicTest.cs", + "src/Config.Net.Tests/ConfigurableMethodsTest.cs" + ], + "report_sha256": "df6fc7c2175c1af30f16390e43b301b0625e7d836f36dd44bfd70cb971c981d6" + }, + "unchanged_canonical_live_gate": { + "suite": "config-net-cache-live", + "passed": true, + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 1.0, + "secret_leaks": 0, + "report_sha256": "2c88997af63ab39e3d30ebf13944ea6d5b7fdd24fe95b1fdf9c41c8af75a9426" + }, + "implemented_worktree_correction": { + "base_commit": "5d7d1099efdae913ae3db14fa4373f111b679f3d", + "committed": false, + "change": "replace_one_long_full_task_query_guidance_with_small_focused_implementation_and_test_queries", + "context_package_remains_single_entry_tool": true, + "python_tests_passed": 125, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "clean_candidate_configured_preflight": "pending_commit_authority", + "clean_candidate_focused_live_gate": "pending_commit_authority" + }, + "claim_ceiling": { + "established": [ + "the_frozen_full_task_query_misses_six_of_seven_required_targets", + "four_focused_task_derived_queries_cover_all_seven_required_targets", + "the_unchanged_canonical_live_suite_still_passes", + "a_bounded_guidance_correction_is_implemented_and_deterministically_verified_in_the_worktree" + ], + "not_established": [ + "clean_candidate_verification_of_the_guidance_correction", + "normal_agent_compliance_with_focused_query_guidance", + "correctness_or_efficiency_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "The focused suite definition was uncommitted when the clean b16eee7 evaluator ran it, so the report validates the retrieval strategy but is not an exact clean-candidate correction gate.", + "The retained paid campaign did not preserve context-package call arguments, raw package payloads, final messages, or workspaces, so the diagnosis does not prove which query each treatment used.", + "LazyVar was returned at rank eight in its focused case, leaving limited ranking margin." + ], + "boundaries": { + "model_provider_invoked": false, + "paid_campaign_retry_authorized": false, + "commit_authorized": false, + "push": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/config-net-focused-query-provider-free-3804028.json b/docs/evaluation/evidence/config-net-focused-query-provider-free-3804028.json new file mode 100644 index 0000000..a943e53 --- /dev/null +++ b/docs/evaluation/evidence/config-net-focused-query-provider-free-3804028.json @@ -0,0 +1,126 @@ +{ + "schema_version": "codemesh-config-net-focused-query-provider-free-v1", + "evidence_class": "provider-free-local-configured-integration", + "recorded_at": "2026-09-01T14:06:07Z", + "codemesh": { + "commit": "3804028ccc7d25304d95ff933c7291b79f3e43b5", + "clean_detached_checkout": true, + "change": "replace_one_long_full_task_query_guidance_with_small_focused_implementation_and_test_queries", + "context_package_remains_single_entry_tool": true, + "python_tests_passed": 125, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": "not_selected_no_dotnet_contract_or_source_change" + }, + "config_net": { + "repository_url": "https://github.com/aloneguid/config.git", + "commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "branch": "master", + "tracked_files_changed": false, + "project_id": "prj_e5d8c89734dc41958892d6f8f9878b89", + "checkout_id": "chk_6d1d16158afa4964bd19c9bf4d3e9193", + "snapshot_id": "snp_165b1edc656765c92df79fa4369c7f49b0a99b1c9a6c884c96c2ab55b1cb0ec5", + "source_view_hash": "b4981658b9c7ae39100f3b92d871727234e4b6ec4adaee4e9918789187173e57", + "freshness_status": "fresh" + }, + "configured_installation": { + "profile": "normal", + "provider": "none", + "configuration_only": true, + "plan_hash": "af015e6371b524b79159ff648c3121eb7952c72e50df167c25582bd2ea0f90f7", + "plan_report_sha256": "cb603917add6122ac987a6b3a5cac133293abcd08d01e4937d0d5a88b5a02fb7", + "launch_sha256": "25a5ab5148a841608bc8e837ad2914f09a790f31587b763c7cff09f9b62d1080", + "installation_file": { + "path": ".codex/config.toml", + "source": "installed-ignored", + "sha256": "65e12917b25979d51562c06df5f982712e9687c72d72e4959db0007b048753f2" + }, + "guidance": { + "path": "AGENTS.override.md", + "source": "installed-ignored", + "sha256": "abb5fcdce32a53d14c618b18e842f0070e83fe9061e52d61a380d3a3086dd371" + }, + "expected_tools": [ + "codemesh_get_context_package", + "codemesh_get_repository_status", + "codemesh_list_repositories", + "codemesh_get_node" + ] + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "provider": "none", + "probe_query": "IoHandler CacheFor Read Write", + "expected_paths": [ + "src/Config.Net/Core/IoHandler.cs" + ], + "returned_path_count": 8, + "wrong_checkout_rejected": true, + "positive_probe_sha256": "24acada02426c12f88a17b6816ebc9e5bf89e6b26cb013cd188c88dbbeb4a4e8", + "rejection_probe_sha256": "18ea383a4f8f03e7e425fe3a05591529921e0944e6c96757c1bd6a4d922ef668", + "report_sha256": "771e851049f0383cbacef15c5c6102cc64323178bd8ebad6dd2d68bda589feb2" + }, + "focused_live_gate": { + "suite": "config-net-cache-focused-live", + "passed": true, + "comparable": true, + "case_count": 5, + "tool_call_count": 15, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.660714, + "mean_ndcg_at_k": 0.76259, + "secret_leaks": 0, + "maximum_primary_rank": 8, + "maximum_primary_rank_target": "src/Config.Net/Core/LazyVar.cs", + "model_providers": [], + "report_sha256": "c81bd78d3cf67b15b098311dfa41bac6f088cd33ea41e87b51c420c61298074a" + }, + "unchanged_canonical_live_gate": { + "suite": "config-net-cache-live", + "passed": true, + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 1.0, + "secret_leaks": 0, + "model_providers": [], + "report_sha256": "79e0d7c234eeddb1dee67c186b95276962788634627d48ca62a725c67b91f0d0" + }, + "claim_ceiling": { + "established": [ + "focused_query_guidance_correction_committed_and_clean_candidate_verified", + "provider_free_configured_plan_and_runtime_identity", + "fresh_fail_closed_config_net_binding", + "focused_queries_cover_all_seven_required_targets", + "unchanged_canonical_retrieval_gate_preserved" + ], + "not_established": [ + "normal_agent_compliance_with_focused_query_guidance", + "codemesh_correctness_or_efficiency_benefit_on_config_net", + "repeatable_cross_repository_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "The provider-free gates verify the installed guidance, runtime identity, fail-closed rejection, and retrieval strategy; they do not test model behavior.", + "LazyVar remains at the focused suite's maximum accepted primary rank of eight.", + "The retained paid campaign did not preserve raw context-package calls or final workspaces, so this evidence cannot attribute its misses to a specific query." + ], + "boundaries": { + "model_provider_invoked": false, + "paid_campaign_retry_authorized": false, + "push": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-config-net-agent-3804028.json b/docs/evaluation/evidence/configured-config-net-agent-3804028.json new file mode 100644 index 0000000..17000dc --- /dev/null +++ b/docs/evaluation/evidence/configured-config-net-agent-3804028.json @@ -0,0 +1,133 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-configured-onboarded-agent-campaign", + "generated_at": "2026-09-01T14:49:29.048020+00:00", + "codemesh": { + "commit": "3804028ccc7d25304d95ff933c7291b79f3e43b5", + "clean_detached_checkout": true, + "configured_suite": "config-net-cache-configured-agent", + "focused_query_guidance_correction": true + }, + "repository": { + "repository_url": "https://github.com/aloneguid/config.git", + "project_id": "prj_e5d8c89734dc41958892d6f8f9878b89", + "checkout_id": "chk_6d1d16158afa4964bd19c9bf4d3e9193", + "commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "branch": "master", + "tracked_files_changed": false, + "snapshot_id": "snp_165b1edc656765c92df79fa4369c7f49b0a99b1c9a6c884c96c2ab55b1cb0ec5", + "source_view_hash": "b4981658b9c7ae39100f3b92d871727234e4b6ec4adaee4e9918789187173e57" + }, + "configured_integration": { + "profile": "normal", + "plan_hash": "af015e6371b524b79159ff648c3121eb7952c72e50df167c25582bd2ea0f90f7", + "plan_report_sha256": "cb603917add6122ac987a6b3a5cac133293abcd08d01e4937d0d5a88b5a02fb7", + "installation_file_sha256": "65e12917b25979d51562c06df5f982712e9687c72d72e4959db0007b048753f2", + "guidance_sha256": "abb5fcdce32a53d14c618b18e842f0070e83fe9061e52d61a380d3a3086dd371", + "baseline_mcp_servers": [], + "prompt_parity": true, + "evaluator_only_treatment_guidance": false, + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "preflight_report_sha256": "771e851049f0383cbacef15c5c6102cc64323178bd8ebad6dd2d68bda589feb2" + }, + "campaign": { + "suite": "config-net-cache-configured-agent", + "provider": "default", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "completed_pair_count": 3, + "all_runs_completed": true, + "verdict": "improved", + "harness_passed": true, + "harness_failure_count": 0, + "treatment_wins": 1, + "codemesh_attributable_wins": 1, + "treatment_regressions": 0, + "treatment_safety_violations": 0, + "control_success_rate": 0.666667, + "treatment_success_rate": 1.0, + "codemesh_adoption_rate": 1.0, + "codemesh_adopted_pair_count": 3, + "context_package_call_attempt_count": 8, + "context_package_call_attempts_per_treatment": [ + 3, + 2, + 3 + ], + "mcp_failed_call_count": 0, + "command_policy_violation_count": 0, + "matched_successful_pair_count": 2, + "matched_control_duration_ms": 238113.932, + "matched_treatment_duration_ms": 205524.894, + "matched_duration_delta": -0.136863, + "matched_control_tokens": 428294.5, + "matched_treatment_tokens": 412906.5, + "matched_token_delta": -0.035929, + "report_sha256": "f18e70e160907f326afa1e52d87cce8134963dd6b6fd57cebe20054409514a1f" + }, + "control_failures": [ + { + "repetition": 2, + "missing_required_citations": [ + "src/Config.Net/ConfigurationBuilder.cs" + ] + } + ], + "treatment_failures": [], + "measured_outcome": { + "context_package_adopted_in_every_treatment": true, + "multiple_context_package_attempts_in_every_treatment": true, + "all_treatments_correct_and_safe": true, + "correctness_result": "one_codemesh_attributable_win_and_zero_treatment_regressions", + "efficiency_result": "favorable_medians_across_two_jointly_successful_pairs" + }, + "comparison_to_retained_b16eee7_campaign": { + "same_suite_prompt_targets_thresholds_repository_model_reasoning_seed_and_repetitions": true, + "prior_verdict": "regressed", + "prior_treatment_success_rate": 0.333333, + "prior_treatment_regressions": 2, + "current_verdict": "improved", + "current_treatment_success_rate": 1.0, + "current_treatment_regressions": 0, + "causal_attribution_to_guidance_wording_established": false + }, + "claim_ceiling": { + "established": [ + "configured_prompt_parity_and_exact_binding", + "context_package_adoption_in_every_treatment", + "all_three_treatments_correct_and_safe", + "one_codemesh_attributable_win", + "zero_treatment_regressions", + "favorable_efficiency_medians_across_two_jointly_successful_pairs", + "improved_verdict_for_this_single_config_net_campaign" + ], + "not_established": [ + "causality_of_the_focused_query_wording", + "repeatable_configured_product_benefit", + "generalization_to_another_task_model_or_repository", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "The campaign covers one answer task, one repository, one model, and three repetitions.", + "Efficiency medians use the two pairs where both control and treatment passed; the third control missed ConfigurationBuilder.cs.", + "Raw traces, context-package arguments and payloads, final messages, and temporary workspaces were not retained.", + "The report proves multiple context-package attempts in every treatment but not the exact query wording used." + ], + "boundaries": { + "campaign_consumed": true, + "automatic_retry_authorized": false, + "additional_model_spend": "not_authorized", + "next_work": "independent_configured_replication_requires_separate_authority", + "additional_onenine_checkout_onboarding": "not_authorized", + "push": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-config-net-agent-b16eee7.json b/docs/evaluation/evidence/configured-config-net-agent-b16eee7.json new file mode 100644 index 0000000..8fff8a3 --- /dev/null +++ b/docs/evaluation/evidence/configured-config-net-agent-b16eee7.json @@ -0,0 +1,124 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-configured-onboarded-agent-campaign", + "generated_at": "2026-09-01T13:36:49.926119+00:00", + "codemesh": { + "commit": "b16eee7c82ee1c43e5dd73e9c92ff4c7451edae2", + "clean_detached_checkout": true, + "configured_suite": "config-net-cache-configured-agent" + }, + "repository": { + "repository_url": "https://github.com/aloneguid/config.git", + "project_id": "prj_e5d8c89734dc41958892d6f8f9878b89", + "checkout_id": "chk_6d1d16158afa4964bd19c9bf4d3e9193", + "commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "clean_detached_checkout": true, + "snapshot_id": "snp_165b1edc656765c92df79fa4369c7f49b0a99b1c9a6c884c96c2ab55b1cb0ec5", + "source_view_hash": "b4981658b9c7ae39100f3b92d871727234e4b6ec4adaee4e9918789187173e57", + "restored_branch": "master", + "restored_commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b" + }, + "configured_integration": { + "profile": "normal", + "plan_hash": "5b5f45a664a3e967175ca9dc43d281531925736fb28fe8ea4ad1f2188c966f02", + "installation_file_sha256": "ccbba4f6ab343b57134198bcfc796e32dfc185936d9805258cf1cbc974c18eb0", + "guidance_sha256": "f5ae3d36c08466b33db4459ec25f52954300b11d9f9fdc70630575e7a00f9961", + "baseline_mcp_servers": [], + "prompt_parity": true, + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "preflight_report_sha256": "71396b0e59f58520dd9060c0e77f1709e9b0b3c3ad8bc8d17baa770c35a95b13" + }, + "campaign": { + "suite": "config-net-cache-configured-agent", + "provider": "default", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "completed_pair_count": 3, + "all_runs_completed": true, + "verdict": "regressed", + "harness_passed": false, + "harness_failure_count": 0, + "treatment_wins": 0, + "codemesh_attributable_wins": 0, + "treatment_regressions": 2, + "treatment_safety_violations": 0, + "control_success_rate": 1.0, + "treatment_success_rate": 0.333333, + "codemesh_adoption_rate": 1.0, + "codemesh_adopted_pair_count": 3, + "context_retrieval_call_count": 3, + "mcp_failed_call_count": 0, + "command_policy_violation_count": 0, + "matched_successful_pair_count": 1, + "matched_control_duration_ms": 210468.373, + "matched_treatment_duration_ms": 255318.224, + "matched_duration_delta": 0.213095, + "matched_control_tokens": 348427.0, + "matched_treatment_tokens": 756728.0, + "matched_token_delta": 1.171841, + "report_sha256": "b88c5ace666a1d141c8067c1e562c7dfcbead7b7265aba221dafbda25b10e821" + }, + "treatment_failures": [ + { + "repetition": 1, + "missing_required_citations": [ + "src/Config.Net/ConfigurationBuilder.cs" + ] + }, + { + "repetition": 2, + "missing_required_citations": [ + "src/Config.Net/Core/LazyVar.cs", + "src/Config.Net/ConfigurationBuilder.cs", + "src/Config.Net.Tests/LogicTest.cs", + "src/Config.Net.Tests/ConfigurableMethodsTest.cs" + ] + } + ], + "measured_outcome": { + "context_package_adopted_in_every_treatment": true, + "configured_cross_repository_adoption_observed": true, + "correctness_result": "two_treatment_regressions_from_missing_required_citations", + "efficiency_result": "regressed_on_the_only_jointly_successful_pair" + }, + "comparison_to_retained_evidence": { + "assisted_config_net_campaign": "improved_but_not_prompt_parity_configured_evidence", + "prompt_parity_config_net_campaign": "zero_adoption_negative_discoverability_evidence", + "configured_onenine_campaign": "neutral_full_adoption_on_a_different_task_and_repository" + }, + "claim_ceiling": { + "established": [ + "configured_prompt_parity_and_exact_binding", + "context_package_adoption_in_every_treatment", + "configured_adoption_observed_on_two_repositories", + "two_config_net_treatment_correctness_regressions", + "no_safety_or_mcp_transport_failure" + ], + "not_established": [ + "codemesh_correctness_benefit", + "codemesh_efficiency_benefit", + "repeatable_configured_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "Only one pair had both control and treatment pass, so the reported efficiency deltas describe that matched successful pair rather than three successful pairs.", + "The sanitized report identifies missing required citations but does not retain raw context-package payloads, final messages, prompts, source excerpts, or temporary workspaces.", + "The campaign covers one answer task, one repository, one model, and three repetitions." + ], + "boundaries": { + "campaign_consumed": true, + "automatic_retry_authorized": false, + "next_work": "provider_free_task_shaped_context_package_coverage_diagnosis", + "additional_model_spend": "not_authorized", + "additional_onenine_checkout_onboarding": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-config-net-provider-free-b16eee7.json b/docs/evaluation/evidence/configured-config-net-provider-free-b16eee7.json new file mode 100644 index 0000000..fed65ba --- /dev/null +++ b/docs/evaluation/evidence/configured-config-net-provider-free-b16eee7.json @@ -0,0 +1,110 @@ +{ + "schema_version": "codemesh-configured-config-net-provider-free-v1", + "evidence_class": "provider-free-local-configured-integration", + "recorded_at": "2026-09-01T12:54:23Z", + "codemesh": { + "commit": "b16eee7c82ee1c43e5dd73e9c92ff4c7451edae2", + "clean_detached_checkout": true, + "configured_suite_added": "config-net-cache-configured-agent", + "frozen_prompt_targets_thresholds_and_repository_commit_preserved": true, + "python_tests_passed": 125, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": "not_selected_evaluation_fixture_only_change" + }, + "config_net": { + "repository_url": "https://github.com/aloneguid/config.git", + "commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "clean_detached_checkout_during_gates": true, + "restored_branch": "master", + "restored_commit": "ae0af7d4b3781c66233bbb677b6e11df1f5b416b", + "project_id": "prj_e5d8c89734dc41958892d6f8f9878b89", + "checkout_id": "chk_6d1d16158afa4964bd19c9bf4d3e9193", + "snapshot_id": "snp_165b1edc656765c92df79fa4369c7f49b0a99b1c9a6c884c96c2ab55b1cb0ec5", + "source_view_hash": "b4981658b9c7ae39100f3b92d871727234e4b6ec4adaee4e9918789187173e57", + "freshness_status": "fresh", + "tracked_files_changed": false + }, + "configured_installation": { + "profile": "normal", + "provider": "none", + "plan_hash": "5b5f45a664a3e967175ca9dc43d281531925736fb28fe8ea4ad1f2188c966f02", + "plan_report_sha256": "6feb780b9a8111a46aac65e2409780de0cb60b8d53417b659c994667b4aeaf93", + "installation_file": { + "path": ".codex/config.toml", + "source": "installed-ignored", + "sha256": "ccbba4f6ab343b57134198bcfc796e32dfc185936d9805258cf1cbc974c18eb0" + }, + "guidance": { + "path": "AGENTS.override.md", + "source": "installed-ignored", + "sha256": "f5ae3d36c08466b33db4459ec25f52954300b11d9f9fdc70630575e7a00f9961" + }, + "expected_tools": [ + "codemesh_get_context_package", + "codemesh_get_repository_status", + "codemesh_list_repositories", + "codemesh_get_node" + ] + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "provider": "none", + "probe_query": "IoHandler CacheFor Read Write", + "expected_paths": [ + "src/Config.Net/Core/IoHandler.cs" + ], + "returned_path_count": 8, + "wrong_checkout_rejected": true, + "report_sha256": "71396b0e59f58520dd9060c0e77f1709e9b0b3c3ad8bc8d17baa770c35a95b13" + }, + "config_net_live": { + "passed": true, + "comparable": true, + "suite": "config-net-cache-live", + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 1.0, + "secret_leaks": 0, + "p50_latency_ms": 4451.787, + "p95_latency_ms": 4640.26, + "model_providers": [], + "report_sha256": "fb6f45211bc732f387f21916604345a0be04e1f050d498397ca2f19d41d2b124" + }, + "retained_diagnostic": { + "stage": "configured_positive_probe_query_selection", + "model_provider_invoked": false, + "result": "The first broad task query returned tests but missed the two requested implementation paths. The accepted preflight used the existing canonical IoHandler live-suite query and target; the full live gate separately passed every frozen implementation and test case." + }, + "claim_ceiling": { + "established": [ + "provider_free_configured_plan_and_runtime_identity", + "prompt_parity_preflight_configuration", + "fresh_fail_closed_config_net_binding", + "comparable_config_net_retrieval_gate" + ], + "not_established": [ + "normal_agent_context_package_adoption_on_config_net", + "codemesh_correctness_or_efficiency_benefit_on_config_net", + "repeatable_cross_repository_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "raw_prompts_or_payloads_retained": false, + "model_provider_invoked": false, + "config_net_model_campaign": "not_run_requires_separate_explicit_model_spend_authority", + "additional_onenine_checkout_onboarding": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-evaluator-preflight-de239b7.json b/docs/evaluation/evidence/configured-evaluator-preflight-de239b7.json new file mode 100644 index 0000000..67bc7cb --- /dev/null +++ b/docs/evaluation/evidence/configured-evaluator-preflight-de239b7.json @@ -0,0 +1,55 @@ +{ + "schema_version": "codemesh-configured-evaluator-candidate-v1", + "evidence_class": "provider-free-local-candidate", + "recorded_at": "2026-08-31T19:39:47Z", + "codemesh": { + "commit": "de239b7ba115ee726dfe8891c407707b4edd2898", + "clean_checkout": true + }, + "deterministic_verification": { + "python_tests": { + "passed": 120, + "warnings": 2 + }, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite": { + "passed": true, + "cases": 5 + }, + "documentation_standards": "passed" + }, + "youtube_downloader": { + "status": "passed", + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "provider": "none", + "report_sha256": "783a7a72a4879d3d27ec29c77237a49b45e23dabaf34627f957057c7e065baff" + }, + "onenine_primary": { + "status": "not_run_stale_candidate", + "frozen_indexed_commit": "e12d0281b687e58c2baac85f771a684df6fa8552", + "current_checkout_commit": "499ae2630e5c8c31a4a7904f91a67eacc6ba5621", + "working_tree_dirty": false, + "freshness_status": "stale", + "configured_plan_available": false, + "required_resolution": "Select and index an explicit clean one|nine candidate, then generate, review, apply, and retain its exact normal-profile plan before configured preflight." + }, + "boundaries": { + "configured_agent_preflight": "not_run_until_onenine_identity_matches", + "model_backed_agent_evaluation": "not_run_requires_explicit_spend_authority", + "dotnet_suite": "not_run_python_evaluator_and_documentation_only_change", + "reindex_or_repoint_onenine": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "provider_access": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-onenine-agent-0378740.json b/docs/evaluation/evidence/configured-onenine-agent-0378740.json new file mode 100644 index 0000000..e009124 --- /dev/null +++ b/docs/evaluation/evidence/configured-onenine-agent-0378740.json @@ -0,0 +1,114 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-configured-onboarded-agent-campaign", + "generated_at": "2026-09-01T10:09:10.517601+00:00", + "codemesh": { + "commit": "03787409bfb1bbbb53ba83ea1a3de4f3bf8f6fc9", + "clean_detached_checkout": true, + "status_to_context_correction_included": true + }, + "repository": { + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "commit": "fe2f761583bf78601961ff17934185c4b6a632f9", + "clean_detached_checkout": true, + "snapshot_id": "snp_bf026d3f2e2b5a40f3c9fa112ae36b7e026b0f5a0f0483a23796b0a6adc9243e", + "source_view_hash": "fea1951d1eeb93ac265112aef7000dcac1a9832b000ea81b360b0851df56e69b", + "restored_main_commit": "9bc95680f9e6d77c0a10c46f463b00312d23b5b0" + }, + "configured_integration": { + "profile": "normal", + "plan_hash": "a535de566f63be65c20bb2a01f398f6fcef8b4da571970bc573ced9a6809e248", + "installation_file_sha256": "7b8a9df8aec48670c5e4558de05fe5281d19b1173462259791e82c7fb3942807", + "guidance_sha256": "dbd0faca021b144f57594d91eb04b02a089fd6599234b713efe78693c92deb76", + "baseline_mcp_servers": [], + "prompt_parity": true, + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "preflight_report_sha256": "ffc33889f61e284b07da8e1e9b35f1e2ab6ab92c4b67fbcc0a7fd2c499382416" + }, + "campaign": { + "suite": "onenine-native-replay-configured-agent", + "provider": "default", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "completed_pair_count": 3, + "verdict": "insufficient", + "harness_passed": true, + "treatment_wins": 0, + "codemesh_attributable_wins": 0, + "treatment_regressions": 0, + "treatment_safety_violations": 0, + "control_success_rate": 1.0, + "treatment_success_rate": 1.0, + "codemesh_adoption_rate": 0.333333, + "codemesh_adopted_pair_count": 1, + "context_retrieval_call_count": 0, + "repository_status_call_count": 1, + "mcp_failed_call_count": 0, + "command_policy_violation_count": 0, + "median_control_duration_ms": 253288.454, + "median_treatment_duration_ms": 268671.189, + "overall_duration_delta": 0.060732, + "median_control_tokens": 1744959.0, + "median_treatment_tokens": 1817666.0, + "overall_token_delta": 0.041667, + "status_adopting_pair_control_duration_ms": 253288.454, + "status_adopting_pair_treatment_duration_ms": 365300.207, + "status_adopting_pair_duration_delta": 0.44223, + "status_adopting_pair_control_tokens": 1495572.0, + "status_adopting_pair_treatment_tokens": 1817666.0, + "status_adopting_pair_token_delta": 0.215365, + "report_sha256": "983766c108d17dcf246699d755509b86110c5767c588ed29cbcd864f23040b95" + }, + "measured_correction_outcome": { + "fresh_status_result_action_present": true, + "treatment_status_call_count": 1, + "treatment_context_package_call_count": 0, + "context_handoff_observed": false, + "result": "did_not_produce_context_adoption" + }, + "comparison_to_retained_0bde606_campaign": { + "same_task_prompt_grading_control_guidance_model_and_repository_identity": true, + "prior_repository_status_call_count": 2, + "prior_context_retrieval_call_count": 0, + "current_repository_status_call_count": 1, + "current_context_retrieval_call_count": 0, + "both_verdicts": "insufficient" + }, + "claim_ceiling": { + "established": [ + "configured_prompt_parity_and_exact_binding", + "three_completed_correct_and_safe_pairs", + "repository_status_adoption_in_one_treatment", + "no_context_handoff_observed", + "no_codemesh_attributable_win" + ], + "not_established": [ + "context_package_adoption", + "codemesh_correctness_benefit", + "codemesh_efficiency_benefit", + "repeatable_configured_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "The harness passed because it recorded no matched regression; the insufficient verdict is not a product-benefit pass.", + "Only one treatment called repository status and it did not continue to context retrieval, so timing and token differences are not evidence of context-package utility.", + "The campaign covers one answer task in one private repository with one model and three repetitions.", + "Raw events, final messages, workspaces, prompts, source excerpts, and credentials were not retained." + ], + "boundaries": { + "campaign_consumed": true, + "automatic_retry_authorized": false, + "config_net_replication": "not_activated_without_valid_context_adopting_onenine_result", + "additional_model_spend": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-onenine-agent-0bde606.json b/docs/evaluation/evidence/configured-onenine-agent-0bde606.json new file mode 100644 index 0000000..f7760d3 --- /dev/null +++ b/docs/evaluation/evidence/configured-onenine-agent-0bde606.json @@ -0,0 +1,87 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-configured-onboarded-agent-campaign", + "generated_at": "2026-09-01T00:03:50.903659+00:00", + "codemesh": { + "commit": "0bde606cc73b4a0b081067a039822a53672459a1", + "clean_detached_checkout": true + }, + "repository": { + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "commit": "fe2f761583bf78601961ff17934185c4b6a632f9", + "clean_detached_checkout": true, + "snapshot_id": "snp_bf026d3f2e2b5a40f3c9fa112ae36b7e026b0f5a0f0483a23796b0a6adc9243e", + "source_view_hash": "fea1951d1eeb93ac265112aef7000dcac1a9832b000ea81b360b0851df56e69b" + }, + "configured_integration": { + "profile": "normal", + "plan_hash": "6f5e68c225f88b6162f386e4c3bfb9ae1d974f70d56b04a50c142be7542c004e", + "installation_file_sha256": "f27c4060e9ae39f6d055e8fb7b052e0784a768ce61a79cb671cd19e71a755dfc", + "guidance_sha256": "dbd0faca021b144f57594d91eb04b02a089fd6599234b713efe78693c92deb76", + "baseline_mcp_servers": [], + "prompt_parity": true, + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "preflight_report_sha256": "c4ea8e3bc644c81b37b9197f9216049165dff6ba7b1afaa6a75a5578d367aefb" + }, + "campaign": { + "suite": "onenine-native-replay-configured-agent", + "provider": "default", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "completed_pair_count": 3, + "verdict": "insufficient", + "harness_passed": true, + "treatment_wins": 0, + "codemesh_attributable_wins": 0, + "treatment_regressions": 0, + "treatment_safety_violations": 0, + "control_success_rate": 1.0, + "treatment_success_rate": 1.0, + "codemesh_adoption_rate": 0.666667, + "codemesh_adopted_pair_count": 2, + "context_retrieval_call_count": 0, + "repository_status_call_count": 2, + "mcp_failed_call_count": 0, + "command_policy_violation_count": 0, + "median_control_duration_ms": 199159.923, + "median_treatment_duration_ms": 259492.728, + "overall_duration_delta": 0.302936, + "median_control_tokens": 1558710.0, + "median_treatment_tokens": 2285269.0, + "overall_token_delta": 0.466128, + "adopted_pair_median_control_duration_ms": 232170.665, + "adopted_pair_median_treatment_duration_ms": 237317.304, + "adopted_pair_duration_delta": 0.022167, + "adopted_pair_median_control_tokens": 1867908.5, + "adopted_pair_median_treatment_tokens": 1982526.0, + "adopted_pair_token_delta": 0.061361, + "report_sha256": "a10de0d676e2abbe875c23616cf5bb5304fc975ec348bd3a340f5df0f540aff2" + }, + "claim_ceiling": { + "established": [ + "configured_prompt_parity_and_exact_binding", + "three_completed_correct_and_safe_pairs", + "repository_status_adoption_in_two_treatment_runs", + "no_codemesh_attributable_win" + ], + "not_established": [ + "context_package_adoption", + "codemesh_correctness_benefit", + "codemesh_efficiency_benefit", + "repeatable_configured_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "The harness passed because it recorded no matched regression; the insufficient verdict is not a product-benefit pass.", + "Two treatment runs called only repository status and did not continue to context retrieval, so their timing and token differences are not evidence of context-package utility.", + "The campaign covers one answer task in one private repository with one model and three repetitions.", + "Raw events, final messages, workspaces, prompts, source excerpts, and credentials were not retained." + ] +} diff --git a/docs/evaluation/evidence/configured-onenine-agent-8844c21.json b/docs/evaluation/evidence/configured-onenine-agent-8844c21.json new file mode 100644 index 0000000..e36412b --- /dev/null +++ b/docs/evaluation/evidence/configured-onenine-agent-8844c21.json @@ -0,0 +1,114 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-configured-onboarded-agent-campaign", + "generated_at": "2026-09-01T11:59:09.113856+00:00", + "codemesh": { + "commit": "8844c21baf7aa6b5fcce2c535d510c2ad2788e71", + "clean_detached_checkout": true, + "one_step_context_package_entry_included": true + }, + "repository": { + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "commit": "fe2f761583bf78601961ff17934185c4b6a632f9", + "clean_detached_checkout": true, + "snapshot_id": "snp_bf026d3f2e2b5a40f3c9fa112ae36b7e026b0f5a0f0483a23796b0a6adc9243e", + "source_view_hash": "fea1951d1eeb93ac265112aef7000dcac1a9832b000ea81b360b0851df56e69b", + "restored_main_commit": "162fa515f7837efbc3a67372c5bd0d2e2d36b6fd" + }, + "configured_integration": { + "profile": "normal", + "plan_hash": "fcadac37ae4df261e8b746e14f9191e22a7b6399b991925d4d1e12a5947daf34", + "installation_file_sha256": "d2f6605df75dc5f93be6d8b0b7aea10b3754cb333ddd6d12187e84b262ed484c", + "guidance_sha256": "72f9a446e9e77886ba901f3e1a1eb28705230c8232ff42756920c23cc975cd79", + "baseline_mcp_servers": [], + "prompt_parity": true, + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "preflight_report_sha256": "15c75818160ccf47edbfeb7a5966043bcd4f206acef91510d403d51c61b69585" + }, + "campaign": { + "suite": "onenine-native-replay-configured-agent", + "provider": "default", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "completed_pair_count": 3, + "verdict": "neutral", + "harness_passed": true, + "treatment_wins": 0, + "codemesh_attributable_wins": 0, + "treatment_regressions": 0, + "treatment_safety_violations": 0, + "control_success_rate": 1.0, + "treatment_success_rate": 1.0, + "codemesh_adoption_rate": 1.0, + "codemesh_adopted_pair_count": 3, + "context_retrieval_call_count": 3, + "repository_status_call_count": 0, + "mcp_failed_call_count": 0, + "command_policy_violation_count": 0, + "median_control_duration_ms": 259055.847, + "median_treatment_duration_ms": 255449.851, + "duration_delta": -0.01392, + "median_control_tokens": 1784386.0, + "median_treatment_tokens": 1858085.0, + "token_delta": 0.041302, + "median_control_command_count": 22, + "median_treatment_command_count": 19, + "maximum_treatment_command_count": 110, + "report_sha256": "a76a94ff9caa50cfd52673c7d15dc9e65454129ca57f292ae54eb584fa26e0a5" + }, + "measured_correction_outcome": { + "one_step_context_package_entry_present": true, + "treatment_context_package_call_count": 3, + "context_handoff_observed_in_every_treatment": true, + "result": "produced_full_context_package_adoption" + }, + "comparison_to_retained_campaigns": { + "same_task_prompt_grading_control_model_and_repository_identity": true, + "prior_campaign_context_retrieval_call_count": 0, + "current_context_retrieval_call_count": 3, + "prior_verdicts": [ + "insufficient", + "insufficient" + ], + "current_verdict": "neutral" + }, + "claim_ceiling": { + "established": [ + "configured_prompt_parity_and_exact_binding", + "three_completed_correct_and_safe_pairs", + "context_package_adoption_in_every_treatment", + "one_step_entry_resolved_the_measured_adoption_gap", + "neutral_outcome_with_no_matched_regression" + ], + "not_established": [ + "codemesh_correctness_benefit", + "codemesh_efficiency_benefit", + "repeatable_configured_product_benefit", + "cross_repository_generalization", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "Both conditions passed every repetition, so the campaign produced no CodeMesh-attributable correctness win.", + "Median treatment duration was 1.39 percent lower and median tokens were 4.13 percent higher; both remain inside the 10 percent neutrality threshold.", + "One treatment used 110 shell commands while the other treatments used 19 each; the campaign median is retained, but the outlier remains diagnostic.", + "The campaign covers one answer task in one private repository with one model and three repetitions.", + "Raw events, final messages, workspaces, prompts, source excerpts, and credentials were not retained." + ], + "boundaries": { + "campaign_consumed": true, + "automatic_retry_authorized": false, + "config_net_provider_free_preparation": "eligible_next_work_subject_to_exact_identity_and_setup_review", + "config_net_model_campaign": "requires_separate_explicit_model_spend_authority", + "additional_onenine_checkout_onboarding": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-onenine-provider-free-0378740.json b/docs/evaluation/evidence/configured-onenine-provider-free-0378740.json new file mode 100644 index 0000000..c26f8a4 --- /dev/null +++ b/docs/evaluation/evidence/configured-onenine-provider-free-0378740.json @@ -0,0 +1,131 @@ +{ + "schema_version": "codemesh-configured-onenine-provider-free-v1", + "evidence_class": "provider-free-local-configured-integration", + "recorded_at": "2026-09-01T09:06:15Z", + "codemesh": { + "commit": "03787409bfb1bbbb53ba83ea1a3de4f3bf8f6fc9", + "clean_detached_checkout": true, + "status_to_context_correction_included": true, + "python_tests_passed": 125, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": { + "status": "passed_with_warning_and_skip", + "locked_restore": "passed", + "build_passed": true, + "build_warning_count": 1, + "tests_passed": 69, + "tests_failed": 0, + "tests_skipped": 1, + "skip_reason": "CODEMESH_SAMPLE_CSHARP_ROOT not configured for YoutubeDownloader sample; synthetic parser tests passed", + "format_verify_no_changes": true + }, + "repository_standards": { + "markdownlint": "passed", + "links": "passed", + "publication_safety": "passed", + "workflow_policy": "passed", + "diff_check": "passed" + } + }, + "onenine_primary": { + "commit": "fe2f761583bf78601961ff17934185c4b6a632f9", + "clean_detached_checkout_during_gates": true, + "restored_main_commit": "02971901b9c5f14269ebefc89bd670bf2a543a2e", + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "snapshot_id": "snp_bf026d3f2e2b5a40f3c9fa112ae36b7e026b0f5a0f0483a23796b0a6adc9243e", + "source_view_hash": "fea1951d1eeb93ac265112aef7000dcac1a9832b000ea81b360b0851df56e69b", + "freshness_status": "fresh" + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "profile": "normal", + "provider": "none", + "plan_hash": "a535de566f63be65c20bb2a01f398f6fcef8b4da571970bc573ced9a6809e248", + "installation_file_sha256": "7b8a9df8aec48670c5e4558de05fe5281d19b1173462259791e82c7fb3942807", + "guidance": { + "path": "AGENTS.override.md", + "source": "installed-ignored", + "sha256": "dbd0faca021b144f57594d91eb04b02a089fd6599234b713efe78693c92deb76" + }, + "expected_paths": [ + "app/feed_hub/native_replay.py", + "native/feed_hub_replay/src/lib.rs" + ], + "returned_path_count": 8, + "wrong_checkout_rejected": true, + "report_sha256": "ffc33889f61e284b07da8e1e9b35f1e2ab6ab92c4b67fbcc0a7fd2c499382416" + }, + "onenine_frozen_pilot": { + "passed": true, + "gate_count": 8, + "positive_gate_count": 4, + "rejection_gate_count": 4, + "feedback_packets_valid": 3, + "positive_report_sha256": [ + "358323fc78b1c721ac44665e0a717d3c22370cd08858342bacdafe669cc92ff5", + "1f40c5e5f49d2c2a15a5bf2a8fc69e4b789a4ec5e869b5e1945fffffaaafd7f5", + "20c53d821cf9522aa56b6efd55ac1ce9423819fe9b2a01523e14608b57f1156a", + "9f8688f07dd2bea243c124de8376290dd182af36e1573e1179e14ac341455670" + ], + "rejection_report_sha256": [ + "f72c3851c0f6f7cb5b4b83176f92499b654b6f39d565e9dea6de96b9e2657740", + "88e4c22275231c81bfcaee20ae08b08fdecdad1baa19c2bc19be3ca010a0a011", + "4fd7ca020a92614754195a601d691745d0b2fbbf1eea77668d2d84797ac53ccf", + "5d2434386cc7e3c2eb56de2439426adc49b060a9c49df6ff11bf400a03b06712" + ], + "feedback_validation_report_sha256": "7640482902c08935902527bd4ddc38ecc709a5a29d8385c4385151d9ac5318ad", + "feedback_summary_report_sha256": "572790433c75207000467344d66458be0727947d3d7c114c68be2137ae9a4b63" + }, + "youtube_downloader_live": { + "passed": true, + "comparable": true, + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "provider": "none", + "report_sha256": "2663476baa6e216d10f5f26dc5831cfd6af64f034191983b60dd1f59985a462d" + }, + "retained_prior_campaign": { + "codemesh_commit": "0bde606cc73b4a0b081067a039822a53672459a1", + "verdict": "insufficient", + "context_retrieval_call_count": 0, + "preserved_unchanged": true + }, + "claim_ceiling": { + "established": [ + "provider_free_status_to_context_candidate_verification", + "provider_free_configured_plan_and_runtime_identity", + "prompt_parity_preflight_configuration", + "frozen_onenine_retrieval_and_rejection_gates", + "comparable_youtube_downloader_retrieval_gate" + ], + "not_established": [ + "normal_agent_context_package_adoption", + "model_backed_correctness_or_efficiency_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "raw_prompts_or_payloads_retained": false, + "model_provider_invoked": false, + "model_backed_agent_evaluation": "not_run_requires_new_explicit_spend_authority", + "config_net_configured_replication": "not_run_requires_valid_onenine_model_campaign_first", + "secondary_tertiary_release_hotfix_onboarding": "not_authorized", + "run_only": "excluded", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-onenine-provider-free-0bde606.json b/docs/evaluation/evidence/configured-onenine-provider-free-0bde606.json new file mode 100644 index 0000000..ccd83f7 --- /dev/null +++ b/docs/evaluation/evidence/configured-onenine-provider-free-0bde606.json @@ -0,0 +1,150 @@ +{ + "schema_version": "codemesh-configured-onenine-provider-free-v1", + "evidence_class": "provider-free-local-configured-integration", + "recorded_at": "2026-08-31T22:10:21Z", + "last_updated_at": "2026-08-31T22:28:22Z", + "codemesh": { + "commit": "0bde606cc73b4a0b081067a039822a53672459a1", + "clean_detached_checkout": true, + "python_tests_passed": 124, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": { + "status": "passed_with_warning_and_skip", + "evidence_source": "attended_user_supplied_terminal_output", + "verification_commit": "9c8aca324d355a2fbb29c93fc6092bd553f05a53", + "candidate_dotnet_source_equivalent": true, + "candidate_dotnet_source_equivalence_basis": "no_sln_csproj_or_cs_diff_from_0bde606_to_verification_commit", + "locked_restore": "passed", + "build": { + "passed": true, + "warning_count": 1, + "warnings": [ + "tests/CodeMesh.Tests/Program.cs:4595 CS8625 null literal to non-nullable reference type" + ] + }, + "tests": { + "passed": 64, + "failed": 0, + "skipped": 1, + "skip_reason": "CODEMESH_SAMPLE_CSHARP_ROOT not configured for YoutubeDownloader sample; synthetic parser tests passed" + }, + "format_verify_no_changes": "passed_no_diagnostics", + "working_tree_clean_after": true + } + }, + "onenine_primary": { + "commit": "fe2f761583bf78601961ff17934185c4b6a632f9", + "clean_checkout": true, + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "snapshot_id": "snp_bf026d3f2e2b5a40f3c9fa112ae36b7e026b0f5a0f0483a23796b0a6adc9243e", + "source_view_hash": "fea1951d1eeb93ac265112aef7000dcac1a9832b000ea81b360b0851df56e69b", + "ingestion_run_id": "98b439c4b0d484297852dae7723be555e75bf8fcf7c565c96a9b0bad43b66062", + "parser_profile": "codemesh-parser-composite-v1[python=codemesh-parser-python;rust=codemesh-parser-rust]", + "node_count": 45650, + "relationship_count": 95218, + "content_count": 12873, + "embedding_count": 0, + "summary_count": 0, + "redacted_content_item_count": 324, + "freshness_status": "fresh" + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "profile": "normal", + "provider": "none", + "plan_hash": "6f5e68c225f88b6162f386e4c3bfb9ae1d974f70d56b04a50c142be7542c004e", + "installation_file_sha256": "f27c4060e9ae39f6d055e8fb7b052e0784a768ce61a79cb671cd19e71a755dfc", + "guidance": { + "path": "AGENTS.override.md", + "source": "installed-ignored", + "sha256": "dbd0faca021b144f57594d91eb04b02a089fd6599234b713efe78693c92deb76" + }, + "expected_paths": [ + "app/feed_hub/native_replay.py", + "native/feed_hub_replay/src/lib.rs" + ], + "returned_path_count": 8, + "wrong_checkout_rejected": true, + "rejection_issues": [ + "bound_repository_missing", + "snapshot_not_fresh", + "snapshot_provenance_unknown" + ], + "report_sha256": "3c50bec4131309ceaf65b7ebc90073fa2979841d71a9e9cd075a178194cdad19" + }, + "onenine_frozen_pilot": { + "passed": true, + "gate_count": 8, + "positive_gate_count": 4, + "rejection_gate_count": 4, + "feedback_packets_valid": 3, + "report_sha256": "d0294c8993bbab1e575dd66b899fe08d74733945e4999878e0ddaa20fd10b4bf" + }, + "youtube_downloader_live": { + "passed": true, + "comparable": true, + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "provider": "none", + "report_sha256": "ddd99511b894121f261bb5604d27f44e0424bbf2d731587ffb2e831779ae8e1e" + }, + "retained_attended_failures": [ + { + "codemesh_commit": "4eccd80fc25fe9d7cd07633735e0250a50c278cf", + "stage": "configured_positive_probe", + "result": "failed_missing_rust_expected_path", + "model_invoked": false + }, + { + "codemesh_commit": "a6ebaa56ee7f98c729d8f785671254fe80b61ccd", + "stage": "configured_wrong_checkout_probe", + "result": "failed_expected_diagnostic_did_not_match_live_issue_code", + "model_invoked": false + }, + { + "codemesh_commit": "854d6789965dcd818b69304a129f83d4a773c361", + "stage": "youtube_downloader_live", + "result": "failed_mean_mrr_threshold", + "mean_mrr": 0.766667, + "required_mean_mrr": 0.8, + "report_sha256": "70946a43c92b389f9521be3ab168869d5572cb8cf915fe62d0e7933b0efe5b16" + } + ], + "claim_ceiling": { + "established": [ + "provider_free_configured_plan_and_runtime_identity", + "prompt_parity_preflight_configuration", + "frozen_onenine_retrieval_and_rejection_gates", + "comparable_youtube_downloader_retrieval_gate" + ], + "not_established": [ + "normal_agent_codemesh_adoption", + "model_backed_correctness_or_efficiency_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "raw_prompts_or_payloads_retained": false, + "model_provider_invoked": false, + "model_backed_agent_evaluation": "not_run_requires_explicit_spend_authority", + "config_net_configured_replication": "not_run_requires_valid_onenine_model_campaign_first", + "secondary_tertiary_release_hotfix_onboarding": "not_authorized", + "run_only": "excluded", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-onenine-provider-free-8844c21.json b/docs/evaluation/evidence/configured-onenine-provider-free-8844c21.json new file mode 100644 index 0000000..daff3a0 --- /dev/null +++ b/docs/evaluation/evidence/configured-onenine-provider-free-8844c21.json @@ -0,0 +1,140 @@ +{ + "schema_version": "codemesh-configured-onenine-provider-free-v1", + "evidence_class": "provider-free-local-configured-integration", + "recorded_at": "2026-09-01T11:05:42Z", + "codemesh": { + "commit": "8844c21baf7aa6b5fcce2c535d510c2ad2788e71", + "clean_detached_checkout": true, + "one_step_context_package_entry_included": true, + "status_next_action_compatibility_retained": true, + "python_tests_passed": 125, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": { + "status": "passed_with_warning_and_skip", + "locked_restore": "passed", + "build_passed": true, + "build_warning_count": 1, + "tests_passed": 69, + "tests_failed": 0, + "tests_skipped": 1, + "skip_reason": "CODEMESH_SAMPLE_CSHARP_ROOT not configured for YoutubeDownloader sample; synthetic parser tests passed", + "format_verify_no_changes": true, + "retained_diagnostic": "The first attended service-backed attempt encountered a Neo4j cold-start cancellation; the initialized-store rerun passed and is the accepted gate." + }, + "repository_standards": { + "markdownlint": "passed", + "links": "passed", + "publication_safety": "passed", + "workflow_policy": "passed", + "diff_check": "passed" + } + }, + "onenine_instance_configuration": { + "commit": "f9f199d568d1450690459075dc0be38c8155a0c1", + "primary_binding_updated": true, + "all_generated_instance_checks_passed": true, + "additional_instances_activated": false + }, + "onenine_primary": { + "commit": "fe2f761583bf78601961ff17934185c4b6a632f9", + "clean_detached_checkout_during_gates": true, + "restored_main_commit": "162fa515f7837efbc3a67372c5bd0d2e2d36b6fd", + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "snapshot_id": "snp_bf026d3f2e2b5a40f3c9fa112ae36b7e026b0f5a0f0483a23796b0a6adc9243e", + "source_view_hash": "fea1951d1eeb93ac265112aef7000dcac1a9832b000ea81b360b0851df56e69b", + "freshness_status": "fresh" + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "profile": "normal", + "provider": "none", + "plan_hash": "fcadac37ae4df261e8b746e14f9191e22a7b6399b991925d4d1e12a5947daf34", + "installation_file_sha256": "d2f6605df75dc5f93be6d8b0b7aea10b3754cb333ddd6d12187e84b262ed484c", + "guidance": { + "path": "AGENTS.override.md", + "source": "installed-ignored", + "sha256": "72f9a446e9e77886ba901f3e1a1eb28705230c8232ff42756920c23cc975cd79" + }, + "expected_paths": [ + "app/feed_hub/native_replay.py", + "native/feed_hub_replay/src/lib.rs" + ], + "returned_path_count": 8, + "wrong_checkout_rejected": true, + "report_sha256": "15c75818160ccf47edbfeb7a5966043bcd4f206acef91510d403d51c61b69585" + }, + "onenine_frozen_pilot": { + "passed": true, + "gate_count": 8, + "positive_gate_count": 4, + "rejection_gate_count": 4, + "rejection_issue_codes": [ + "bound_repository_missing", + "repository_root_mismatch", + "source_view_hash_mismatch", + "snapshot_not_fresh" + ], + "feedback_packets_valid": 3, + "report_sha256": "2697a5e927ce473951b683fc06a28f7907b11af78841f92ea97c0d52a4afc227" + }, + "youtube_downloader_live": { + "passed": true, + "comparable": true, + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "provider": "none", + "report_sha256": "254f442da4fba50cfcf93d053aa076ecb4fb15e4cd7ce75246db639ac6ddd7c2" + }, + "retained_prior_campaigns": [ + { + "codemesh_commit": "0bde606cc73b4a0b081067a039822a53672459a1", + "verdict": "insufficient", + "context_retrieval_call_count": 0, + "preserved_unchanged": true + }, + { + "codemesh_commit": "03787409bfb1bbbb53ba83ea1a3de4f3bf8f6fc9", + "verdict": "insufficient", + "context_retrieval_call_count": 0, + "preserved_unchanged": true + } + ], + "claim_ceiling": { + "established": [ + "provider_free_one_step_entry_candidate_verification", + "provider_free_configured_plan_and_runtime_identity", + "prompt_parity_preflight_configuration", + "frozen_onenine_retrieval_and_rejection_gates", + "comparable_youtube_downloader_retrieval_gate" + ], + "not_established": [ + "normal_agent_context_package_adoption", + "model_backed_correctness_or_efficiency_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "raw_prompts_or_payloads_retained": false, + "model_provider_invoked": false, + "model_backed_agent_evaluation": "not_run_requires_new_explicit_spend_authority", + "config_net_configured_replication": "not_run_requires_valid_onenine_model_campaign_first", + "secondary_tertiary_release_hotfix_onboarding": "not_authorized", + "run_only": "excluded", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-onenine-stale-rejection-9f3216c.json b/docs/evaluation/evidence/configured-onenine-stale-rejection-9f3216c.json new file mode 100644 index 0000000..30fc46c --- /dev/null +++ b/docs/evaluation/evidence/configured-onenine-stale-rejection-9f3216c.json @@ -0,0 +1,52 @@ +{ + "schema_version": "codemesh-configured-preflight-rejection-v1", + "evidence_class": "provider-free-live-rejection", + "recorded_at": "2026-08-31T19:45:46Z", + "passed": true, + "codemesh": { + "commit": "9f3216cc9f32a187b7e4cda64f882cd4d84a0339", + "clean_checkout": true, + "python_tests_passed": 121, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5 + }, + "onenine_primary": { + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "configured_suite_commit": "499ae2630e5c8c31a4a7904f91a67eacc6ba5621", + "indexed_commit": "e12d0281b687e58c2baac85f771a684df6fa8552", + "current_commit": "499ae2630e5c8c31a4a7904f91a67eacc6ba5621", + "working_tree_dirty": false, + "indexed_source_view_hash": "ce3c63a66e3e4821bf5431d9997314070d60ec61f9ab8b35aa0cdcafbacace4a", + "freshness_status": "stale", + "rejection_class": "stale-index", + "configured_preflight_rejected": true + }, + "instance_registry": { + "repository_commit": "515af59b94926f2dea3e1a5b3d28b0316217c554", + "pinned_primary_commit": "e12d0281b687e58c2baac85f771a684df6fa8552", + "pinned_codemesh_commit": "ee1d4d9306a5fce2322d624ae17648c82ff0918a", + "detached_codemesh_candidate_role": "default oneNine CodeMesh candidate worktree", + "registry_matches_active_primary": false, + "registry_contains_configured_evaluator_candidate": false + }, + "diagnostic_contract": { + "reported_status": true, + "reported_indexed_commit": true, + "reported_current_commit": true, + "reported_dirty_state": true, + "reported_indexed_source_view_hash": true + }, + "boundaries": { + "model_provider_invoked": false, + "installation_plan_loaded": false, + "reindex_or_repoint_onenine": "not_performed", + "candidate_or_binding_registry_repin": "not_performed_requires_explicit_authority", + "model_backed_agent_evaluation": "not_run_requires_explicit_spend_authority", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-runtime-verification-a158bc0.json b/docs/evaluation/evidence/configured-runtime-verification-a158bc0.json new file mode 100644 index 0000000..28459f0 --- /dev/null +++ b/docs/evaluation/evidence/configured-runtime-verification-a158bc0.json @@ -0,0 +1,217 @@ +{ + "schema_version": "codemesh-configured-runtime-verification-v1", + "recorded_at": "2026-09-05", + "candidate": "a158bc0308d9fdb8f373ec0d9d6f03233b984f9f", + "evidence_class": "local-installed-cli-and-configured-source-backed-mcp-runtime", + "private_archive": ".codemesh-evals/review-a158bc0 in main checkout", + "source_and_target_clean_before_and_after": true, + "installation_plan_hash": "67e9668b170d75016348f83574f011fc1e9638a543b00501ca778f4aa55ddfc4", + "installation_files": [ + { + "path": ".codex/config.toml", + "previous_sha256": null, + "new_sha256": "644b30d73ced8b948ce2f7544f6fb29d554483eb847059b8cca07b312b46427f" + }, + { + "path": "AGENTS.md", + "previous_sha256": null, + "new_sha256": "bd356195afbb8b945d003ff7b6802e270a8c6a97566697ce95b1c4a7c980aba3" + } + ], + "installation_review": "Full launch, binding, file contents and hashes reviewed before exact-plan application. Only two previously absent files in the isolated sample were written.", + "operator_cli": "Installed wheel from the verified a158bc0 package archive, used outside the source checkout.", + "mcp_launch": "Unmodified shipped uv source-backed launch from a separate clean a158bc0 checkout. This does not claim the server ran from the installed wheel.", + "target": "Separate frozen YoutubeDownloader clone at 05e63cddb6d2a96fc2d21b097129a0765824f251; local Git info/exclude keeps task-only configuration/onboarding outside source status.", + "images": [ + { + "service": "neo4j", + "image_id": "sha256:362542416de6c09a971484d1893878016cc3b5cdec166e54b1c824a220ecd6b9" + }, + { + "service": "mongodb", + "image_id": "sha256:340c1c56fb10e95cf79ff547f8664b96bc6ead9909bc355238cbf865a9695a6f" + }, + { + "service": "qdrant", + "image_id": "sha256:0bd98fa7977f1e75694779359ca4e212822e5a71334e28421182f72f209d5286" + }, + { + "service": "agent-access", + "image_id": "sha256:0ee08cb249a387e639840dda0b3327b975dd768f2771ede7025127eab89c7806" + } + ], + "reports": { + "runtime-positive.json": { + "binding_status": "accepted", + "checkout_id": "chk_7bd6f1df64644128a714bc9b3de7caa7", + "context_item_count": 8, + "context_query": "download filename template apply", + "current_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "duration_ms": 3086.595, + "expected_paths": [ + "YoutubeDownloader.Core/Downloading/FileNameTemplate.cs" + ], + "freshness_status": "fresh", + "indexed_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "passed": true, + "profile": "normal", + "project_id": "prj_c4c08e04f9ea4c7b903bc7f4be604f9d", + "provider_mode": "none", + "returned_paths": [ + "YoutubeDownloader.Core/Downloading/FFmpeg.cs", + "YoutubeDownloader.Core/Downloading/FileNameTemplate.cs", + "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs", + "YoutubeDownloader/Framework/ViewManager.cs", + "YoutubeDownloader/Localization/LocalizationManager.Spanish.cs", + "YoutubeDownloader/Services/SettingsService.AuthCookiesEncryptionConverter.cs", + "YoutubeDownloader/ViewModels/Components/DownloadViewModel.cs", + "YoutubeDownloader/ViewModels/Dialogs/SettingsViewModel.cs" + ], + "schema_version": "codemesh-runtime-probe-v1", + "server_name": "codemesh", + "snapshot_id": "snp_2e655b441a3e8a7bf8dc174b7a45af0c8cf8240050203762f4fdf725e1256ea0", + "source_view_hash": "7def3a130907855efb58f34158ce84c2420275456e2f1810ebed040ccd3e0425", + "tools": [ + "codemesh_get_context_package", + "codemesh_get_node", + "codemesh_get_repository_status", + "codemesh_list_repositories" + ] + }, + "rejection-source-view.json": { + "checkout_id": "chk_7bd6f1df64644128a714bc9b3de7caa7", + "diagnostic": "Indexed commit matches the local checkout and the working tree is clean. Configured repository binding or freshness was rejected: source_view_hash_mismatch", + "duration_ms": 1183.604, + "expected_substrings": [ + "source_view_hash_mismatch" + ], + "passed": true, + "profile": "normal", + "project_id": "prj_c4c08e04f9ea4c7b903bc7f4be604f9d", + "provider_mode": "none", + "rejected": true, + "rejection_issues": [ + "source_view_hash_mismatch" + ], + "schema_version": "codemesh-runtime-rejection-probe-v1", + "source_view_hash": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "rejection-unknown-checkout.json": { + "checkout_id": "chk_00000000000000000000000000000000", + "diagnostic": "Repository was not found in the CodeMesh registry. Configured repository binding or freshness was rejected: bound_repository_missing, snapshot_not_fresh, snapshot_provenance_unknown", + "duration_ms": 1160.919, + "expected_substrings": [ + "bound_repository_missing" + ], + "passed": true, + "profile": "normal", + "project_id": "prj_c4c08e04f9ea4c7b903bc7f4be604f9d", + "provider_mode": "none", + "rejected": true, + "rejection_issues": [ + "bound_repository_missing", + "snapshot_not_fresh", + "snapshot_provenance_unknown" + ], + "schema_version": "codemesh-runtime-rejection-probe-v1", + "source_view_hash": "7def3a130907855efb58f34158ce84c2420275456e2f1810ebed040ccd3e0425" + }, + "rejection-dirty.json": { + "checkout_id": "chk_7bd6f1df64644128a714bc9b3de7caa7", + "diagnostic": "Indexed commit, checkout state, or source-view provenance does not match a verifiable clean checkout. Configured repository binding or freshness was rejected: snapshot_not_fresh", + "duration_ms": 1179.081, + "expected_substrings": [ + "snapshot_not_fresh" + ], + "passed": true, + "profile": "normal", + "project_id": "prj_c4c08e04f9ea4c7b903bc7f4be604f9d", + "provider_mode": "none", + "rejected": true, + "rejection_issues": [ + "snapshot_not_fresh" + ], + "schema_version": "codemesh-runtime-rejection-probe-v1", + "source_view_hash": "7def3a130907855efb58f34158ce84c2420275456e2f1810ebed040ccd3e0425" + }, + "runtime-restored.json": { + "binding_status": "accepted", + "checkout_id": "chk_7bd6f1df64644128a714bc9b3de7caa7", + "context_item_count": 8, + "context_query": "download filename template apply", + "current_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "duration_ms": 1180.876, + "expected_paths": [ + "YoutubeDownloader.Core/Downloading/FileNameTemplate.cs" + ], + "freshness_status": "fresh", + "indexed_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "passed": true, + "profile": "normal", + "project_id": "prj_c4c08e04f9ea4c7b903bc7f4be604f9d", + "provider_mode": "none", + "returned_paths": [ + "YoutubeDownloader.Core/Downloading/FFmpeg.cs", + "YoutubeDownloader.Core/Downloading/FileNameTemplate.cs", + "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs", + "YoutubeDownloader/Framework/ViewManager.cs", + "YoutubeDownloader/Localization/LocalizationManager.Spanish.cs", + "YoutubeDownloader/Services/SettingsService.AuthCookiesEncryptionConverter.cs", + "YoutubeDownloader/ViewModels/Components/DownloadViewModel.cs", + "YoutubeDownloader/ViewModels/Dialogs/SettingsViewModel.cs" + ], + "schema_version": "codemesh-runtime-probe-v1", + "server_name": "codemesh", + "snapshot_id": "snp_2e655b441a3e8a7bf8dc174b7a45af0c8cf8240050203762f4fdf725e1256ea0", + "source_view_hash": "7def3a130907855efb58f34158ce84c2420275456e2f1810ebed040ccd3e0425", + "tools": [ + "codemesh_get_context_package", + "codemesh_get_node", + "codemesh_get_repository_status", + "codemesh_list_repositories" + ] + } + }, + "artifact_hashes": [ + { + "file": "runtime-positive.json", + "sha256": "a831ad72c70c08da6a5390ce7582579ea279deadbc38751f9b459335654f9567" + }, + { + "file": "rejection-source-view.json", + "sha256": "a990ae2f69075fb41dfd3ead037df651c8d90a7d7deecf8091db795632397046" + }, + { + "file": "rejection-unknown-checkout.json", + "sha256": "a113ced932bae3551c0ea35d00c875de25b07ae2a851dc03a043bb6362dc780c" + }, + { + "file": "rejection-dirty.json", + "sha256": "bb7e5f0bf381fb905ba0920cb26706fb8ad72ec998220ccadc472d3bdca777ac" + }, + { + "file": "runtime-restored.json", + "sha256": "013ce9c8b84bb749e911fde2225f7de90faf90f859b2b1c76c045d287416077d" + }, + { + "file": "installation-plan.json", + "sha256": "f16d552134d1fa46fd408c0eb1b5c4809b73f0b662203ac35fdd2215f3fc15af" + }, + { + "file": "installation-apply.json", + "sha256": "8b407185095ed2f18809e7d6cfe086795121261b737ef957634c34479100981f" + }, + { + "file": "ingest.json", + "sha256": "5e335591e495129dd4647e377c2bbab5808861051e3f27f14d41d3de8a1e1a90" + } + ], + "retained_failed_assertion": "The first unknown-checkout probe expected the word checkout, but the server correctly rejected the missing bound repository. Original output is retained; a new probe passed against the observed bound_repository_missing diagnostic. No unsafe acceptance occurred.", + "cleanup": "All task containers and network removed; source bytes restored and both source/target checkouts clean; isolated volumes and ignored configuration preserved.", + "limits": [ + "Provider-free configured runtime evidence only; no agent/model campaign or product-benefit claim.", + "Frozen sample dependency advisory remains open; restored assets were reused with parser advisory warnings retained.", + "The plan was applied only to a task-owned sample, not any one|nine or user working checkout.", + "No push, PR, merge, tag, publication or deployment was performed." + ] +} diff --git a/docs/evaluation/evidence/configured-youtube-downloader-agent-0f18071.json b/docs/evaluation/evidence/configured-youtube-downloader-agent-0f18071.json new file mode 100644 index 0000000..dae3641 --- /dev/null +++ b/docs/evaluation/evidence/configured-youtube-downloader-agent-0f18071.json @@ -0,0 +1,174 @@ +{ + "schema_version": "codemesh-youtube-downloader-capped-agent-v1", + "evidence_class": "incomplete-configured-onboarded-hard-cap-campaign", + "recorded_at": "2026-09-01T18:54:31.112867+00:00", + "codemesh": { + "candidate_commit": "0f180713407e4852c81232b87b234307097c2b20", + "clean_candidate_before_campaign": true, + "instrumentation_commit": "66033addceb9537566fb67567e50ae1f21a55eea", + "runner_commit": "0f180713407e4852c81232b87b234307097c2b20", + "hydration_optimization_applied": false + }, + "youtube_downloader": { + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "branch": "HEAD", + "tracked_files_changed": false, + "project_id": "prj_3425db238d0d4a5fbcc1e3c9a28d1945", + "checkout_id": "chk_7e4ecde2ae4e45b4b9e475e0a587aa47", + "snapshot_id": "snp_c2aa867ffd967d7be6305ba8b051cfafadc8351421309772359d7c23ad24ae2f", + "source_view_hash": "7def3a130907855efb58f34158ce84c2420275456e2f1810ebed040ccd3e0425", + "freshness_status": "fresh", + "index_refreshed": false + }, + "frozen_suites": { + "adaptive_sha256": "40b4d994ee704c7db22a973538c0a34f3bb28b6cab305857d553f2928f392ea8", + "impact_sha256": "b848b252a87713303e11f467bb0f5ef2e24178228141507c215acc958a11443a", + "canonical_sha256": "6b9d41c1bc4e5f002f8cdfb87d3745df3deb802b85a3b9a81cbd470ba73c0be6", + "configured_agent_sha256": "a3fd22c007ef9eaf8506dff231ce9035bdf26bfb74611e2258b56c764494d3d4" + }, + "deterministic_checks": { + "python_tests_passed": 149, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "repository_standards": "passed", + "dotnet_verification": "not_selected_python_only_contract_and_evaluation_change" + }, + "configured_preflight": { + "passed": true, + "provider": "none", + "runner": "capped-codex", + "hard_cap_capability_verified": true, + "prompt_parity": true, + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "plan_hash": "6ff0e3cd1ceef16a18c98daa7371c025ccb4e5576100d4deffe2d43c9e5437ec", + "plan_report_sha256": "6b376bc95b0055769416b354690c68ce3fdd0129a63e479229472c4771144dfb", + "guidance_sha256": "bd356195afbb8b945d003ff7b6802e270a8c6a97566697ce95b1c4a7c980aba3", + "report_sha256": "d790fc26a0e10b3d9f8548d36237cd2e192478491920575974b0f066efd5464a" + }, + "provider_free_runner_compatibility": { + "installed_codex_version": "0.151.0", + "selected_model": "gpt-5.6-sol", + "synthetic_loopback_completed": true, + "request_count": 1, + "completed_request_count": 1, + "request_max_retries": 0, + "stream_max_retries": 0, + "reported_tokens": 101, + "retained_reservation_tokens": 0, + "codex_proxy_usage_agreed": true, + "model_catalog_sha256": "e140f868e626bcbfbcff765ea87c00adf0658029fc00d4425f8b0a56c82565bb", + "provider_invoked": false + }, + "provider_free_gates": { + "adaptive_live": { + "passed": true, + "comparable": true, + "case_count": 5, + "tool_call_count": 25, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.875, + "mean_ndcg_at_k": 0.862253, + "secret_leaks": 0, + "report_sha256": "1aedf6677da02c2dfb870113465ce7228e96054579ad31ba92d646ba881a2f5c" + }, + "impact_live": { + "passed": true, + "comparable": true, + "case_count": 2, + "tool_call_count": 6, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 0.891323, + "secret_leaks": 0, + "report_sha256": "6043c3df127448985a31ef17c606c5fa4324255b89efbc04df88fcb27a67d39b" + }, + "canonical_live": { + "passed": true, + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "report_sha256": "6f20bb3142eab34b81b58ca52de8001e4417a2f6efa2a7752098ea00635564d8" + } + }, + "timing_attribution": { + "evidence_is_separate_from_retrieval_claims": true, + "task_shaped_total_p50_ms": 16740.163, + "task_shaped_search_p50_ms": 16648.307, + "task_shaped_hydration_p50_ms": 88.87, + "hydration_share": 0.005309, + "hydration_dominance_gate_passed": false, + "decision": "retain_diagnosis_and_do_not_apply_hydration_concurrency" + }, + "authorized_campaign": { + "suite": "youtube-downloader-impact-configured-agent", + "runner": "capped-codex", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "requested_repetitions": 3, + "reported_token_ceiling": 1600000, + "automatic_retries": false, + "verdict": "insufficient", + "harness_passed": false, + "completed_pair_count": 0, + "execution_count": 1, + "completed_execution_count": 0, + "stopped_after_incomplete_execution": true, + "provider_failure": "upstream_generation_rejected_for_unavailable_credits", + "aggregate_completed_response_usage": 0, + "aggregate_usage_is_not_a_spend_claim": true, + "report_sha256": "1c45cba02fad306e1167197b87a73fa4a6e66187c7306cd8c11712eb9ca47bcf" + }, + "hard_cap_failure_ledger": { + "accounting": "input-output-reasoning-v1", + "ceiling": 1600000, + "committed_tokens": 267528, + "reported_tokens": 0, + "retained_reservation_tokens": 267528, + "remaining_tokens": 1332472, + "request_count": 1, + "completed_request_count": 0, + "request_max_retries": 0, + "stream_max_retries": 0, + "complete": false, + "failure": "stream_ended_without_completed_usage_and_reservation_was_retained", + "subsequent_request_admitted": false + }, + "claim_ceiling": { + "established": [ + "exact_candidate_provider_free_gates_passed", + "configured_positive_and_rejection_probes_passed", + "capped_runner_failed_closed_on_interrupted_provider_response", + "ambiguous_reservation_was_retained", + "no_retry_or_top_up_occurred" + ], + "not_established": [ + "a_completed_control_treatment_pair", + "agent_adoption_or_task_correctness", + "codemesh_efficiency_or_correctness_benefit", + "provider_spend_from_missing_usage", + "repeatable_product_benefit", + "causality", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "campaign_consumed": true, + "retry_authorized": false, + "top_up_authorized": false, + "additional_model_spend_authorized": false, + "raw_traces_retained": false, + "push": "not_performed", + "release": "not_performed", + "deployment": "not_performed" + } +} diff --git a/docs/evaluation/evidence/configured-youtube-downloader-agent-64d4052.json b/docs/evaluation/evidence/configured-youtube-downloader-agent-64d4052.json new file mode 100644 index 0000000..e7cf691 --- /dev/null +++ b/docs/evaluation/evidence/configured-youtube-downloader-agent-64d4052.json @@ -0,0 +1,130 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-configured-onboarded-agent-campaign", + "generated_at": "2026-09-01T15:18:22.348223+00:00", + "codemesh": { + "commit": "64d405282ffcb96207479dd46eab9947a0eb7aaf", + "clean_detached_checkout": true, + "configured_suite": "youtube-downloader-impact-configured-agent", + "focused_query_guidance": true + }, + "repository": { + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "project_id": "prj_3425db238d0d4a5fbcc1e3c9a28d1945", + "checkout_id": "chk_7e4ecde2ae4e45b4b9e475e0a587aa47", + "commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "branch": "HEAD", + "tracked_files_changed": false, + "snapshot_id": "snp_1c7772010bcb7c8430fe062d975094271762a68627dbc53f4e2b9f0557687eae", + "source_view_hash": "d741906fcdcc5aa33a9d14f8a5e0cbdf7d2e572266a098beadddb595ce6853d9" + }, + "configured_integration": { + "profile": "normal", + "plan_hash": "c0e6a2e09d785cea4c1b4f33ac37a3b43cdb683832390e74dceea7508c98dc92", + "plan_report_sha256": "ddda6adc58636291bcbe70f059027e18a00c97d8a87195f39a1e1ae301017527", + "installation_file_sha256": "9bb852f8ee26992009a8a2f1f709e4db2b95f42bf826df78e5efa66ee84275e8", + "guidance_sha256": "abb5fcdce32a53d14c618b18e842f0070e83fe9061e52d61a380d3a3086dd371", + "baseline_mcp_servers": [], + "prompt_parity": true, + "evaluator_only_treatment_guidance": false, + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "preflight_report_sha256": "534f83ae9fb14834a8170ea2cfcebfdb9907bdd98dca16ab602a0a95111c28ba" + }, + "campaign": { + "suite": "youtube-downloader-impact-configured-agent", + "provider": "default", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "completed_pair_count": 3, + "all_runs_completed": true, + "verdict": "regressed", + "harness_passed": false, + "harness_failure_count": 0, + "treatment_wins": 0, + "codemesh_attributable_wins": 0, + "treatment_regressions": 0, + "treatment_safety_violations": 0, + "control_success_rate": 1.0, + "treatment_success_rate": 1.0, + "codemesh_adoption_rate": 1.0, + "codemesh_adopted_pair_count": 3, + "context_package_call_attempt_count": 7, + "context_package_call_attempts_per_treatment": [ + 2, + 2, + 3 + ], + "mcp_failed_call_count": 0, + "command_policy_violation_count": 0, + "matched_successful_pair_count": 3, + "matched_control_duration_ms": 144330.561, + "matched_treatment_duration_ms": 216190.176, + "matched_duration_delta": 0.497882, + "matched_control_tokens": 592729.0, + "matched_treatment_tokens": 835770.0, + "matched_token_delta": 0.410037, + "report_sha256": "0d06bff166a4049364abd29fe08bb079a19e4390d1499fca9f172c4d424aea16" + }, + "control_failures": [], + "treatment_failures": [], + "measured_outcome": { + "context_package_adopted_in_every_treatment": true, + "multiple_context_package_attempts_in_every_treatment": true, + "all_controls_and_treatments_correct_and_safe": true, + "correctness_result": "no_treatment_win_or_regression", + "efficiency_result": "regressed_beyond_the_ten_percent_threshold_on_all_three_matched_pairs" + }, + "comparison_to_retained_assisted_campaign": { + "assisted_candidate": "6e8cc6bc2569c79ff19430e2520c51883d4a1591", + "assisted_prompt_parity": false, + "assisted_context_package_attempts_per_treatment": [ + 1, + 1, + 1 + ], + "assisted_duration_delta": -0.34699, + "assisted_token_delta": -0.645419, + "configured_prompt_parity": true, + "configured_duration_delta": 0.497882, + "configured_token_delta": 0.410037, + "direct_performance_comparison_valid": false + }, + "claim_ceiling": { + "established": [ + "configured_prompt_parity_and_exact_binding", + "context_package_adoption_in_every_treatment", + "all_controls_and_treatments_correct_and_safe", + "zero_treatment_correctness_regressions", + "efficiency_regression_across_three_matched_pairs", + "configured_adoption_and_correctness_replicated_on_an_independent_csharp_repository" + ], + "not_established": [ + "causality_of_focused_query_guidance", + "repeatable_configured_product_benefit", + "exact_context_package_query_wording", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "The campaign covers one answer task, one repository, one model, and three repetitions.", + "Raw traces, context-package arguments and payloads, final messages, and temporary workspaces were not retained.", + "The report proves two or three context-package attempts in every treatment but not the exact query wording used.", + "The earlier assisted campaign used a different evidence class, candidate, and treatment-only guidance, so its favorable efficiency result is contextual rather than directly comparable." + ], + "boundaries": { + "campaign_consumed": true, + "automatic_retry_authorized": false, + "additional_model_spend": "not_authorized", + "next_work": "provider_free_task_adaptive_query_strategy_diagnosis", + "additional_onenine_checkout_onboarding": "not_authorized", + "push": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/configured-youtube-downloader-agent-fbc1433.json b/docs/evaluation/evidence/configured-youtube-downloader-agent-fbc1433.json new file mode 100644 index 0000000..8a0001c --- /dev/null +++ b/docs/evaluation/evidence/configured-youtube-downloader-agent-fbc1433.json @@ -0,0 +1,153 @@ +{ + "schema_version": "codemesh-youtube-downloader-task-adaptive-configured-agent-v1", + "evidence_class": "configured-onboarded-prompt-parity-agent", + "recorded_at": "2026-09-01T16:26:38.929368+00:00", + "codemesh": { + "commit": "fbc1433642ad9e20acd3379554f2ee09a6d635d9", + "clean_detached_checkout": true, + "configured_suite": "youtube-downloader-impact-configured-agent", + "task_adaptive_guidance": true + }, + "youtube_downloader": { + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "branch": "HEAD", + "tracked_files_changed": false, + "project_id": "prj_3425db238d0d4a5fbcc1e3c9a28d1945", + "checkout_id": "chk_7e4ecde2ae4e45b4b9e475e0a587aa47", + "snapshot_id": "snp_c2aa867ffd967d7be6305ba8b051cfafadc8351421309772359d7c23ad24ae2f", + "source_view_hash": "7def3a130907855efb58f34158ce84c2420275456e2f1810ebed040ccd3e0425", + "freshness_status": "fresh" + }, + "authorization": { + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "automatic_retries": false, + "reported_token_ceiling": 1600000, + "aggregate_reported_tokens": 2596056, + "aggregate_uncached_tokens": 376792, + "aggregate_cached_input_tokens": 2219264, + "reported_token_ceiling_met": false, + "boundary_failure": "The evaluator had no hard aggregate-token stop and the six fixed executions exceeded the authorized reported-token ceiling. No retry or additional campaign was run." + }, + "deterministic_checks": { + "python_tests_passed": 125, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": "not_selected_python_guidance_and_evaluation_suite_change" + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "provider": "none", + "plan_hash": "ae7bedc3b62532e2c359f60f64fae4b221231ce733a712d53e7c4deecf33c65a", + "guidance_sha256": "bd356195afbb8b945d003ff7b6802e270a8c6a97566697ce95b1c4a7c980aba3", + "positive_probe_passed": true, + "wrong_checkout_rejected": true, + "report_sha256": "bad129ed0ba3609adfb7ff0796bc3940509cb616ae527fb55ae3677e00a0648f" + }, + "provider_free_gates": { + "adaptive_live": { + "passed": true, + "comparable": true, + "case_count": 5, + "tool_call_count": 15, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.875, + "mean_ndcg_at_k": 0.862253, + "secret_leaks": 0, + "report_sha256": "5cfe98e11cbb2c03ea6e1c1a3a7b7459d217f67fec828b940db833abec845746" + }, + "impact_live": { + "passed": true, + "comparable": true, + "case_count": 2, + "tool_call_count": 6, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 0.891323, + "secret_leaks": 0, + "report_sha256": "094281adcfcdd37b95b84517d0f1148cbb582c3b312ef21b522ac1f5782d06b0" + }, + "canonical_live": { + "passed": true, + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "report_sha256": "6c3828337af8d4ca83269ed164630e7d527df593edefb25ddd2f6abbbd792fb4" + } + }, + "campaign": { + "suite": "youtube-downloader-impact-configured-agent", + "provider": "default", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium", + "seed": 42, + "repetitions": 3, + "pair_count": 3, + "completed_pair_count": 3, + "all_runs_completed": true, + "all_runs_passed": true, + "verdict": "regressed", + "harness_passed": false, + "harness_failure_count": 0, + "treatment_wins": 0, + "codemesh_attributable_wins": 0, + "treatment_regressions": 0, + "treatment_safety_violations": 0, + "control_success_rate": 1.0, + "treatment_success_rate": 1.0, + "codemesh_adoption_rate": 1.0, + "codemesh_adopted_pair_count": 3, + "context_package_call_attempt_count": 3, + "context_package_call_attempts_per_treatment": [1, 1, 1], + "mcp_failed_call_count": 0, + "command_policy_violation_count": 0, + "median_control_duration_ms": 136218.474, + "median_treatment_duration_ms": 149866.42, + "duration_delta": 0.100192, + "median_control_tokens": 420798.0, + "median_treatment_tokens": 392904.0, + "token_delta": -0.066288, + "raw_traces_retained": false, + "report_sha256": "2249a0b2251b93aaf138ee31dbcd46cb8ac899f4defb95f988b27c6d25d9165b" + }, + "claim_ceiling": { + "established": [ + "configured_prompt_parity_and_exact_runtime_identity", + "single_context_package_adoption_in_all_three_treatments", + "correct_and_safe_completion_in_all_six_runs", + "no_mcp_or_command_policy_failures", + "bounded_token_efficiency_improvement_with_duration_regression" + ], + "not_established": [ + "exact_agent_query_arguments", + "codemesh_attributable_correctness_win", + "overall_efficiency_benefit", + "causality_for_duration_or_token_deltas", + "repeatable_configured_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "campaign_consumed": true, + "automatic_retry_authorized": false, + "additional_model_spend_authorized": false, + "reported_token_ceiling_breached": true, + "raw_prompts_or_traces_retained": false, + "push": "not_performed", + "release": "not_performed", + "deployment": "not_performed" + } +} diff --git a/docs/evaluation/evidence/configured-youtube-downloader-provider-free-64d4052.json b/docs/evaluation/evidence/configured-youtube-downloader-provider-free-64d4052.json new file mode 100644 index 0000000..9a7713d --- /dev/null +++ b/docs/evaluation/evidence/configured-youtube-downloader-provider-free-64d4052.json @@ -0,0 +1,122 @@ +{ + "schema_version": "codemesh-configured-youtube-downloader-provider-free-v1", + "evidence_class": "provider-free-local-configured-integration", + "recorded_at": "2026-09-01T14:59:34Z", + "codemesh": { + "commit": "64d405282ffcb96207479dd46eab9947a0eb7aaf", + "clean_detached_checkout": true, + "configured_suite_added": "youtube-downloader-impact-configured-agent", + "frozen_prompt_targets_thresholds_and_repository_commit_preserved": true, + "python_tests_passed": 125, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": "not_selected_evaluation_fixture_only_change" + }, + "youtube_downloader": { + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "branch": "HEAD", + "tracked_files_changed": false, + "project_id": "prj_3425db238d0d4a5fbcc1e3c9a28d1945", + "checkout_id": "chk_7e4ecde2ae4e45b4b9e475e0a587aa47", + "snapshot_id": "snp_1c7772010bcb7c8430fe062d975094271762a68627dbc53f4e2b9f0557687eae", + "source_view_hash": "d741906fcdcc5aa33a9d14f8a5e0cbdf7d2e572266a098beadddb595ce6853d9", + "freshness_status": "fresh" + }, + "configured_installation": { + "profile": "normal", + "provider": "none", + "configuration_only": true, + "plan_hash": "c0e6a2e09d785cea4c1b4f33ac37a3b43cdb683832390e74dceea7508c98dc92", + "plan_report_sha256": "ddda6adc58636291bcbe70f059027e18a00c97d8a87195f39a1e1ae301017527", + "launch_sha256": "495b174bae76ba589c94631b8fff6b7a31bb72c1d1a8a967decd5a27d51f10cc", + "installation_file": { + "path": ".codex/config.toml", + "source": "installed-ignored", + "sha256": "9bb852f8ee26992009a8a2f1f709e4db2b95f42bf826df78e5efa66ee84275e8" + }, + "guidance": { + "path": "AGENTS.override.md", + "source": "installed-ignored", + "sha256": "abb5fcdce32a53d14c618b18e842f0070e83fe9061e52d61a380d3a3086dd371" + }, + "expected_tools": [ + "codemesh_get_context_package", + "codemesh_get_repository_status", + "codemesh_list_repositories", + "codemesh_get_node" + ] + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "provider": "none", + "probe_query": "audio-only container VideoDownloadOption DownloadMultipleSetupViewModel", + "expected_paths": [ + "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs" + ], + "returned_path_count": 8, + "wrong_checkout_rejected": true, + "positive_probe_sha256": "fbff77ffdc1c860bb2241b8e9acb2d4ac587b5fc69f2ae512b1d24fb39d76281", + "rejection_probe_sha256": "24e652f49c397e407632fba3d89cc8bc43a4f1c427a5f3bda8842be5f51f9209", + "report_sha256": "534f83ae9fb14834a8170ea2cfcebfdb9907bdd98dca16ab602a0a95111c28ba" + }, + "impact_live_gate": { + "suite": "youtube-downloader-impact-live", + "passed": true, + "comparable": true, + "case_count": 2, + "tool_call_count": 6, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 0.891323, + "primary_rank": 2, + "secret_leaks": 0, + "model_providers": [], + "report_sha256": "54fc35442c28b85d5db492cf320b39c20d76e1f3dab5658ba0994a04da5a3709" + }, + "unchanged_canonical_live_gate": { + "suite": "youtube-downloader-live", + "passed": true, + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "model_providers": [ + "none" + ], + "report_sha256": "c510ad547bff20e3ce1c2d80ea4ffb6fb182224f4ef00bfec33c57db44a0cf3e" + }, + "claim_ceiling": { + "established": [ + "provider_free_configured_plan_and_runtime_identity", + "prompt_parity_preflight_configuration", + "fresh_fail_closed_youtube_downloader_binding", + "comparable_impact_retrieval_gate", + "unchanged_canonical_retrieval_gate_preserved" + ], + "not_established": [ + "normal_agent_context_package_adoption", + "codemesh_correctness_or_efficiency_benefit", + "repeatable_configured_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "provider_free_gates_invoked_model": false, + "subsequent_model_campaign_recorded_separately": true, + "push": "not_authorized", + "release": "not_authorized", + "deployment": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/context-package-latency-attribution-66033ad.json b/docs/evaluation/evidence/context-package-latency-attribution-66033ad.json new file mode 100644 index 0000000..22c7d73 --- /dev/null +++ b/docs/evaluation/evidence/context-package-latency-attribution-66033ad.json @@ -0,0 +1,44 @@ +{ + "schema_version": "codemesh-context-package-latency-attribution-v1", + "evidence_state": "provider-free-live-timing", + "generated_at": "2026-09-01T18:27:28.077465+00:00", + "codemesh_candidate": "66033addceb9537566fb67567e50ae1f21a55eea", + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "suite": "youtube-downloader-impact-adaptive-live", + "suite_sha256": "40b4d994ee704c7db22a973538c0a34f3bb28b6cab305857d553f2928f392ea8", + "retained_report_sha256": "54e806e4c1fdce99590ab15d5fea7105b6483e0e253b3107624c57992e140961", + "warmup": 1, + "repetitions": 5, + "provider": "none", + "passed": true, + "comparable": true, + "retrieval_summary": { + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.875, + "mean_ndcg_at_k": 0.862253, + "secret_leaks": 0 + }, + "task_shaped_package_p50_ms": { + "total": 16830.931, + "repository_resolution": 1.05, + "search": 16725.37, + "repository_metadata": 1.515, + "item_hydration": 102.668, + "assembly": 0.298 + }, + "task_shaped_hydration_operation_p50_ms": { + "node": 45.305, + "content_fallback": 4.093, + "summary": 8.179, + "relationships": 44.055 + }, + "dominance_gate": { + "required_largest_stage": true, + "required_total_share": 0.5, + "measured_largest_stage": "search", + "measured_hydration_share": 0.0061, + "passed": false + }, + "decision": "Do not apply bounded item-hydration concurrency or substitute another unmeasured optimization." +} diff --git a/docs/evaluation/evidence/local-codeql-review-b73e19a.json b/docs/evaluation/evidence/local-codeql-review-b73e19a.json new file mode 100644 index 0000000..82ddd36 --- /dev/null +++ b/docs/evaluation/evidence/local-codeql-review-b73e19a.json @@ -0,0 +1,1836 @@ +{ + "schema_version": "codemesh-local-codeql-review-v1", + "recorded_at": "2026-09-05", + "evidence_class": "local-static-analysis-and-bounded-source-review", + "private_archive": ".codemesh-evals/codeql-v2.26.4 in main checkout", + "cli": { + "version": "2.26.4", + "release": "github/codeql-cli-binaries v2.26.4", + "archive_sha256": "d372d54345e058fe6f7ff8074acd155522fff6770b8d72ee1371ea89deafa193" + }, + "query_packs": { + "python": "codeql/python-queries@1.8.9", + "csharp": "codeql/csharp-queries@1.9.2" + }, + "suite": "security-extended", + "resolved_query_counts_including_diagnostics_and_metrics": { + "python": 52, + "csharp": 70 + }, + "coverage": { + "python": "52/52 Python files; 1/1 Actions file reported by Python extraction, not a separate Actions security suite", + "csharp": "58/58 C# files, complete traced solution build" + }, + "csharp_source_unchanged_between_base_and_final": true, + "scans": [ + { + "sarif": "python-8e9c0da-local.sarif", + "sha256": "ab8bcd79116f8f873786efdf3974e8a1425f77fd0dab8f0f04defcb8d665a741", + "candidate": "8e9c0da7b497fe51126e3ef66509a8587cfe8d83", + "threat_models": [ + "default", + "local" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 50, + "results": { + "py/command-line-injection": 3, + "py/reflective-xss": 1, + "py/path-injection": 35 + }, + "result_count": 39, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "python-e0beba9.sarif", + "sha256": "29b040ecf91f9736262d69415ad3dc6ded568e21a4c6340d8aeba0613f8fa59a", + "candidate": "e0beba9bbe3829c8bf8070deaeff9f1912719a18", + "threat_models": [ + "default" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 50, + "results": { + "py/reflective-xss": 1 + }, + "result_count": 1, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "csharp-e0beba9.sarif", + "sha256": "169bc172e22941897225b5f33937f2976f6fd16c17d1001ca09c90cd6c8bbf85", + "candidate": "e0beba9bbe3829c8bf8070deaeff9f1912719a18", + "threat_models": [ + "default" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 63, + "results": {}, + "result_count": 0, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "python-e0beba9-local.sarif", + "sha256": "06cee3f00c872d6f249b0a502f81f06238f51987a1f1a3329c31ea5e609f0e16", + "candidate": "e0beba9bbe3829c8bf8070deaeff9f1912719a18", + "threat_models": [ + "default", + "local" + ], + "admissible_for_named_threat_models": false, + "tool_version": "2.26.4", + "rule_count": 50, + "results": { + "py/reflective-xss": 1 + }, + "result_count": 1, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "csharp-e0beba9-local.sarif", + "sha256": "1d34480208f35cfe58ba52870a9810f0fa824ee60cd0acb3bb39955e64656dd5", + "candidate": "e0beba9bbe3829c8bf8070deaeff9f1912719a18", + "threat_models": [ + "default", + "local" + ], + "admissible_for_named_threat_models": false, + "tool_version": "2.26.4", + "rule_count": 63, + "results": {}, + "result_count": 0, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "python-e0beba9-local-rerun.sarif", + "sha256": "5e2d289baaf1f7e8a0cc71e684c31cd6af12ee8be3eb6c06df829e9f7aea1e2a", + "candidate": "e0beba9bbe3829c8bf8070deaeff9f1912719a18", + "threat_models": [ + "default", + "local" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 50, + "results": { + "py/command-line-injection": 3, + "py/reflective-xss": 1, + "py/path-injection": 58 + }, + "result_count": 62, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "csharp-e0beba9-local-rerun.sarif", + "sha256": "6aefa77493b47743a1ee2dcc907f92b97e6e2b6f3a1689a8f26e8f0236c8e345", + "candidate": "e0beba9bbe3829c8bf8070deaeff9f1912719a18", + "threat_models": [ + "default", + "local" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 63, + "results": { + "cs/command-line-injection": 3, + "cs/path-injection": 189 + }, + "result_count": 192, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "python-c4e215f-local.sarif", + "sha256": "3e62a15e6cfcb2a0fa2bc3a5fb805bd3bdcea4b59db8ead827988bd8f76a9605", + "candidate": "c4e215faf2a83972eb5fb469ab94742833ab95b4", + "threat_models": [ + "default", + "local" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 50, + "results": { + "py/command-line-injection": 3, + "py/path-injection": 58 + }, + "result_count": 61, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "python-b73e19a-local.sarif", + "sha256": "654c45e0baa3aa24ed82894f19330b04d40550a0df3d33164b696f926bef8abf", + "candidate": "b73e19afef49fadb2c420cef24ebf0251d31f226", + "threat_models": [ + "default", + "local" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 50, + "results": { + "py/command-line-injection": 3, + "py/path-injection": 58 + }, + "result_count": 61, + "execution_successful": true, + "warnings_and_errors": 0 + }, + { + "sarif": "python-b73e19a-default.sarif", + "sha256": "6c72b8acc00fad00ef3a59e8bde0a64baee5ed42ab9396b48b449fca1c8a0221", + "candidate": "b73e19afef49fadb2c420cef24ebf0251d31f226", + "threat_models": [ + "default" + ], + "admissible_for_named_threat_models": true, + "tool_version": "2.26.4", + "rule_count": 50, + "results": {}, + "result_count": 0, + "execution_successful": true, + "warnings_and_errors": 0 + } + ], + "review": [ + { + "sarif": "python-b73e19a-local.sarif", + "disposition": "locally-inspected-boundaries-not-dismissed", + "groups": [ + { + "boundary": "commands", + "rationale": "Operator-selected trusted suite argv or Codex executable. subprocess argv remains shell-free; an explicitly selected shell or executable can execute operator-authorized code. No untrusted-suite sandbox claim.", + "findings": [ + { + "sarif_result_index": 0, + "rule": "py/command-line-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 1339 + }, + { + "sarif_result_index": 1, + "rule": "py/command-line-injection", + "file": "agent-access/codemesh_agent_access/evaluation/codex.py", + "line": 127 + }, + { + "sarif_result_index": 2, + "rule": "py/command-line-injection", + "file": "agent-access/codemesh_agent_access/evaluation/codex.py", + "line": 209 + } + ] + }, + { + "boundary": "evaluation", + "rationale": "Operator-selected suite, repository, run root, or report/review path. Generated descendants and setup patches undergo resolved containment and identifier validation; fixed report files inherit the selected output root. Internal response/audit paths originate in the evaluator. See the earlier per-boundary review.", + "findings": [ + { + "sarif_result_index": 3, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 213 + }, + { + "sarif_result_index": 4, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 324 + }, + { + "sarif_result_index": 5, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 341 + }, + { + "sarif_result_index": 6, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 623 + }, + { + "sarif_result_index": 7, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 628 + }, + { + "sarif_result_index": 8, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 665 + }, + { + "sarif_result_index": 9, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 996 + }, + { + "sarif_result_index": 10, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 997 + }, + { + "sarif_result_index": 11, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 1001 + }, + { + "sarif_result_index": 12, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 1004 + }, + { + "sarif_result_index": 13, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 1006 + }, + { + "sarif_result_index": 14, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 1007 + }, + { + "sarif_result_index": 15, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 137 + }, + { + "sarif_result_index": 16, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 194 + }, + { + "sarif_result_index": 17, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 227 + }, + { + "sarif_result_index": 18, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 228 + }, + { + "sarif_result_index": 19, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 311 + }, + { + "sarif_result_index": 20, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 310 + }, + { + "sarif_result_index": 21, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 301 + }, + { + "sarif_result_index": 22, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 306 + }, + { + "sarif_result_index": 23, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 302 + }, + { + "sarif_result_index": 24, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/agent.py", + "line": 1308 + }, + { + "sarif_result_index": 25, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 590 + }, + { + "sarif_result_index": 26, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 592 + }, + { + "sarif_result_index": 27, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 646 + }, + { + "sarif_result_index": 28, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 1102 + }, + { + "sarif_result_index": 29, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark.py", + "line": 1121 + }, + { + "sarif_result_index": 32, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/codex.py", + "line": 127 + } + ] + }, + { + "boundary": "response-output", + "rationale": "Internal response helper receives an evaluator-owned temporary output path. The answer tool arguments do not choose the destination.", + "findings": [ + { + "sarif_result_index": 30, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/benchmark_response.py", + "line": 71 + } + ] + }, + { + "boundary": "commit-input", + "rationale": "Explicit read-only local --message-file input to the Conventional Commit checker.", + "findings": [ + { + "sarif_result_index": 31, + "rule": "py/path-injection", + "file": "tools/check_conventional_commit.py", + "line": 64 + } + ] + }, + { + "boundary": "cli-output", + "rationale": "Explicit local --output or --preflight-output destination selected by the operator; parent creation and report writes are intentional. No repository-only destination contract.", + "findings": [ + { + "sarif_result_index": 33, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/cli.py", + "line": 617 + }, + { + "sarif_result_index": 34, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/cli.py", + "line": 618 + }, + { + "sarif_result_index": 35, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/cli.py", + "line": 673 + }, + { + "sarif_result_index": 36, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/cli.py", + "line": 674 + }, + { + "sarif_result_index": 37, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/cli.py", + "line": 718 + }, + { + "sarif_result_index": 38, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/cli.py", + "line": 719 + }, + { + "sarif_result_index": 39, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/cli.py", + "line": 801 + } + ] + }, + { + "boundary": "feedback", + "rationale": "Exact operator-selected Git root, contained diagnostic paths, and content-derived packet name. b73e19a rejects linked outboxes and uses exclusive packet creation. The prior three overwrite regressions are retained; no concurrent hostile directory-replacement defense is claimed.", + "findings": [ + { + "sarif_result_index": 40, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/feedback.py", + "line": 225 + }, + { + "sarif_result_index": 41, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/feedback.py", + "line": 226 + }, + { + "sarif_result_index": 42, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/feedback.py", + "line": 355 + }, + { + "sarif_result_index": 43, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/feedback.py", + "line": 420 + } + ] + }, + { + "boundary": "installer", + "rationale": "Explicit target, CodeMesh root, and plan file. The binding must match the selected target; apply requires the reviewed plan hash, checks resolved target containment and prior content hashes, then writes through a same-directory temporary file and atomic replacement. Plan input remains trusted, not signed.", + "findings": [ + { + "sarif_result_index": 44, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 74 + }, + { + "sarif_result_index": 45, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 75 + }, + { + "sarif_result_index": 46, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 173 + }, + { + "sarif_result_index": 47, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 194 + }, + { + "sarif_result_index": 48, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 415 + }, + { + "sarif_result_index": 49, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 417 + }, + { + "sarif_result_index": 50, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 425 + }, + { + "sarif_result_index": 51, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 425 + }, + { + "sarif_result_index": 52, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/installer.py", + "line": 428 + } + ] + }, + { + "boundary": "suite-selection", + "rationale": "Existing explicit suite file is operator-selected; packaged fallback names pass safe-segment validation. JSON models and suite artifacts are validated before execution.", + "findings": [ + { + "sarif_result_index": 53, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/models.py", + "line": 493 + }, + { + "sarif_result_index": 54, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/models.py", + "line": 502 + }, + { + "sarif_result_index": 55, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/models.py", + "line": 504 + } + ] + }, + { + "boundary": "validation-paths", + "rationale": "Resolved setup-patch and generated-artifact checks deliberately examine operator-selected roots. Reject escaping descendants; static CLI taint persists through these checks. These checks are not a local-process sandbox.", + "findings": [ + { + "sarif_result_index": 56, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/paths.py", + "line": 45 + }, + { + "sarif_result_index": 57, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/paths.py", + "line": 46 + }, + { + "sarif_result_index": 58, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/paths.py", + "line": 51 + }, + { + "sarif_result_index": 59, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/paths.py", + "line": 58 + }, + { + "sarif_result_index": 60, + "rule": "py/path-injection", + "file": "agent-access/codemesh_agent_access/evaluation/paths.py", + "line": 59 + } + ] + } + ] + }, + { + "sarif": "csharp-e0beba9-local-rerun.sarif", + "disposition": "locally-inspected-boundaries-not-dismissed", + "groups": [ + { + "boundary": "git-root", + "rationale": "Observed test-origin path reaches Git WorkingDirectory. Executable is fixed to git, arguments use ArgumentList, and UseShellExecute is false; no shell command interpolation.", + "findings": [ + { + "sarif_result_index": 0, + "rule": "cs/command-line-injection", + "file": "src/CodeMesh.Ingestion/RepositoryGitSnapshot.cs", + "line": 56 + }, + { + "sarif_result_index": 1, + "rule": "cs/command-line-injection", + "file": "src/CodeMesh.Ingestion/RepositoryIdentityService.cs", + "line": 175 + } + ] + }, + { + "boundary": "test-fixtures", + "rationale": "Test harness flows originate in GetTempPath or the explicitly configured sample-root environment variable. Generated temporary fixtures, deliberate adversarial paths, and shell-free Git fixture setup run under the local test operator. These are retained findings, not excluded test files.", + "findings": [ + { + "sarif_result_index": 2, + "rule": "cs/command-line-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4756 + }, + { + "sarif_result_index": 7, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 275 + }, + { + "sarif_result_index": 8, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 316 + }, + { + "sarif_result_index": 9, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 513 + }, + { + "sarif_result_index": 10, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 526 + }, + { + "sarif_result_index": 11, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 527 + }, + { + "sarif_result_index": 12, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 528 + }, + { + "sarif_result_index": 13, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 537 + }, + { + "sarif_result_index": 14, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 540 + }, + { + "sarif_result_index": 15, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 569 + }, + { + "sarif_result_index": 16, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 598 + }, + { + "sarif_result_index": 17, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1999 + }, + { + "sarif_result_index": 18, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2045 + }, + { + "sarif_result_index": 19, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2075 + }, + { + "sarif_result_index": 20, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2087 + }, + { + "sarif_result_index": 22, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2143 + }, + { + "sarif_result_index": 23, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2187 + }, + { + "sarif_result_index": 24, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2188 + }, + { + "sarif_result_index": 25, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2281 + }, + { + "sarif_result_index": 26, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2292 + }, + { + "sarif_result_index": 27, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3530 + }, + { + "sarif_result_index": 28, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3620 + }, + { + "sarif_result_index": 29, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3696 + }, + { + "sarif_result_index": 30, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3779 + }, + { + "sarif_result_index": 31, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3788 + }, + { + "sarif_result_index": 32, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3800 + }, + { + "sarif_result_index": 33, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3812 + }, + { + "sarif_result_index": 34, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3876 + }, + { + "sarif_result_index": 35, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3887 + }, + { + "sarif_result_index": 36, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4052 + }, + { + "sarif_result_index": 37, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4062 + }, + { + "sarif_result_index": 38, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4104 + }, + { + "sarif_result_index": 39, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4105 + }, + { + "sarif_result_index": 40, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4115 + }, + { + "sarif_result_index": 41, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4118 + }, + { + "sarif_result_index": 42, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4121 + }, + { + "sarif_result_index": 43, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4168 + }, + { + "sarif_result_index": 44, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4170 + }, + { + "sarif_result_index": 45, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4185 + }, + { + "sarif_result_index": 46, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4205 + }, + { + "sarif_result_index": 47, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4281 + }, + { + "sarif_result_index": 48, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4287 + }, + { + "sarif_result_index": 49, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4291 + }, + { + "sarif_result_index": 50, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4306 + }, + { + "sarif_result_index": 51, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4378 + }, + { + "sarif_result_index": 52, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4483 + }, + { + "sarif_result_index": 53, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4526 + }, + { + "sarif_result_index": 54, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4527 + }, + { + "sarif_result_index": 55, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4529 + }, + { + "sarif_result_index": 56, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4564 + }, + { + "sarif_result_index": 57, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4624 + }, + { + "sarif_result_index": 58, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4625 + }, + { + "sarif_result_index": 59, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4627 + }, + { + "sarif_result_index": 60, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4636 + }, + { + "sarif_result_index": 61, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4645 + }, + { + "sarif_result_index": 62, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4648 + }, + { + "sarif_result_index": 63, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4785 + }, + { + "sarif_result_index": 66, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 161 + }, + { + "sarif_result_index": 67, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 162 + }, + { + "sarif_result_index": 68, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 168 + }, + { + "sarif_result_index": 69, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 169 + }, + { + "sarif_result_index": 70, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 174 + }, + { + "sarif_result_index": 71, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 175 + }, + { + "sarif_result_index": 72, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 198 + }, + { + "sarif_result_index": 73, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 199 + }, + { + "sarif_result_index": 74, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 273 + }, + { + "sarif_result_index": 75, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 300 + }, + { + "sarif_result_index": 76, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 302 + }, + { + "sarif_result_index": 77, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 317 + }, + { + "sarif_result_index": 78, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 337 + }, + { + "sarif_result_index": 79, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 339 + }, + { + "sarif_result_index": 80, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 352 + }, + { + "sarif_result_index": 81, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 359 + }, + { + "sarif_result_index": 82, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 361 + }, + { + "sarif_result_index": 83, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 376 + }, + { + "sarif_result_index": 84, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 377 + }, + { + "sarif_result_index": 85, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 386 + }, + { + "sarif_result_index": 86, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 388 + }, + { + "sarif_result_index": 87, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 402 + }, + { + "sarif_result_index": 88, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 403 + }, + { + "sarif_result_index": 89, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 437 + }, + { + "sarif_result_index": 90, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 439 + }, + { + "sarif_result_index": 91, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 453 + }, + { + "sarif_result_index": 92, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 454 + }, + { + "sarif_result_index": 93, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 492 + }, + { + "sarif_result_index": 94, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 494 + }, + { + "sarif_result_index": 95, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 514 + }, + { + "sarif_result_index": 96, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 515 + }, + { + "sarif_result_index": 97, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 516 + }, + { + "sarif_result_index": 98, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 517 + }, + { + "sarif_result_index": 99, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 518 + }, + { + "sarif_result_index": 100, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 523 + }, + { + "sarif_result_index": 101, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 536 + }, + { + "sarif_result_index": 102, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 539 + }, + { + "sarif_result_index": 103, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 544 + }, + { + "sarif_result_index": 104, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 550 + }, + { + "sarif_result_index": 105, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 552 + }, + { + "sarif_result_index": 106, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 567 + }, + { + "sarif_result_index": 107, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 568 + }, + { + "sarif_result_index": 108, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 578 + }, + { + "sarif_result_index": 109, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 580 + }, + { + "sarif_result_index": 110, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 595 + }, + { + "sarif_result_index": 111, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 596 + }, + { + "sarif_result_index": 112, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 597 + }, + { + "sarif_result_index": 113, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 634 + }, + { + "sarif_result_index": 114, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 636 + }, + { + "sarif_result_index": 115, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 649 + }, + { + "sarif_result_index": 116, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 656 + }, + { + "sarif_result_index": 117, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 658 + }, + { + "sarif_result_index": 118, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1119 + }, + { + "sarif_result_index": 119, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1170 + }, + { + "sarif_result_index": 120, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1177 + }, + { + "sarif_result_index": 121, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1235 + }, + { + "sarif_result_index": 122, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1242 + }, + { + "sarif_result_index": 123, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1354 + }, + { + "sarif_result_index": 124, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1724 + }, + { + "sarif_result_index": 125, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1762 + }, + { + "sarif_result_index": 126, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1769 + }, + { + "sarif_result_index": 127, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1830 + }, + { + "sarif_result_index": 128, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1837 + }, + { + "sarif_result_index": 129, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1900 + }, + { + "sarif_result_index": 130, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1910 + }, + { + "sarif_result_index": 131, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1939 + }, + { + "sarif_result_index": 132, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1992 + }, + { + "sarif_result_index": 133, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2031 + }, + { + "sarif_result_index": 134, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1946 + }, + { + "sarif_result_index": 135, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 1985 + }, + { + "sarif_result_index": 136, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2038 + }, + { + "sarif_result_index": 137, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2114 + }, + { + "sarif_result_index": 138, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2116 + }, + { + "sarif_result_index": 139, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2127 + }, + { + "sarif_result_index": 140, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2168 + }, + { + "sarif_result_index": 141, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2170 + }, + { + "sarif_result_index": 142, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2180 + }, + { + "sarif_result_index": 143, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2214 + }, + { + "sarif_result_index": 144, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2216 + }, + { + "sarif_result_index": 145, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2224 + }, + { + "sarif_result_index": 146, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2268 + }, + { + "sarif_result_index": 147, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2275 + }, + { + "sarif_result_index": 148, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 2308 + }, + { + "sarif_result_index": 149, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3525 + }, + { + "sarif_result_index": 150, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3607 + }, + { + "sarif_result_index": 151, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3615 + }, + { + "sarif_result_index": 152, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3684 + }, + { + "sarif_result_index": 153, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3691 + }, + { + "sarif_result_index": 154, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3753 + }, + { + "sarif_result_index": 155, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3764 + }, + { + "sarif_result_index": 156, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3765 + }, + { + "sarif_result_index": 157, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3771 + }, + { + "sarif_result_index": 158, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3828 + }, + { + "sarif_result_index": 159, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3864 + }, + { + "sarif_result_index": 160, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3871 + }, + { + "sarif_result_index": 161, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 3928 + }, + { + "sarif_result_index": 162, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4030 + }, + { + "sarif_result_index": 163, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4036 + }, + { + "sarif_result_index": 164, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4096 + }, + { + "sarif_result_index": 165, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4106 + }, + { + "sarif_result_index": 166, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4112 + }, + { + "sarif_result_index": 167, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4125 + }, + { + "sarif_result_index": 168, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4156 + }, + { + "sarif_result_index": 169, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4157 + }, + { + "sarif_result_index": 170, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4167 + }, + { + "sarif_result_index": 171, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4250 + }, + { + "sarif_result_index": 172, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4252 + }, + { + "sarif_result_index": 173, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4280 + }, + { + "sarif_result_index": 174, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4286 + }, + { + "sarif_result_index": 175, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4346 + }, + { + "sarif_result_index": 176, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4348 + }, + { + "sarif_result_index": 177, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4376 + }, + { + "sarif_result_index": 178, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4442 + }, + { + "sarif_result_index": 179, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4444 + }, + { + "sarif_result_index": 180, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4481 + }, + { + "sarif_result_index": 181, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4514 + }, + { + "sarif_result_index": 182, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4516 + }, + { + "sarif_result_index": 183, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4612 + }, + { + "sarif_result_index": 184, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4614 + }, + { + "sarif_result_index": 185, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4687 + }, + { + "sarif_result_index": 186, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4689 + }, + { + "sarif_result_index": 187, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4717 + }, + { + "sarif_result_index": 188, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4782 + }, + { + "sarif_result_index": 189, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4783 + }, + { + "sarif_result_index": 190, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4788 + }, + { + "sarif_result_index": 191, + "rule": "cs/path-injection", + "file": "tests/CodeMesh.Tests/Program.cs", + "line": 4791 + } + ] + }, + { + "boundary": "environment-input", + "rationale": "Observed test-origin root reaches the explicit local environment-file loader. File existence and read are intentional; the selected environment input is private and trusted.", + "findings": [ + { + "sarif_result_index": 3, + "rule": "cs/path-injection", + "file": "src/CodeMesh.Control/Configuration/EnvironmentFileLoader.cs", + "line": 11 + }, + { + "sarif_result_index": 4, + "rule": "cs/path-injection", + "file": "src/CodeMesh.Control/Configuration/EnvironmentFileLoader.cs", + "line": 17 + } + ] + }, + { + "boundary": "path-policy", + "rationale": "Observed test-origin root reaches File.GetAttributes during fail-closed reparse-point checking, after repository-relative containment validation.", + "findings": [ + { + "sarif_result_index": 5, + "rule": "cs/path-injection", + "file": "src/CodeMesh.Domain/Utilities/RepositoryPathPolicy.cs", + "line": 201 + } + ] + }, + { + "boundary": "parser-root", + "rationale": "Observed test-origin root reaches Directory.Exists on the explicitly selected ParseRequest repository root. File enumeration subsequently applies RepositoryPathPolicy; no remote multi-user repository authorization is implied.", + "findings": [ + { + "sarif_result_index": 6, + "rule": "cs/path-injection", + "file": "src/CodeMesh.Parser.Deployment/DeploymentParseService.cs", + "line": 22 + }, + { + "sarif_result_index": 21, + "rule": "cs/path-injection", + "file": "src/CodeMesh.Parser.Markdown/MarkdownParseService.cs", + "line": 22 + }, + { + "sarif_result_index": 64, + "rule": "cs/path-injection", + "file": "src/CodeMesh.Parser.Python/PythonParseService.cs", + "line": 22 + }, + { + "sarif_result_index": 65, + "rule": "cs/path-injection", + "file": "src/CodeMesh.Parser.Rust/RustParseService.cs", + "line": 22 + } + ] + } + ] + } + ], + "corrections": [ + "c4e215f explicitly escapes dynamic HTML attributes; reflective-XSS result absent in fresh same-configuration scan.", + "b73e19a preserves existing feedback packet files, symlinks and hardlinks using exclusive creation; explicit linked-outbox rejection supplements the existing Linux Git ignore rejection." + ], + "retained_failures": [ + "First local-threat analyses reused default-threat cached BQRS; both are invalid for local-threat claims. --rerun evaluations supersede them without deleting the original SARIF.", + "Custom-directory pack resolution failed using pack@version:suite; absolute pinned suite paths resolved successfully.", + "Three feedback overwrite regression cases failed before correction. The original linked-outbox test failed on expected diagnostic wording because Git already rejected it; this was not an observed outbox escape." + ], + "limits": [ + "All 253 final local-input results remain retained and were not suppressed or dismissed.", + "Local scans do not alter hosted alerts or prove absence of vulnerabilities, sandboxing, production security or release acceptance.", + "Hosted Python findings remain 39 on published 8e9c0da; the local historical control reproduces their rule/count distribution, not hosted alert identity.", + "No provider call, paid campaign, upload, push, release or deployment." + ] +} diff --git a/docs/evaluation/evidence/local-development-upgrade-ce23f2e.json b/docs/evaluation/evidence/local-development-upgrade-ce23f2e.json new file mode 100644 index 0000000..9819771 --- /dev/null +++ b/docs/evaluation/evidence/local-development-upgrade-ce23f2e.json @@ -0,0 +1,269 @@ +{ + "schema_version": "codemesh-local-development-upgrade-rehearsal-v1", + "recorded_at": "2026-09-05", + "evidence_class": "provider-free-isolated-linux-upgrade-and-whole-set-rollback", + "baseline": "8e9c0da7b497fe51126e3ef66509a8587cfe8d83", + "candidate": "ce23f2e5eb2439353e6207fc373d5e82f9a6dd28", + "sample_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "sample_was_separate_clean_clone": true, + "private_archive": ".codemesh-evals/upgrade-8e9c0da-ce23f2e in main checkout", + "projects": { + "baseline": "codemesh-upgrade-old8e9", + "candidate": "codemesh-upgrade-newce23", + "rollback": "codemesh-upgrade-rollback8e9" + }, + "baseline_images": { + "neo4j": "sha256:362542416de6c09a971484d1893878016cc3b5cdec166e54b1c824a220ecd6b9", + "mongodb": "sha256:340c1c56fb10e95cf79ff547f8664b96bc6ead9909bc355238cbf865a9695a6f", + "qdrant": "sha256:0bd98fa7977f1e75694779359ca4e212822e5a71334e28421182f72f209d5286", + "agent-access": "sha256:c066b6fd567f4545827ff5a9f471ba7534784fe61afd77ca8c3e2d1ccb46d0f5" + }, + "candidate_images": { + "neo4j": "sha256:362542416de6c09a971484d1893878016cc3b5cdec166e54b1c824a220ecd6b9", + "mongodb": "sha256:340c1c56fb10e95cf79ff547f8664b96bc6ead9909bc355238cbf865a9695a6f", + "qdrant": "sha256:0bd98fa7977f1e75694779359ca4e212822e5a71334e28421182f72f209d5286", + "agent-access": "sha256:df889c292e2a0e74fe29db971cd68c35a6adc4730c474642a48732a5e4a86f0b" + }, + "backups": [ + { + "logical_volume": "neo4j_data", + "archive": "neo4j_data.tar.gz", + "bytes": 4799450, + "sha256": "e1412666b3750b590c5dd403983267d2471aae004f98b26afd59bf526cd8cee5" + }, + { + "logical_volume": "neo4j_logs", + "archive": "neo4j_logs.tar.gz", + "bytes": 10377, + "sha256": "70246eeb2c5ea5101079ea80412a185aaf29c4ee240aab1d18b499685f236e40" + }, + { + "logical_volume": "mongodb_data", + "archive": "mongodb_data.tar.gz", + "bytes": 920093, + "sha256": "b760645e39179fbef632cf7a68c80c4dab785e68c6c65fa6c528cb57deb758d1" + }, + { + "logical_volume": "qdrant_data", + "archive": "qdrant_data.tar.gz", + "bytes": 389476, + "sha256": "ddc53b98eaf5879e7325a310091098ea2bd8e2a17afe1a9aa09492dd65536a9a" + } + ], + "passed": { + "original_stop_backup_and_restoration": "Complete stopped four-volume set restored into initially absent volumes; every graph, MongoDB, vector, registry, snapshot, repository and ordered context-item digest matches.", + "candidate_reader": "Existing stored data and indexes unchanged before re-ingestion; package ordering/scoring differs across reader revisions and is retained separately.", + "reingestion": "Project, checkout, commit and root preserved; new snapshots published; no summaries or embeddings requested.", + "candidate_source_citations": "Eight contained source spans, snapshot bindings, content hashes and snippets verified.", + "retrieval_after_dependency_assets": { + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "p50_latency_ms": 117.877, + "p95_latency_ms": 247.941 + }, + "scoped_deletion": { + "project": "codemesh-upgrade-newce23", + "result": { + "repository_id": "prj_c4c08e04f9ea4c7b903bc7f4be604f9d", + "deleted": true, + "registry_entry_existed": true, + "graph_cleaned": true, + "content_cleaned": true, + "embeddings_cleaned": true, + "summaries_cleaned": true + }, + "graph_nodes": 0, + "graph_relationships": 0, + "mongo_collection_counts": { + "content": 0, + "snapshot_checkouts": 0, + "ingestion_runs": 0, + "node_summaries": 0, + "snapshot_projects": 0, + "ingestion_generations": 0, + "snapshot_observations": 0, + "snapshots": 0, + "repositories": 0 + }, + "unrelated_synthetic_vector_points_preserved": 3 + }, + "rollback": "Original application/store image identities and complete original backup restored into another new four-volume set; every original data, metadata and ordered context-item digest matches exactly.", + "cleanup": "All three task projects and networks removed; original, candidate and rollback volumes and private backups preserved." + }, + "baseline_counts": { + "graph_nodes": 1683, + "graph_relationships": 3573, + "content": 1292, + "synthetic_vector_points": 3 + }, + "candidate_parsed_counts": { + "NodeCount": 1685, + "RelationshipCount": 3634, + "ContentCount": 1310, + "EmbeddingCount": 0, + "SummaryCount": 0 + }, + "failed_and_retained": { + "initial_retrieval": "5/6 cases passed; download-orchestration-impact relationship coverage was 0 instead of 1 on a fresh clone without dependency assets. All 18 tool calls succeeded; recall 1.0 and no secret leaks do not override this failure.", + "sample_locked_restore": "dotnet restore YoutubeDownloader.slnx --locked-mode exited 1: frozen AngleSharp 1.4.0 advisory GHSA-pgww-w46g-26qg is NU1902 treated as error. No audit or warning gate was suppressed and the frozen sample was not edited.", + "subsequent_source_analysis": "Assets emitted by that restore enabled the missing relationships after re-ingestion. The unchanged six-case/18-call suite passed, while two CMSHARP015 advisory warnings remained. This does not turn dependency restore into a pass.", + "helper_corrections": [ + "Initial readiness helper expected ok rather than the actual healthy status; health was verified before recording baseline acceptance.", + "Initial offline citation helper used splitlines and omitted the final empty Roslyn source line. Corrected line accounting verifies all eight legacy citations." + ] + }, + "validation_boundary": { + "dotnet": "Historical baseline locked restore/build pass; candidate .NET source is unchanged from f314043, whose 71-test/full-smoke evidence is separately retained.", + "python": "Candidate source unchanged from b73e19a, whose 235-test three-runtime and local CodeQL evidence is separately retained.", + "new_changes": "Documentation and retained evidence only." + }, + "limits": [ + "No supported published-version upgrade, Windows acceptance, production recovery or release approval.", + "Sample dependency acceptance remains failed for the frozen advisory.", + "No paid campaign, provider invocation, external publication, existing user-store mutation or one|nine instance change.", + "This is a local development-baseline rehearsal, not repeatable configured-agent product benefit." + ], + "artifacts": [ + { + "file": "backup-copy.log", + "sha256": "c4110816172cffd7cae52889b5ed64ef05c77efa128ffb2136cd8fd5c42c3655" + }, + { + "file": "candidate-after-assets-ingest.json", + "sha256": "8b3c9ab7877b3aef242bf553026ebcb7f9d66eaebbbfb18fcef4d333e679c29e" + }, + { + "file": "candidate-after-assets-ingest.stderr.log", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "file": "candidate-after-assets-live.json", + "sha256": "9509393acc7ce415e953d9e00bd7418556010d8008697ab55b83da5c3c5e46af" + }, + { + "file": "candidate-after-assets-live.stderr.log", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "file": "candidate-after-assets-live.stdout.log", + "sha256": "9509393acc7ce415e953d9e00bd7418556010d8008697ab55b83da5c3c5e46af" + }, + { + "file": "candidate-after-assets.json", + "sha256": "68bc4023ddbf0d4f41d70775dda5cd07d0b52f362f71c78d7e50c6ebda6d3f7d" + }, + { + "file": "candidate-after-citations.json", + "sha256": "1894695e631895b791c07cab380d9955c9362395a991c82778d3f794b0a243c3" + }, + { + "file": "candidate-after-package.json", + "sha256": "147cf64a8ac06a3b9b03f01d009d16ee3a8cfa465fa33f2a8aa9d12d42d653d0" + }, + { + "file": "candidate-after-reingest.json", + "sha256": "c762d3f49ea01c8b150c1f1aba19b5bdac99994c9bf6a727138904d9f137c816" + }, + { + "file": "candidate-before-package.json", + "sha256": "36e28d4689c4a4a1aad43dda933b07693268a91587af50b1eb7355d792c9fd03" + }, + { + "file": "candidate-before-reingest.json", + "sha256": "eca4d97908ac57c9201f51e8cf0b5016c133512a75c76869b483d0e90ae600b4" + }, + { + "file": "candidate-ingest.json", + "sha256": "b9a6c8de4abc3db2f098e1e615cee63396b948f27e98b4d2d20ab8f0479f7aac" + }, + { + "file": "candidate-ingest.stderr.log", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "file": "candidate-live.json", + "sha256": "a665b74d649ecd9786e50c1c360296be721ab54604ab0525d1890fcf061e30a3" + }, + { + "file": "candidate-live.stderr.log", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "file": "candidate-live.stdout.log", + "sha256": "a665b74d649ecd9786e50c1c360296be721ab54604ab0525d1890fcf061e30a3" + }, + { + "file": "candidate.override.json", + "sha256": "7ded024bfe59cd9fd9a7961b2f91404b81711dc373e437b919e54f01b68698eb" + }, + { + "file": "codemesh-upgrade-new-image.log", + "sha256": "202a222c4c3b7ecdab1fc4fd944f6861ef4185479dd9e6550c485d37a9d30b96" + }, + { + "file": "codemesh-upgrade-old-build.log", + "sha256": "1d02a64921cfa39d63d367cb4d33a2d01d95cc4f730a7c89915140c0302cc5c0" + }, + { + "file": "codemesh-upgrade-old-image.log", + "sha256": "0d83dd8fcf654e6bc4a04daa38a7cd1dcfd76c99425d3eead46b279088bea5f5" + }, + { + "file": "codemesh-upgrade-old-restore.log", + "sha256": "f72e309cef263c92c1dc1f3c0d8d9db5f47c84278048bd39eca995fb58409c19" + }, + { + "file": "deletion-verification.json", + "sha256": "9c72f12b8fa3226e51cc9135a8daa24009274013f27ef6a1d6b94dcb01a1bb91" + }, + { + "file": "legacy-span-probe-corrected.json", + "sha256": "dee4a9d3998348ce7096bb56e120c57e8a1ec1bc129786d06eac1773f90ab987" + }, + { + "file": "legacy-span-probe.json", + "sha256": "8b7ad5dbffac7e938982f1828e47eda4861a7638c0d206625a34658b9ae5b5d8" + }, + { + "file": "old-before.json", + "sha256": "b963fc20fd8e82c10a1adf452455bbba3456f2bc3c017770e79949f5df994bd1" + }, + { + "file": "old-ingest.stderr.log", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "file": "old-ingest.stdout.log", + "sha256": "6145b3b0df2a9b81dbeed397272478ce31d20660ffac3a0f81c0ec6d543eb51f" + }, + { + "file": "old-restored.json", + "sha256": "b963fc20fd8e82c10a1adf452455bbba3456f2bc3c017770e79949f5df994bd1" + }, + { + "file": "old.override.json", + "sha256": "384397625da72c0e47bd216d418c8734be7324b51276f125c82f21c31619db98" + }, + { + "file": "reader-package-difference.json", + "sha256": "e52dea200c9854a9c827ea4b51af55966f9d1874f69707509451a6269a0783b0" + }, + { + "file": "rollback-after.json", + "sha256": "b963fc20fd8e82c10a1adf452455bbba3456f2bc3c017770e79949f5df994bd1" + }, + { + "file": "rollback-package.json", + "sha256": "1701f3cd437c1fe6ac26381df5a35bc86eb0e4cf16e892935c85f19f05032ddf" + }, + { + "file": "sample-restore.log", + "sha256": "00dfcfb94c13beb13f86477b0ea0103dd8d7be27e5e3f77735378ee712369236" + } + ] +} diff --git a/docs/evaluation/evidence/local-package-verification-96b4ebf.json b/docs/evaluation/evidence/local-package-verification-96b4ebf.json new file mode 100644 index 0000000..13d2ca4 --- /dev/null +++ b/docs/evaluation/evidence/local-package-verification-96b4ebf.json @@ -0,0 +1,133 @@ +{ + "schema_version": "codemesh-local-package-verification-v1", + "recorded_at": "2026-09-05", + "evidence_class": "local-provider-free-package-and-container-verification", + "source_version": "0.1.0", + "application_candidate": "96b4ebf7680332f3feeb6341a13a236a70203144", + "parser_container_correction": "1ef5ca94c40a3dceca4a8d4b128bfb7eea261e98", + "candidate_clean_during_package_build": true, + "environment": { + "os": "Linux x86_64", + "dotnet_sdk": "10.0.111", + "python": "3.14.7", + "local_uv": "0.12.9", + "agent_access_docker_uv": "0.12.3" + }, + "artifact_location": ".codemesh-evals/completion-packages-96b4ebf (ignored local archive in main checkout)", + "artifacts": [ + { + "name": "codemesh_agent_access-0.1.0-py3-none-any.whl", + "size": 157651, + "sha256": "e56e5a32c8e196b03acc3f003a840c75c6521d91aae381f1be62e52a72fc7649" + }, + { + "name": "codemesh_agent_access-0.1.0.tar.gz", + "size": 231269, + "sha256": "cd1b1acb5c84213ced36d36436586e461ed47dcacffb7c068477aa27f10b5522" + }, + { + "name": "codemesh-dotnet-0.1.0-linux-build.tar.gz", + "size": 12581544, + "sha256": "e3457468a818bdc68ff9b238b96e9df1b6a543629b7987fe1fe3f29524d293b8" + }, + { + "name": "dotnet-file-manifest.json", + "size": 21969, + "sha256": "c738181c8e10be7f475c43887f17fa28b3129bcb056c746f71b20a624f48cc0a" + } + ], + "container_images": [ + { + "container": "codemesh-completion-20260905-agent-access-1", + "image_id": "sha256:272da83da7bdd81f071f198d33ebc47c6465cdd470201b2e5da1c5563e007e34" + }, + { + "container": "codemesh-completion-20260905-neo4j-1", + "image_id": "sha256:362542416de6c09a971484d1893878016cc3b5cdec166e54b1c824a220ecd6b9" + }, + { + "container": "codemesh-completion-20260905-qdrant-1", + "image_id": "sha256:0bd98fa7977f1e75694779359ca4e212822e5a71334e28421182f72f209d5286" + }, + { + "container": "codemesh-completion-20260905-mongodb-1", + "image_id": "sha256:340c1c56fb10e95cf79ff547f8664b96bc6ead9909bc355238cbf865a9695a6f" + }, + { + "container": "codemesh-completion-20260905-csharp-parser-1", + "image_id": "sha256:af15d4ac5ac8aa7d0b55efa2202a8aca9318f093b304b64b0ae0361df549be4b" + } + ], + "passed": { + "dotnet_tests": 70, + "dotnet_skipped": 0, + "python_tests": 228, + "python_skipped": 0, + "dotnet_build_and_format": true, + "python_lint_and_format": true, + "version_and_repository_standards": true, + "full_e2e_smoke_at_application_candidate": true, + "installed_wheel_fixture_cases": 5, + "installed_wheel_cli_version": "codemesh-agent-access 0.1.0", + "installed_wheel_rest_openapi_version": "0.1.0", + "installed_wheel_normal_manifest_tools": 4, + "installed_agent_and_model_suites_and_patches_load": true, + "dotnet_package_version": "CodeMesh 0.1.0+96b4ebf7680332f3feeb6341a13a236a70203144", + "runtime_dependencies_installed_from_locked_hashes": true, + "dotnet_advisory_check": "No vulnerable packages reported in 11 CodeMesh projects", + "python_advisory_check": "No known vulnerabilities or adverse statuses reported in 48 packages", + "parser_container_build_health_and_capabilities": true, + "parser_project_probe": { + "counts": { + "nodes": 1229, + "relationships": 1906, + "contents": 1229, + "diagnostics": 2 + }, + "diagnostics": [ + { + "code": "CMSHARP015", + "message": "Msbuild failed when processing the file '/workspace/src/CodeMesh.Domain/CodeMesh.Domain.csproj' with message: Read-only file system : '/workspace/src/CodeMesh.Domain/obj/Debug/net10.0/CodeMesh.Domain.GeneratedMSBuildEditorConfig.editorconfig'", + "severity": "warning", + "span": null + }, + { + "code": "CMSHARP011", + "message": "Loaded project with MSBuildWorkspace: /workspace/src/CodeMesh.Domain/CodeMesh.Domain.csproj", + "severity": "info", + "span": null + } + ] + } + }, + "failed_then_corrected": [ + { + "check": "dotnet publish -o output-directory", + "result": "MSB1008 during option parsing; equivalent -p:PublishDir=... command succeeded. Cause not attributed beyond observed command behavior." + }, + { + "check": "parser Docker build after version introduction", + "result": "Restricted .dockerignore omitted Directory.Build.props and VERSION; allow-list corrected and image built." + }, + { + "check": "runtime-only parser image project parse", + "result": "CMSHARP012 no SDK and CMSHARP010 filesystem fallback; corrected SDK runtime loaded the project without either diagnostic. One probe made before recreation still hit the old image and is retained separately." + } + ], + "warnings": [ + "Existing .NET nullable warning CS8625 and two Python dependency warnings remain.", + "Read-only parser mount emits CMSHARP015 for generated editor-config output; project still loads with CMSHARP011. Environment acceptance must assess this limitation.", + "Frozen YoutubeDownloader dependency advisory is separate from CodeMesh own dependency checks." + ], + "unavailable_or_not_run": [ + "Exact-current-head hosted CodeQL and workflow evidence; no publication performed.", + "Windows acceptance and supported upgrade baseline selection.", + "Quiesced multi-store backup and verified restoration for a selected supported upgrade environment.", + "A newly authorized model-backed configured benefit campaign.", + "Owner release-procedure acceptance, tag, package publication, deployment and stable promotion." + ], + "claim_limits": [ + "Local verification and artifacts are not released products, security-clean evidence, repeatable agent benefit, or production authority.", + "Application tests and artifacts bind application candidate; the later parser-container correction changes Docker packaging only and has its separate image/probe evidence." + ] +} diff --git a/docs/evaluation/evidence/local-package-verification-a158bc0.json b/docs/evaluation/evidence/local-package-verification-a158bc0.json new file mode 100644 index 0000000..8145438 --- /dev/null +++ b/docs/evaluation/evidence/local-package-verification-a158bc0.json @@ -0,0 +1,119 @@ +{ + "schema_version": "codemesh-local-package-verification-v2", + "recorded_at": "2026-09-05", + "candidate": "a158bc0308d9fdb8f373ec0d9d6f03233b984f9f", + "source_version": "0.1.0", + "candidate_clean_during_build": true, + "evidence_class": "provider-free-local-artifact-and-smoke-verification", + "private_archive": ".codemesh-evals/completion-packages-a158bc0 in main checkout", + "artifacts": [ + { + "file": "python/codemesh_agent_access-0.1.0-py3-none-any.whl", + "bytes": 157731, + "sha256": "b07d638a88c761fc546bcd4ad46bb28a508a3590251350b5edb6be174c9fce0f" + }, + { + "file": "python/codemesh_agent_access-0.1.0.tar.gz", + "bytes": 231686, + "sha256": "c1341687fe62ce06ddcbcf1b25100d489593663b91d51b7c9e4ca862cf96b072" + }, + { + "file": "codemesh-dotnet-0.1.0-linux-build.tar.gz", + "bytes": 12583162, + "sha256": "6ce9ff6daf7674cfad4184be4b131b574e12210a43deadf386020fb7ff0dc8b3" + }, + { + "file": "dotnet-file-manifest.json", + "bytes": 22251, + "sha256": "cfdee26f4341750b36e2a701f0bf33f09a7e9f3131dfbebbaf8bcead9b4b81d1" + } + ], + "passed": { + "dotnet_publish": "Release, 132 files including CodeMesh.DesignTime.targets", + "published_dotnet_version": "CodeMesh 0.1.0+a158bc0308d9fdb8f373ec0d9d6f03233b984f9f", + "python_wheel_and_source_build": true, + "installed_wheel_outside_source_checkout": true, + "runtime_dependencies_installed_from_locked_hashes": true, + "installed_wheel_version": "0.1.0", + "installed_rest_openapi_version": "0.1.0", + "normal_manifest_tools": 4, + "packaged_suites_loaded_including_patch_validation": 16, + "installed_fixture_cases": 5, + "full_smoke": true, + "dotnet_advisory_projects_without_known_vulnerabilities": 11, + "python_advisory_packages_without_known_vulnerabilities": 48, + "parser_read_only_concurrent_probe": { + "normalization": "Exclude only CodeContent.createdAt and updatedAt observation timestamps; preserve every node, edge, content field and list order.", + "parses": [ + { + "counts": { + "nodes": 1229, + "relationships": 1906, + "contents": 1229 + }, + "content_sha256": "f665d16b2ec14f4112606e8e1383557280b9955c37bd3147a7195631bf1fac03", + "diagnostics": [ + { + "code": "CMSHARP011", + "message": "Loaded project with MSBuildWorkspace: /workspace/src/CodeMesh.Domain/CodeMesh.Domain.csproj", + "severity": "info", + "span": null + } + ] + }, + { + "counts": { + "nodes": 1229, + "relationships": 1906, + "contents": 1229 + }, + "content_sha256": "f665d16b2ec14f4112606e8e1383557280b9955c37bd3147a7195631bf1fac03", + "diagnostics": [ + { + "code": "CMSHARP011", + "message": "Loaded project with MSBuildWorkspace: /workspace/src/CodeMesh.Domain/CodeMesh.Domain.csproj", + "severity": "info", + "span": null + } + ] + } + ], + "scratch_directories_remaining": 0 + }, + "task_container_cleanup": true + }, + "selected_smoke_image_identities": [ + { + "selected_image_tag": "neo4j:5-community", + "image_id": "sha256:362542416de6c09a971484d1893878016cc3b5cdec166e54b1c824a220ecd6b9" + }, + { + "selected_image_tag": "mongo:7", + "image_id": "sha256:340c1c56fb10e95cf79ff547f8664b96bc6ead9909bc355238cbf865a9695a6f" + }, + { + "selected_image_tag": "qdrant/qdrant:latest", + "image_id": "sha256:0bd98fa7977f1e75694779359ca4e212822e5a71334e28421182f72f209d5286" + }, + { + "selected_image_tag": "codemesh-final-a158bc0-agent-access:latest", + "image_id": "sha256:0ee08cb249a387e639840dda0b3327b975dd768f2771ede7025127eab89c7806" + } + ], + "observed_parser_image": { + "image_id": "sha256:06708986508b04fdbd5a2cd0be992dbc46789de437509df0bddc3c543ee82708", + "read_only_workspace_mount": true, + "source_candidate": "a158bc0308d9fdb8f373ec0d9d6f03233b984f9f" + }, + "retained_source_test_evidence": { + "dotnet": "71 passing tests and no skips at f314043; source, tests and build/version files unchanged through a158bc0.", + "python": "235 passing tests on Python 3.12.14, 3.13.15 and 3.14.7 at b73e19a; Python source, tests, locks and tools unchanged through a158bc0." + }, + "limits": [ + "The full smoke exercises four services and the local parser; the separately retained concurrent probe verifies the read-only parser container.", + "Source version and local artifacts do not declare a release candidate, publish a version or authorize deployment.", + "Default-threat CodeQL has no findings; 253 local-input results remain reviewed and retained without dismissal. Hosted alerts remain on the older published source.", + "Frozen sample AngleSharp advisory, supported environment/upgrade acceptance, repeatable configured-agent benefit and owner release approval remain open.", + "Python retains existing dependency warnings; no model provider or paid campaign was invoked." + ] +} diff --git a/docs/evaluation/evidence/local-recovery-rehearsal-f314043.json b/docs/evaluation/evidence/local-recovery-rehearsal-f314043.json new file mode 100644 index 0000000..665a667 --- /dev/null +++ b/docs/evaluation/evidence/local-recovery-rehearsal-f314043.json @@ -0,0 +1,221 @@ +{ + "schema_version": "codemesh-local-recovery-rehearsal-v1", + "evidence_class": "provider-free-single-node-linux-volume-restore", + "recorded_at": "2026-09-05", + "application_candidate": "f314043305b335352061ddfa868981e6616e85c7", + "private_archive": ".codemesh-evals/recovery-f314043 in the main checkout", + "source_project": "codemesh-readonly-20260905", + "restore_project": "codemesh-recovery-f314043", + "source_stopped_before_copy": true, + "destination_volumes_were_new": true, + "source_volumes_preserved": true, + "same_image_identities_verified": { + "neo4j": "sha256:362542416de6c09a971484d1893878016cc3b5cdec166e54b1c824a220ecd6b9", + "mongodb": "sha256:340c1c56fb10e95cf79ff547f8664b96bc6ead9909bc355238cbf865a9695a6f", + "qdrant": "sha256:0bd98fa7977f1e75694779359ca4e212822e5a71334e28421182f72f209d5286", + "agent-access": "sha256:272da83da7bdd81f071f198d33ebc47c6465cdd470201b2e5da1c5563e007e34" + }, + "archives": [ + { + "logical_volume": "neo4j_data", + "archive": "neo4j_data.tar.gz", + "bytes": 4813484, + "sha256": "d718bb900df4e626a3aa76ed78b2d5b62f839d0e70bb32e1052069536f4827cd" + }, + { + "logical_volume": "neo4j_logs", + "archive": "neo4j_logs.tar.gz", + "bytes": 10978, + "sha256": "1b9d54376f4ca29ad494f23435b0dd1c600d77f70db93440b1e5e4eaf78e41e8" + }, + { + "logical_volume": "mongodb_data", + "archive": "mongodb_data.tar.gz", + "bytes": 1040386, + "sha256": "4e3d86c7981d94d3f12aa277e8b4dfb9962617c82814d1f59c3a0daa2c0048ed" + }, + { + "logical_volume": "qdrant_data", + "archive": "qdrant_data.tar.gz", + "bytes": 389493, + "sha256": "ae0761671c01b5fb1aa4b5f618fb15efa32d26043c250ba0aeb37a73cf42824a" + } + ], + "verification": { + "context_package_items": { + "count": 8, + "sha256": "699aea8c30bb02d8649d69747ebd6f9bd6724ca92c16139552695128fbefdb0d" + }, + "graph_indexes": [ + { + "entityType": "NODE", + "labelsOrTypes": [ + "CodeMeshNode" + ], + "name": "codemesh_node_code_id", + "properties": [ + "id" + ], + "state": "ONLINE", + "type": "RANGE" + }, + { + "entityType": "NODE", + "labelsOrTypes": [ + "CodeMeshNode" + ], + "name": "codemesh_node_repository_id", + "properties": [ + "repositoryId", + "id" + ], + "state": "ONLINE", + "type": "RANGE" + }, + { + "entityType": "NODE", + "labelsOrTypes": [ + "CodeMeshNode" + ], + "name": "codemesh_node_repository_run", + "properties": [ + "repositoryId", + "ingestionRunId" + ], + "state": "ONLINE", + "type": "RANGE" + }, + { + "entityType": "NODE", + "labelsOrTypes": [ + "CodeMeshNode" + ], + "name": "codemesh_node_storage_key", + "properties": [ + "storageKey" + ], + "state": "ONLINE", + "type": "RANGE" + }, + { + "entityType": "RELATIONSHIP", + "labelsOrTypes": [ + "CODEMESH_REL" + ], + "name": "codemesh_relationship_id", + "properties": [ + "id" + ], + "state": "ONLINE", + "type": "RANGE" + }, + { + "entityType": "RELATIONSHIP", + "labelsOrTypes": [ + "CODEMESH_REL" + ], + "name": "codemesh_relationship_repository_run", + "properties": [ + "repositoryId", + "ingestionRunId" + ], + "state": "ONLINE", + "type": "RANGE" + }, + { + "entityType": "RELATIONSHIP", + "labelsOrTypes": [ + "CODEMESH_REL" + ], + "name": "codemesh_relationship_storage_key", + "properties": [ + "storageKey" + ], + "state": "ONLINE", + "type": "RANGE" + }, + { + "entityType": "NODE", + "labelsOrTypes": null, + "name": "index_343aff4e", + "properties": null, + "state": "ONLINE", + "type": "LOOKUP" + }, + { + "entityType": "RELATIONSHIP", + "labelsOrTypes": null, + "name": "index_f7700477", + "properties": null, + "state": "ONLINE", + "type": "LOOKUP" + } + ], + "graph_nodes": { + "count": 1685, + "sha256": "dbbf866e788a61499afc3ade84df83557a29d54254f91199148e2eb691577ec1" + }, + "graph_relationships": { + "count": 3634, + "sha256": "7ebf668d1f3049b98ad2bc07f882d718222a884698f1067bee652adefec7d5e7" + }, + "mongo": { + "content": { + "count": 1310, + "sha256": "81b3716c608dbf082051ecd6075d806a86a3cca55e5c06743d56579e9510214e" + }, + "ingestion_generations": { + "count": 1, + "sha256": "25476250c959e677bb0bdae260e2729f61e004189a446b52e023214d68263679" + }, + "ingestion_runs": { + "count": 1, + "sha256": "a93413b9a34322c22f97754dad5a57b6f99b45e0ed567aab97f1f0795777a483" + }, + "node_summaries": { + "count": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "repositories": { + "count": 1, + "sha256": "3560c23fa8ec0ce473176197db5efcae13b4a3dd440c5e2bf0927404eeb9b41e" + }, + "snapshot_checkouts": { + "count": 1, + "sha256": "b308990dc0eb07f117df4258c2143bc6065692b8ae77a7e08102d7937db32d75" + }, + "snapshot_observations": { + "count": 1, + "sha256": "6a779d80a39bdd881f6b815d0e083665dc6b8e4053cbca6038199fdf4335df02" + }, + "snapshot_projects": { + "count": 1, + "sha256": "440f6cc7ddafc86566d6fd75f7e6c14138619a5e6f5912968d0613068f17e01a" + }, + "snapshots": { + "count": 1, + "sha256": "a5719b4a011d020868ce5efd2f6d8a15ff2a06867e1bafef2bd7e220964b5718" + } + }, + "repository_id": "prj_3425db238d0d4a5fbcc1e3c9a28d1945", + "vectors": { + "codemesh-recovery-probe": { + "count": 3, + "sha256": "d4b04d7f5e2b988bd69cb82120ebb4d5e0f85cd13f39e7449cf1ef30b50c333b" + } + } + }, + "repository_metadata_sha256": "6ca1a086dfa690313f91076cad112f7347ad4c8eae9e8c999a5bbda18096f059", + "all_before_after_sections_identical": true, + "notes": [ + "Graph includes the indexed YoutubeDownloader repository at frozen source 05e63cddb6d2a96fc2d21b097129a0765824f251.", + "All CodeMesh MongoDB collection documents, including registry and snapshot lineage, were compared.", + "The three vectors are synthetic recovery fixtures, not provider-generated embedding evidence.", + "Archive contents and source credentials remain private; no model provider was invoked." + ], + "limits": [ + "One stopped single-node Linux Docker profile at identical image versions.", + "No upgrade/migration, Windows, online backup, clustered, production or release acceptance.", + "No existing user store replaced; no publication or deployment." + ] +} diff --git a/docs/evaluation/evidence/onenine-priority1-ee1d4d9.json b/docs/evaluation/evidence/onenine-priority1-ee1d4d9.json new file mode 100644 index 0000000..90e777d --- /dev/null +++ b/docs/evaluation/evidence/onenine-priority1-ee1d4d9.json @@ -0,0 +1,100 @@ +{ + "schema_version": "codemesh-priority1-candidate-evidence-v1", + "evidence_class": "provider-free-local-diagnostic", + "recorded_at": "2026-08-31T18:49:46Z", + "passed": true, + "codemesh": { + "commit": "ee1d4d9306a5fce2322d624ae17648c82ff0918a", + "clean_detached_worktree": true + }, + "deterministic_verification": { + "dotnet": { + "restore": "passed", + "build": "passed", + "tests_passed": 64, + "tests_skipped": 1, + "format": "passed", + "warnings": 1 + }, + "python": { + "lint": "passed", + "format": "passed", + "tests_passed": 115, + "warnings": 2 + }, + "documentation": { + "markdownlint": "passed", + "links": "passed", + "publication_safety": "passed", + "diff_check": "passed" + }, + "live_store_smoke": { + "status": "passed", + "compose_start": "skipped", + "compose_skip_reason": "The detached candidate intentionally has no local secret environment file; already-running local stores and Agent Access were used." + } + }, + "youtube_downloader": { + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "snapshot_id": "snp_1c7772010bcb7c8430fe062d975094271762a68627dbc53f4e2b9f0557687eae", + "ingestion": { + "nodes": 1685, + "relationships": 3634, + "content_items": 1310, + "embeddings": 0, + "summaries": 0, + "report_sha256": "4b44086b7e17d6e35abdadd8ecc818add9be9a751ec66e007067bede77f504da" + }, + "live_gate": { + "passed": true, + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "provider": "none", + "report_sha256": "ee6edd3953d7775944d91656fd4e190b532a2a03d46c1da48bef15ea4804f380" + }, + "corrective_evidence": { + "failed_candidate_commit": "90956eff80fe0a28fb431dcf4a7ea206fccd3c3e", + "failure": "Mean MRR 0.767 was below the frozen 0.800 gate after the SettingsService primary path ranked third.", + "failed_report_sha256": "3abc82116b36d80bbf720e9f401b43cecca50284d5831fecc26dd3ec95a0bb2f", + "correction": "Select and version a deterministic canonical source path for partial C# symbols.", + "verified_primary_rank": 1 + } + }, + "onenine_primary": { + "repository_commit": "e12d0281b687e58c2baac85f771a684df6fa8552", + "project_id": "prj_a570a726917d4d0ca596085aaf07af50", + "checkout_id": "chk_70c50c5494de4837ab0627aca021eab2", + "snapshot_id": "snp_52a2af6a662c8070cea0ccb6db837a68717f45422aa7574fe5db0af2543169e9", + "source_view_hash": "ce3c63a66e3e4821bf5431d9997314070d60ec61f9ab8b35aa0cdcafbacace4a", + "plan_hash": "6d37def879a062f8a93923514376f14707ddf430582e675aff44aa7b2e4ad40a", + "ingestion": { + "nodes": 45492, + "relationships": 94774, + "content_items": 12766, + "embeddings": 0, + "summaries": 0, + "report_sha256": "c9e013e640938cb0eff3291aa746ebe23e2ecb14921d304df1418cd2cd164026" + }, + "pilot_gate": { + "passed": true, + "positive_cases": 4, + "negative_cases": 4, + "feedback_packets_validated": 3, + "provider": "none", + "report_sha256": "79d38a6dc5fe4519333d12acc872c64cd87e37c812b073b9054a94322281b0bd" + } + }, + "boundaries": { + "model_backed_agent_evaluation": "not_run_requires_explicit_spend_authority", + "release": "not_authorized", + "deployment": "not_authorized", + "provider_access": "not_authorized", + "live_trading": "not_authorized" + } +} diff --git a/docs/evaluation/evidence/parser-readonly-output-verification.json b/docs/evaluation/evidence/parser-readonly-output-verification.json new file mode 100644 index 0000000..4e82a28 --- /dev/null +++ b/docs/evaluation/evidence/parser-readonly-output-verification.json @@ -0,0 +1,66 @@ +{ + "normalization": "Exclude only CodeContent.createdAt and updatedAt observation timestamps; preserve every node, edge, content field and list order.", + "parses": [ + { + "counts": { + "nodes": 1229, + "relationships": 1906, + "contents": 1229 + }, + "content_sha256": "f665d16b2ec14f4112606e8e1383557280b9955c37bd3147a7195631bf1fac03", + "diagnostics": [ + { + "code": "CMSHARP011", + "message": "Loaded project with MSBuildWorkspace: /workspace/src/CodeMesh.Domain/CodeMesh.Domain.csproj", + "severity": "info", + "span": null + } + ] + }, + { + "counts": { + "nodes": 1229, + "relationships": 1906, + "contents": 1229 + }, + "content_sha256": "f665d16b2ec14f4112606e8e1383557280b9955c37bd3147a7195631bf1fac03", + "diagnostics": [ + { + "code": "CMSHARP011", + "message": "Loaded project with MSBuildWorkspace: /workspace/src/CodeMesh.Domain/CodeMesh.Domain.csproj", + "severity": "info", + "span": null + } + ] + } + ], + "scratch_directories_remaining": 0, + "evidence_class": "local-provider-free-container-probe", + "base_commit": "f327291c5027a8f704c178cdfea3d69104d63aaf", + "parser_capability": "0.2.3", + "image_id": "sha256:06708986508b04fdbd5a2cd0be992dbc46789de437509df0bddc3c543ee82708", + "source_mount": { + "destination": "/workspace", + "read_only": true + }, + "dotnet_tests": { + "passed": 71, + "skipped": 0 + }, + "python_tests": { + "passed": 228, + "warnings": 2 + }, + "mcp_fixture_cases": 5, + "earlier_checks": { + "without_live_stores": "66 .NET passed, 5 store tests skipped; rerun with disposable stores passed all 71.", + "editor_config_only_redirect": "Removed editor-config write warning but exposed assembly-info cache write failure; expanded to standard output isolation.", + "raw_hash_comparison": "Included content creation/update timestamps and differed as expected; final equality excludes only those observation timestamps." + }, + "limits": [ + "SDK/container fixture coverage, not all project configurations or Windows acceptance.", + "No hosted scan, provider campaign, release or deployment." + ], + "implementation_commit": "f314043305b335352061ddfa868981e6616e85c7", + "full_smoke_at_implementation_commit": "passed" +} diff --git a/docs/evaluation/evidence/python-runtime-verification-b73e19a.json b/docs/evaluation/evidence/python-runtime-verification-b73e19a.json new file mode 100644 index 0000000..1357455 --- /dev/null +++ b/docs/evaluation/evidence/python-runtime-verification-b73e19a.json @@ -0,0 +1,69 @@ +{ + "schema_version": "codemesh-python-runtime-verification-v1", + "candidate": "b73e19afef49fadb2c420cef24ebf0251d31f226", + "recorded_at": "2026-09-05", + "os": "Linux x86_64", + "environment": "Isolated uv environments with locked dependencies; 3.12/3.13 runtimes stored privately without changing system Python or PATH. Python 3.14 uses the completion worktree environment.", + "private_archive": ".codemesh-evals/runtime-matrix-b73e19a in main checkout", + "results": [ + { + "python": "3.12.14", + "tests_passed": 235, + "tests_failed": 0, + "tests_skipped": 0, + "dependency_warnings": 2, + "fixture_cases_passed": 5, + "files": [ + { + "name": "codemesh-python-3.12-b73e19a-tests.log", + "sha256": "004e273f671bd276f999e176b3cea65b2a9c3ae63af4802aafb32b109f4e496c" + }, + { + "name": "codemesh-python-3.12-b73e19a-fixtures.json", + "sha256": "e5c76e6bb79d56c02a89430325890454759a865f7418497d920eb9523bb1052e" + } + ] + }, + { + "python": "3.13.15", + "tests_passed": 235, + "tests_failed": 0, + "tests_skipped": 0, + "dependency_warnings": 2, + "fixture_cases_passed": 5, + "files": [ + { + "name": "codemesh-python-3.13-b73e19a-tests.log", + "sha256": "a1a46b5299873703071973499fee261a8a9a4f969829503bff8f77ed1f6376d0" + }, + { + "name": "codemesh-python-3.13-b73e19a-fixtures.json", + "sha256": "e5d5da3ced35eadc74d1dcca5028b53cb102c9da52b3c6817fcccc816597b2f3" + } + ] + }, + { + "python": "3.14.7", + "tests_passed": 235, + "tests_failed": 0, + "tests_skipped": 0, + "dependency_warnings": 2, + "fixture_cases_passed": 5, + "files": [ + { + "name": "codemesh-python-3.14-b73e19a-tests.log", + "sha256": "ca8f1edcc336121f320be8f0185a9b7fcfc5e8cad1669345993c2a015b235d64" + }, + { + "name": "codemesh-python-3.14-b73e19a-fixtures.json", + "sha256": "5e484d234fa80007cad3b0e4d8762978a5f84e66807e795af29df3c35757c8bb" + } + ] + } + ], + "limits": [ + "Source-checkout tests and deterministic fixture evidence only; no Windows acceptance or released-version support claim.", + "Full store smoke and parser-container evidence are separately bound to f314043.", + "No provider or paid model invocation." + ] +} diff --git a/docs/evaluation/evidence/review-manifest-a158bc0.json b/docs/evaluation/evidence/review-manifest-a158bc0.json new file mode 100644 index 0000000..8104291 --- /dev/null +++ b/docs/evaluation/evidence/review-manifest-a158bc0.json @@ -0,0 +1,73 @@ +{ + "source_candidate": "a158bc0308d9fdb8f373ec0d9d6f03233b984f9f", + "source_version": "0.1.0", + "artifact_checksums_reverified": true, + "dependency_locks": [ + { + "file": "agent-access/uv.lock", + "sha256": "da4ea09cad507205965c37fc38c1a90235282ff2dc05668a0bada995929418df" + }, + { + "file": "src/CodeMesh.Cli/packages.lock.json", + "sha256": "bc73263b9c57190c717180af41a626c6e42fd31ee7f62c6e887499a14f3963c0" + }, + { + "file": "src/CodeMesh.Control/packages.lock.json", + "sha256": "7451cdea34ec51831f11519302a54b6733d8de02741b874456827a80e3b330f3" + }, + { + "file": "src/CodeMesh.Domain/packages.lock.json", + "sha256": "a29c6aa8cfb81874ff8bb78dc369d7416f28c9b8cc47e99592bfc019b20c41eb" + }, + { + "file": "src/CodeMesh.Ingestion/packages.lock.json", + "sha256": "ca0897279ad707f946c0a9ca590fa60db89ba665abfb74a9be5eb36a2a58479c" + }, + { + "file": "src/CodeMesh.Parser.CSharp/packages.lock.json", + "sha256": "c9a5ddd18785a2e11a9f074b0fc569da36a9de8ce49ea7e992acd36d49e3ee86" + }, + { + "file": "src/CodeMesh.Parser.Deployment/packages.lock.json", + "sha256": "7451cdea34ec51831f11519302a54b6733d8de02741b874456827a80e3b330f3" + }, + { + "file": "src/CodeMesh.Parser.Markdown/packages.lock.json", + "sha256": "7451cdea34ec51831f11519302a54b6733d8de02741b874456827a80e3b330f3" + }, + { + "file": "src/CodeMesh.Parser.Python/packages.lock.json", + "sha256": "7451cdea34ec51831f11519302a54b6733d8de02741b874456827a80e3b330f3" + }, + { + "file": "src/CodeMesh.Parser.Rust/packages.lock.json", + "sha256": "1fc4baf334892374baf16d24e524a8ce6c099c283321c182ab9e7f5784a2476a" + }, + { + "file": "src/CodeMesh.Storage/packages.lock.json", + "sha256": "41c2021c88bdadf9a73e58bfff9878759dffc325aa6791538204371da1520cf2" + }, + { + "file": "tests/CodeMesh.Tests/packages.lock.json", + "sha256": "bc73263b9c57190c717180af41a626c6e42fd31ee7f62c6e887499a14f3963c0" + } + ], + "local_environment": { + "os": "Linux x86_64", + "dotnet_sdk": "10.0.111", + "python": "Python 3.14.7", + "uv": "uv 0.12.9 (9f9286029 2026-09-01 x86_64-unknown-linux-gnu)", + "providers": "none; no generated embeddings or summaries", + "support_acceptance": "pending; observed verification environment only" + }, + "retained_source_test_logs": [ + { + "file": "codemesh-readonly-output-tests.log", + "sha256": "c1907bde6a2c561e97cf9e7a5b6573d0cc7e7bab4eda1516870e02fe4f783951" + }, + { + "file": "codemesh-feedback-corrected-tests.log", + "sha256": "7549e5e03206a491270c8f935cae98275a36969e6f4cc6f7944dadcf16db7ae1" + } + ] +} diff --git a/docs/evaluation/evidence/search-scope-correction-4fb9e81.json b/docs/evaluation/evidence/search-scope-correction-4fb9e81.json new file mode 100644 index 0000000..cb36446 --- /dev/null +++ b/docs/evaluation/evidence/search-scope-correction-4fb9e81.json @@ -0,0 +1,92 @@ +{ + "schema_version": "codemesh-search-scope-correction-v1", + "evidence_class": "provider-free-live-retrieval-with-controlled-synthetic-store-load", + "baseline_candidate": "2879e7f395a5f24ed3643042aae8d332f333d3d1", + "correction_candidate": "4fb9e813bb9bdcee5128c21476d709f99790acd5", + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "provider": "none", + "same_target_snapshot_and_50000_unrelated_nodes": true, + "target_reindexed_between_measurements": false, + "warmup": 1, + "timing_repetitions": 5, + "ordered_candidates_identical_in_all_adaptive_cases": true, + "comparison": { + "baseline_task_shaped_p50_ms": 1504.467, + "correction_task_shaped_p50_ms": 391.808, + "package_reduction_percent": 73.96, + "baseline_lexical_fetch_p50_ms": 1227.914, + "correction_lexical_fetch_p50_ms": 124.113, + "lexical_fetch_reduction_percent": 89.89 + }, + "gates": { + "youtube-downloader-impact-adaptive-live": { + "passed": true, + "comparable": true, + "summary": { + "case_count": 5, + "tool_call_count": 25, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.875, + "mean_ndcg_at_k": 0.862253, + "secret_leaks": 0 + }, + "report_sha256": "b64a11a063e3533cb38601de063376c4f35ef80dc3c531088a9198013e4cf6a1" + }, + "youtube-downloader-impact-live": { + "passed": true, + "comparable": true, + "summary": { + "case_count": 2, + "tool_call_count": 6, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 0.891323, + "secret_leaks": 0 + }, + "report_sha256": "50f118f4bb3acf2f2529717e14357a37ba92510b57a79901b7b2f234276151e9" + }, + "youtube-downloader-live": { + "passed": true, + "comparable": true, + "summary": { + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0 + }, + "report_sha256": "cd8d667cc9fba1fb09c17cea0196576a9ed2e3a194d9aeb8be06e318f264f7a6" + } + }, + "verification": { + "python_tests": "154 passed; two existing dependency warnings", + "ruff_check_and_format": "passed", + "deterministic_mcp_fixtures": "5 passed", + "end_to_end_smoke": "passed after correction in dedicated local Docker project", + "dotnet": "70 tests passed including configured sample, locked restore/build/format passed during instrumentation preparation; .NET source and tests unchanged by either performance commit; one existing CS8625 warning", + "repository_standards": "passed" + }, + "claim_ceiling": { + "established": [ + "scope-first lexical query reduces controlled unrelated-repository scoring overhead", + "unchanged frozen retrieval quality and candidate ordering", + "local provider-free integration and safety" + ], + "not_established": [ + "cause of historical 16.7-second store latency", + "configured-agent adoption or benefit for this correction", + "repeatable product benefit", + "hosted security findings resolved", + "release or deployment acceptance" + ] + }, + "limitations": [ + "Synthetic unrelated nodes model one store-scaling mechanism only.", + "Historical and new isolated-store snapshots are distinct; their absolute timings are not an exact before/after comparison.", + "The frozen external sample retained two dependency-advisory parser warnings; its source and dependencies were not edited." + ] +} diff --git a/docs/evaluation/evidence/search-scope-diagnosis-2879e7f.json b/docs/evaluation/evidence/search-scope-diagnosis-2879e7f.json new file mode 100644 index 0000000..2cc17e4 --- /dev/null +++ b/docs/evaluation/evidence/search-scope-diagnosis-2879e7f.json @@ -0,0 +1,286 @@ +{ + "schema_version": "codemesh-search-scope-diagnosis-v1", + "evidence_class": "provider-free-live-retrieval-and-controlled-synthetic-store-scaling", + "codemesh_candidate": "2879e7f395a5f24ed3643042aae8d332f333d3d1", + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "snapshot_id": "snp_c79b052ddc9d5ae22edc2cd46fa6eb2b653699e0f0c2c87b1226cd00b9528534", + "clean_tool_and_repository_checkouts": true, + "provider": "none", + "summaries": 0, + "embeddings": 0, + "warmup": 1, + "repetitions": 5, + "isolation": "Dedicated codemesh-completion-20260905 Docker project and newly created volumes; existing CodeMesh and one|nine stores were not reused.", + "fixture": { + "target_nodes": 1685, + "unrelated_nodes": 50000, + "unrelated_repository_id": "__codemesh_completion_scale_20260905", + "relationships": 0, + "language": "python", + "kind": "Method", + "name": "synthetic_download_method", + "file_path_pattern": "synthetic/module_<0..49999>.py", + "metadata_description": "Synthetic unrelated repository download settings preferences runtime source. Synthetic unrelated repository download settings preferences runtime source. Synthetic unrelated repository download settings preferences runtime source. Synthetic unrelated repository download settings preferences runtime source. " + }, + "reports": { + "adaptive.json": { + "sha256": "0a56c80eaf3ea8cc15e3140fd597e973d18e86d37b78a73db3d17b644a224912" + }, + "adaptive-50k.json": { + "sha256": "ce781c93b42bb46c767fbdfa5e4cb3bf765f15ad00acbc2ac7281564d5e0de53" + }, + "query-profile.json": { + "sha256": "e22a349b4208a7d9f65c729172f1ad4584a131143449da99aefe3e825fbb335e" + }, + "query-profile-50k.json": { + "sha256": "e191f23614c43df2569bf9872ca8f7c8478b7fa5f85161ff1c3e3e945f192769" + } + }, + "single_repository": { + "passed": true, + "comparable": true, + "retrieval": { + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.875, + "mean_ndcg_at_k": 0.862253, + "secret_leaks": 0 + }, + "task_shaped_timing": { + "calls": 5, + "successful_calls": 5, + "p50_total_ms": 451.07, + "p50_stages_ms": { + "assembly": 0.317, + "item_hydration": 91.297, + "repository_metadata": 1.554, + "repository_resolution": 1.054, + "search": 356.856 + }, + "p50_operations_ms": { + "content_fallback": 3.792, + "node": 38.558, + "relationships": 40.154, + "search_content": 28.751, + "search_declaration": 56.442, + "search_expansion": 86.911, + "search_lexical_fetch": 103.202, + "search_lexical_pipeline": 354.955, + "search_lexical_selection": 58.239, + "search_ranking": 0.145, + "search_repository_resolution": 1.475, + "search_summary_fetch": 0.892, + "search_summary_pipeline": 0.902, + "summary": 8.261 + }, + "counts": { + "content_fallbacks": 12, + "hits": 12, + "hydration_concurrency": 1, + "search_content_calls": 48, + "search_declaration_calls": 48, + "search_expansion_calls": 48, + "search_lexical_fetch_calls": 1, + "search_lexical_pipeline_calls": 1, + "search_lexical_selection_calls": 1, + "search_ranking_calls": 1, + "search_repository_resolution_calls": 1, + "search_summary_fetch_calls": 1, + "search_summary_pipeline_calls": 1 + } + } + }, + "with_unrelated_nodes": { + "passed": true, + "comparable": true, + "retrieval": { + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.875, + "mean_ndcg_at_k": 0.862253, + "secret_leaks": 0 + }, + "task_shaped_timing": { + "calls": 5, + "successful_calls": 5, + "p50_total_ms": 1504.467, + "p50_stages_ms": { + "assembly": 0.276, + "item_hydration": 76.25, + "repository_metadata": 1.492, + "repository_resolution": 1.058, + "search": 1425.164 + }, + "p50_operations_ms": { + "content_fallback": 3.804, + "node": 30.123, + "relationships": 34.506, + "search_content": 20.356, + "search_declaration": 34.915, + "search_expansion": 59.392, + "search_lexical_fetch": 1227.914, + "search_lexical_pipeline": 1423.074, + "search_lexical_selection": 58.205, + "search_ranking": 0.133, + "search_repository_resolution": 1.829, + "search_summary_fetch": 0.938, + "search_summary_pipeline": 0.947, + "summary": 7.512 + }, + "counts": { + "content_fallbacks": 12, + "hits": 12, + "hydration_concurrency": 1, + "search_content_calls": 48, + "search_declaration_calls": 48, + "search_expansion_calls": 48, + "search_lexical_fetch_calls": 1, + "search_lexical_pipeline_calls": 1, + "search_lexical_selection_calls": 1, + "search_ranking_calls": 1, + "search_repository_resolution_calls": 1, + "search_summary_fetch_calls": 1, + "search_summary_pipeline_calls": 1 + } + } + }, + "query_experiment": [ + { + "name": "baseline", + "duration_ms": 1388.115, + "candidate_count": 192, + "ordered_candidates_sha256": "cdf4a6511a7d33168207f75230734063f07a3c687845af6f1ddb77864bbd1e08", + "plan": { + "operatorType": "ProduceResults@neo4j", + "rows": 192, + "dbHits": 0, + "children": [ + { + "operatorType": "Projection@neo4j", + "rows": 192, + "dbHits": 2688, + "children": [ + { + "operatorType": "Top@neo4j", + "rows": 192, + "dbHits": 0, + "children": [ + { + "operatorType": "Projection@neo4j", + "rows": 1685, + "dbHits": 0, + "children": [ + { + "operatorType": "Filter@neo4j", + "rows": 1685, + "dbHits": 51685, + "children": [ + { + "operatorType": "Projection@neo4j", + "rows": 51685, + "dbHits": 0, + "children": [ + { + "operatorType": "CacheProperties@neo4j", + "rows": 51685, + "dbHits": 310110, + "children": [ + { + "operatorType": "NodeByLabelScan@neo4j", + "rows": 51685, + "dbHits": 51686, + "children": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + }, + { + "name": "scoped-query-only", + "duration_ms": 180.716, + "candidate_count": 192, + "ordered_candidates_sha256": "cdf4a6511a7d33168207f75230734063f07a3c687845af6f1ddb77864bbd1e08", + "plan": { + "operatorType": "ProduceResults@neo4j", + "rows": 192, + "dbHits": 0, + "children": [ + { + "operatorType": "Projection@neo4j", + "rows": 192, + "dbHits": 2688, + "children": [ + { + "operatorType": "Top@neo4j", + "rows": 192, + "dbHits": 0, + "children": [ + { + "operatorType": "Projection@neo4j", + "rows": 1685, + "dbHits": 0, + "children": [ + { + "operatorType": "Filter@neo4j", + "rows": 1685, + "dbHits": 0, + "children": [ + { + "operatorType": "Projection@neo4j", + "rows": 1685, + "dbHits": 0, + "children": [ + { + "operatorType": "CacheProperties@neo4j", + "rows": 1685, + "dbHits": 10110, + "children": [ + { + "operatorType": "Filter@neo4j", + "rows": 1685, + "dbHits": 51685, + "children": [ + { + "operatorType": "NodeByLabelScan@neo4j", + "rows": 51685, + "dbHits": 51686, + "children": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + } + ], + "measured_decision": "Scope the initial MATCH by repository before lexical property projection. Preserve unbound reads, parameterization, scoring, ordering, and budgets. No hydration concurrency or candidate-budget change.", + "limitations": [ + "Synthetic unrelated load is a controlled mechanism test, not an agent-benefit campaign.", + "The isolated store did not reproduce historical 16.7-second latency; historical causal attribution remains unproven.", + "Query-profile comparison is a one-shot diagnostic; candidate live-suite verification must follow implementation.", + "External sample parsing retained two CMSHARP015 advisory warnings for its frozen AngleSharp dependency; the sample was not edited.", + "The new isolated snapshot differs from earlier retained store snapshots; only the within-experiment comparisons share this store identity." + ] +} diff --git a/docs/evaluation/evidence/security-boundary-review-20260905.json b/docs/evaluation/evidence/security-boundary-review-20260905.json new file mode 100644 index 0000000..5acb8d9 --- /dev/null +++ b/docs/evaluation/evidence/security-boundary-review-20260905.json @@ -0,0 +1,122 @@ +{ + "schema_version": "codemesh-security-boundary-review-v1", + "recorded_at": "2026-09-05", + "evidence_class": "local-source-review-and-provider-free-regression-tests", + "base_commit": "32579cd", + "hosted_scan_commit": "8e9c0da7b497fe51126e3ef66509a8587cfe8d83", + "hosted_open_alerts_rechecked": 39, + "groups": [ + { + "alerts": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 22, + 23, + 24, + 25, + 26 + ], + "boundary": "suite task, question, scenario, patch and generated artifacts", + "action": "Safe identifiers, case-insensitive uniqueness, suite-relative resolved patch containment, generated descendant containment, exclusive raw directories; validate all artifacts before preparation and revalidate individual execution paths." + }, + { + "alerts": [ + 1, + 2, + 3, + 35 + ], + "boundary": "suite argv and explicitly selected executable", + "action": "Preserve shell-free subprocess.run/create_subprocess_exec; document executable and suite trust. Real subprocess metacharacter test and Codex launch-argument regression passed. Executable paths are intentionally operator-selected." + }, + { + "alerts": [ + 39 + ], + "boundary": "five local HTML routes", + "action": "Source review and four adversarial payloads across every route verify escaped request and stored text, attributes and local URLs. No production rendering change or hosted dismissal." + }, + { + "alerts": [ + 4, + 5, + 6, + 18, + 19, + 20, + 21 + ], + "boundary": "evaluation output root, default run name, fixed report/review files and metadata", + "action": "Explicit output root remains operator-selected and created exclusively. Additional review found an unvalidated suite name in the model benchmark default run name; agent/model suite names now use safe identifiers and pre-execution validation. Fixed filenames and resolved metadata do not supply new destinations." + }, + { + "alerts": [ + 15 + ], + "boundary": "Git repository root resolution", + "action": "Explicit operator-selected repository root passed as one argv value to git -C, then resolved through rev-parse; no managed-directory containment promised." + }, + { + "alerts": [ + 16, + 30 + ], + "boundary": "response output and JSON response/audit input", + "action": "Response filename comes from evaluator-created private temporary directory; audit path is a managed artifact descendant. Model tool accepts answer/facts/citations, not a destination. Internal read helper receives these evaluator-created paths." + }, + { + "alerts": [ + 17 + ], + "boundary": "Conventional Commit message input", + "action": "Explicit local --message-file path is intentionally read-only operator input; no directory restriction promised." + }, + { + "alerts": [ + 27, + 28, + 29, + 31 + ], + "boundary": "selected model report/review input and finalized report output", + "action": "Operator-selected input and output files. Default reviewed-report.json is a fixed sibling of selected input; report metadata suite_path is also trusted input and documented. JSON/Pydantic parsing does not execute input." + }, + { + "alerts": [ + 32, + 33, + 34 + ], + "boundary": "CLI --output", + "action": "Explicit operator-selected report destination, including parent creation where documented; no implied repository-only destination." + }, + { + "alerts": [ + 36, + 37, + 38 + ], + "boundary": "built-in suite selection and explicit suite files", + "action": "Built-in names now validated before constructing packaged suite path; existing custom file paths remain operator-selected. Loaded agent/model suite artifacts are validated before return." + } + ], + "validation": { + "python_tests": 227, + "python_failures": 0, + "python_skipped": 0, + "existing_dependency_warnings": 2, + "dotnet": "Not rerun for Python-only evaluator validation and UI tests; no cross-language contract changed." + }, + "limits": [ + "No hosted scan of these changes or alert dismissal.", + "No claim that trusted local suites or selected executables are sandboxed.", + "No defense claimed against concurrent hostile local filesystem mutation.", + "No provider invocation, spend, release, or deployment." + ] +} diff --git a/docs/evaluation/evidence/youtube-downloader-task-adaptive-provider-free-fbc1433.json b/docs/evaluation/evidence/youtube-downloader-task-adaptive-provider-free-fbc1433.json new file mode 100644 index 0000000..af58bb8 --- /dev/null +++ b/docs/evaluation/evidence/youtube-downloader-task-adaptive-provider-free-fbc1433.json @@ -0,0 +1,157 @@ +{ + "schema_version": "codemesh-youtube-downloader-task-adaptive-provider-free-v1", + "evidence_class": "provider-free-local-configured-integration", + "recorded_at": "2026-09-01T15:45:31Z", + "codemesh": { + "commit": "fbc1433642ad9e20acd3379554f2ee09a6d635d9", + "clean_detached_checkout": true, + "adaptive_suite_sha256": "40b4d994ee704c7db22a973538c0a34f3bb28b6cab305857d553f2928f392ea8", + "python_tests_passed": 125, + "python_test_warnings": 2, + "ruff_check": "passed", + "ruff_format": "passed", + "fixture_suite_cases_passed": 5, + "dotnet_verification": "not_selected_python_guidance_and_evaluation_suite_change" + }, + "youtube_downloader": { + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "branch": "HEAD", + "tracked_files_changed": false, + "project_id": "prj_3425db238d0d4a5fbcc1e3c9a28d1945", + "checkout_id": "chk_7e4ecde2ae4e45b4b9e475e0a587aa47", + "snapshot_id": "snp_1c7772010bcb7c8430fe062d975094271762a68627dbc53f4e2b9f0557687eae", + "source_view_hash": "d741906fcdcc5aa33a9d14f8a5e0cbdf7d2e572266a098beadddb595ce6853d9", + "freshness_status": "fresh" + }, + "diagnosis": { + "record": "youtube-downloader-task-adaptive-query-diagnosis-e2e1bca.json", + "task_shaped_required_target_coverage": "6_of_6", + "focused_set_required_target_coverage": "6_of_6", + "task_shaped_package_calls": 1, + "focused_package_calls": 3, + "task_shaped_agent_formatted_characters": 12000, + "focused_agent_formatted_characters": 35991, + "focused_duplicate_item_occurrences": 5, + "decision": "prefer_one_package_and_decompose_only_a_specific_unresolved_facet" + }, + "configured_installation": { + "profile": "normal", + "provider": "none", + "configuration_only": true, + "plan_hash": "dfd774405b9895fa8427ad4af0817049c8eab4ff39982ead1395a09ba3891e92", + "plan_report_sha256": "a9d94c880e4b8c8fe474f7587656756e3c084244eaa80a62aff0add0b2cfebd0", + "launch_sha256": "bfa3df955bf09a066680345866f919978c165ef9d1be9b168e88ddc77f32a425", + "installation_file": { + "path": ".codex/config.toml", + "source": "installed-ignored", + "sha256": "80bff01aa66dcf6baf4620477c9a98ed8acfb9fe653e9d55326e38030465421c" + }, + "guidance": { + "path": "AGENTS.override.md", + "source": "installed-ignored", + "sha256": "bd356195afbb8b945d003ff7b6802e270a8c6a97566697ce95b1c4a7c980aba3" + }, + "expected_tools": [ + "codemesh_get_context_package", + "codemesh_get_repository_status", + "codemesh_list_repositories", + "codemesh_get_node" + ] + }, + "configured_preflight": { + "passed": true, + "integration_mode": "configured", + "prompt_parity": true, + "provider": "none", + "probe_query": "audio-only container VideoDownloadOption DownloadMultipleSetupViewModel", + "expected_paths": [ + "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs" + ], + "returned_path_count": 8, + "wrong_checkout_rejected": true, + "positive_probe_sha256": "fbff77ffdc1c860bb2241b8e9acb2d4ac587b5fc69f2ae512b1d24fb39d76281", + "rejection_probe_sha256": "24e652f49c397e407632fba3d89cc8bc43a4f1c427a5f3bda8842be5f51f9209", + "report_sha256": "6a61b4c3996637157712ff487cc1390c9213749effe6568eb128fd0811680527" + }, + "adaptive_live_gate": { + "suite": "youtube-downloader-impact-adaptive-live", + "passed": true, + "comparable": true, + "case_count": 5, + "tool_call_count": 15, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.875, + "mean_ndcg_at_k": 0.862253, + "task_shaped_primary_rank": 2, + "task_shaped_p50_latency_ms": 16839.529, + "focused_p50_latency_ms": [ + 5429.193, + 7049.534, + 5428.08 + ], + "secret_leaks": 0, + "model_providers": [], + "report_sha256": "a483a8f4e83a1354df492778f1d49d86f202a13dbf2ff4f773638557480ef74f" + }, + "unchanged_impact_live_gate": { + "suite": "youtube-downloader-impact-live", + "passed": true, + "comparable": true, + "case_count": 2, + "tool_call_count": 6, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 1.0, + "mean_ndcg_at_k": 0.891323, + "primary_rank": 2, + "secret_leaks": 0, + "model_providers": [], + "report_sha256": "96b260c33d4a6c271ed182d4d539bda726048dde6584bbb34aeab98785795fbf" + }, + "unchanged_canonical_live_gate": { + "suite": "youtube-downloader-live", + "passed": true, + "comparable": true, + "case_count": 6, + "tool_call_count": 18, + "tool_success_rate": 1.0, + "mean_recall_at_k": 1.0, + "mean_mrr": 0.9, + "mean_ndcg_at_k": 0.926186, + "secret_leaks": 0, + "model_providers": [ + "none" + ], + "report_sha256": "5396821051adf196cd370b933eefcdfaf9cdfc833f9b3c826d33297ef10d238b" + }, + "claim_ceiling": { + "established": [ + "provider_free_configured_plan_and_runtime_identity", + "prompt_parity_preflight_configuration", + "fresh_fail_closed_youtube_downloader_binding", + "comparable_task_adaptive_retrieval_gate", + "unchanged_impact_retrieval_gate_preserved", + "unchanged_canonical_retrieval_gate_preserved" + ], + "not_established": [ + "normal_agent_compliance_with_task_adaptive_guidance", + "codemesh_correctness_or_efficiency_benefit", + "causality_for_the_retained_efficiency_regression", + "repeatable_configured_product_benefit", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "boundaries": { + "provider_free_gates_invoked_model": false, + "paid_campaign_started": false, + "another_paid_campaign_requires_fresh_explicit_model_spend_authorization": true, + "automatic_paid_retry_authorized": false, + "additional_onenine_checkout_onboarding": "not_authorized_by_this_work", + "push": "not_performed", + "release": "not_performed", + "deployment": "not_performed" + } +} diff --git a/docs/evaluation/evidence/youtube-downloader-task-adaptive-query-diagnosis-e2e1bca.json b/docs/evaluation/evidence/youtube-downloader-task-adaptive-query-diagnosis-e2e1bca.json new file mode 100644 index 0000000..b836489 --- /dev/null +++ b/docs/evaluation/evidence/youtube-downloader-task-adaptive-query-diagnosis-e2e1bca.json @@ -0,0 +1,124 @@ +{ + "schema_version": "1.0", + "evidence_kind": "sanitized-provider-free-task-adaptive-query-diagnosis", + "generated_at": "2026-09-01T15:34:07.294880+00:00", + "frozen_boundary": { + "codemesh_base_commit": "e2e1bca70b1bc7d060942f147e45359591e74ccb", + "codemesh_source_modified_for_diagnosis": true, + "repository_url": "https://github.com/Tyrrrz/YoutubeDownloader.git", + "repository_commit": "05e63cddb6d2a96fc2d21b097129a0765824f251", + "repository_working_tree_dirty": false, + "project_id": "prj_3425db238d0d4a5fbcc1e3c9a28d1945", + "checkout_id": "chk_7e4ecde2ae4e45b4b9e475e0a587aa47", + "snapshot_id": "snp_1c7772010bcb7c8430fe062d975094271762a68627dbc53f4e2b9f0557687eae", + "source_view_hash": "d741906fcdcc5aa33a9d14f8a5e0cbdf7d2e572266a098beadddb595ce6853d9", + "freshness_status": "fresh", + "provider": "none", + "required_target_count": 6, + "thresholds": { + "min_tool_success_rate": 1.0, + "min_recall_at_k": 0.85, + "min_mrr": 0.75, + "max_primary_rank": 3, + "max_secret_leaks": 0 + } + }, + "comparison_suite": { + "name": "youtube-downloader-impact-adaptive-live", + "definition_status": "uncommitted_diagnostic_definition", + "definition_sha256": "40b4d994ee704c7db22a973538c0a34f3bb28b6cab305857d553f2928f392ea8", + "repetitions": 1, + "warmup_calls": 0, + "provider_free": true, + "tool_success_rate": 1.0, + "secret_leaks": 0, + "raw_report_sha256": "ba740259283e38ed26ff820fa88ee130d9b0a3db75a09a8996bcc095f1db3cec" + }, + "task_shaped_package": { + "query_term_count": 29, + "call_count": 1, + "limit": 12, + "maximum_characters": 12000, + "required_targets_covered": 6, + "required_target_count": 6, + "recall_at_k": 1.0, + "reciprocal_rank": 1.0, + "ndcg_at_k": 0.891323, + "primary_rank": 2, + "target_ranks": { + "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs": 2, + "YoutubeDownloader.Core/Downloading/VideoDownloadPreference.cs": 1, + "YoutubeDownloader/ViewModels/Dialogs/DownloadMultipleSetupViewModel.cs": 8, + "YoutubeDownloader/ViewModels/Dialogs/DownloadSingleSetupViewModel.cs": 7, + "YoutubeDownloader/Services/SettingsService.cs": 5, + "YoutubeDownloader/ViewModels/Components/DashboardViewModel.cs": 11 + }, + "latency_ms": 17177.121, + "ranked_item_occurrences": 12, + "snippet_characters": 11173, + "agent_formatted_characters": 12000, + "duplicate_item_occurrences": 0 + }, + "focused_package_set": { + "query_count": 3, + "query_term_count_total": 22, + "call_count": 3, + "per_package_limit": 8, + "per_package_maximum_characters": 12000, + "required_targets_covered": 6, + "required_target_count": 6, + "target_best_ranks": { + "YoutubeDownloader.Core/Downloading/VideoDownloadOption.cs": 1, + "YoutubeDownloader.Core/Downloading/VideoDownloadPreference.cs": 2, + "YoutubeDownloader/ViewModels/Dialogs/DownloadMultipleSetupViewModel.cs": 1, + "YoutubeDownloader/ViewModels/Dialogs/DownloadSingleSetupViewModel.cs": 2, + "YoutubeDownloader/Services/SettingsService.cs": 5, + "YoutubeDownloader/ViewModels/Components/DashboardViewModel.cs": 2 + }, + "latency_ms_total": 18669.373, + "ranked_item_occurrences": 24, + "unique_ranked_items": 19, + "duplicate_item_occurrences": 5, + "duplicate_item_fraction": 0.208333, + "duplicate_snippet_characters": 5995, + "snippet_characters_total": 29270, + "agent_formatted_characters_total": 35991 + }, + "decision": { + "preferred_entry": "one_task_shaped_context_package", + "stop_condition": "the_first_package_covers_every_required_facet", + "decomposition_condition": "a_specific_implementation_or_test_facet_remains_unresolved", + "maximum_context_packages_per_task": 3, + "overlapping_follow_up_queries_allowed": false, + "smallest_correction": "replace_proactive_multi_query_guidance_with_task_adaptive_stop_and_decompose_guidance" + }, + "claim_ceiling": { + "established": [ + "one_frozen_task_shaped_package_covers_all_six_required_targets", + "three_proactive_focused_packages_also_cover_all_six_targets_but_repeat_five_ranked_items", + "the_focused_set_uses_three_times_the_agent_formatted_package_budget", + "the_bounded_guidance_correction_is_supported_for_clean_candidate_verification" + ], + "not_established": [ + "exact_clean_candidate_verification", + "normal_agent_compliance_with_task_adaptive_guidance", + "correctness_or_efficiency_benefit", + "causality_for_the_retained_paid_campaign_regression", + "release_readiness", + "deployment_or_production_acceptance" + ] + }, + "limitations": [ + "The diagnosis used one measured repetition and no warmup because it compares deterministic lexical retrieval shapes rather than latency distributions.", + "The CodeMesh source checkout contained the diagnostic suite and guidance edits, so the report is intentionally non-comparable as exact-candidate evidence.", + "The retained paid campaign did not preserve raw context-package arguments or payloads, so this diagnosis does not establish the cause of its efficiency regression." + ], + "boundaries": { + "model_provider_invoked": false, + "paid_campaign_started": false, + "clean_candidate_verification": "pending", + "push": "not_performed", + "release": "not_performed", + "deployment": "not_performed" + } +} diff --git a/docs/evaluation/mcp-effectiveness.md b/docs/evaluation/mcp-effectiveness.md index b26abad..5f5943e 100644 --- a/docs/evaluation/mcp-effectiveness.md +++ b/docs/evaluation/mcp-effectiveness.md @@ -16,8 +16,8 @@ summary generation is not equivalent to autonomous MCP tool use or repair ability. [Summary Model Qualification](../planning/summary-model-qualification.md) defines a provider-neutral procedure for local and online candidates, including factual accuracy, reliability, speed, token efficiency, privacy, cost, and downstream -retrieval gates. The dedicated qualification runner described there is planned -and is not an available CLI command yet. +retrieval gates. The dedicated qualification runner and report compiler are +implemented, but no summary deployment profile is CodeMesh-qualified. ## Product Decision Role @@ -37,17 +37,49 @@ The evidence position is summarized in [Project Status](../current/project-statu - the independent prompt-parity Config.Net campaign completed three pairs but recorded zero CodeMesh calls or call attempts, so it is negative discoverability evidence and its overall efficiency variance is not - attributable to CodeMesh; and + attributable to CodeMesh; +- the first configured Config.Net campaign achieved full adoption but regressed + correctness in two treatments; its separately corrected campaign improved + with one attributable win, but that bounded result is not repeatable benefit; +- the first independent configured YoutubeDownloader campaign preserved full + correctness and adoption but regressed duration and token efficiency; +- the `fbc1433` task-adaptive campaign preserved full correctness and adoption, + reduced each treatment to one package and median tokens by 6.63%, but + regressed median duration by 10.0192% and exceeded its authorized aggregate + reported-token ceiling because the historical evaluator lacked a hard stop; + and - no summary model is qualified, so the core baseline remains summary-free. Explicit repository onboarding is now the selected v1 product contract; see [Agent Integration Contract](../planning/agent-integration-contract.md). The -current product-proof work is to implement that integration and evaluate the -exact guidance, tool profile, checkout binding, and setup process shipped to -normal users. Preserve the same independent task, prompt, grading targets, and -control behavior. The retained zero-adoption campaign remains negative -spontaneous-discovery evidence and must not be rewritten as a transport defect -or configured-product result. +checkout-local integration and provider-free probes are implemented and passed +their clean-candidate gates. The paired-agent evaluator now represents that +configured integration. The one-step context-package entry contract and its +exact-candidate preflight, frozen one|nine gates, and YoutubeDownloader +provider-free gate pass at candidate `8844c21`. The first configured campaign, +retained at candidate `0bde606`, completed all three pairs correctly and safely +but was insufficient: two treatments called repository status, none retrieved a +context package, and there were no attributable wins. The separately authorized +campaign through corrected candidate `0378740` was also correct and safe but +insufficient: one treatment called status, none retrieved context, and there +were no attributable wins. The one-step `8844c21` campaign then completed all +three pairs correctly and safely with context-package adoption in every +treatment. Its verdict was `neutral`, with no attributable wins or regressions; +median duration was 1.39% lower and median tokens were 4.13% higher. Preserve +all three results and their frozen task, prompt, grading targets, and control +behavior. The separate Config.Net zero-adoption prompt-parity campaign remains +negative spontaneous-discovery evidence and must not be rewritten as a +transport defect or configured-product result. + +The current agent-campaign path now fails closed on reported-token budgeting. +It defines reported tokens as input plus output plus reasoning-output tokens, +requires a runner-declared compatible hard cap before repository preflight, +passes the remaining allowance to every execution, stops at the boundary or an +incomplete execution, and never retries automatically. Cached input is already +part of input tokens and is not counted again. The default `codex` runner +cannot guarantee this contract and is refused before any model call. Explicit +`capped-codex` implements it; its consumed credit-blocked campaign is retained +below without product-benefit claims. Reports must compare correctness, safety, freshness, citations, elapsed time, tokens, tool calls, adoption, and correction loops. A tied-correctness result is @@ -66,11 +98,11 @@ Classify agent evidence by the integration the treatment actually receives: treatment receives the exact repository guidance, tool profile, checkout binding, and setup process shipped to normal users. -The current harness records spontaneous and assisted modes. Extend its report -schema before claiming configured/onboarded evidence so the installed guidance -and profile identities are retained. Never relabel earlier assisted evidence as -configured product evidence. If adoption remains zero, retain that negative -result rather than weakening the comparison. +The harness records spontaneous, assisted, and configured modes. Configured +reports retain installed guidance, baseline MCP, profile, binding, plan, and +probe identities. Never relabel earlier assisted evidence as configured product +evidence. If adoption remains zero, retain that negative result rather than +weakening the comparison. ## Representative External C# Baseline @@ -93,6 +125,15 @@ dotnet run --project src/CodeMesh.Cli -- ingest ` --skip-embeddings ``` +Cross-project semantic relationships also require restored sample dependencies. +The [fresh-clone upgrade rehearsal](evidence/local-development-upgrade-ce23f2e.json) +passed source discovery but failed relationship coverage without those assets. +Run `dotnet restore references/YoutubeDownloader/YoutubeDownloader.slnx --locked-mode` +and retain its result before ingestion. The frozen sample currently reports the +AngleSharp advisory `GHSA-pgww-w46g-26qg` as an error; do not suppress that gate or +change the pinned source to manufacture a pass. Source analysis using emitted +assets can be inspected separately, but does not make dependency acceptance pass. + Explicit `--solution` and `--project` paths may be absolute, relative to the repository root, or relative to the directory that launches the CLI. A missing explicit file fails ingestion instead of silently falling back to filesystem @@ -116,7 +157,8 @@ source/index freshness. After it passes, run the paired agent comparison: uv run python -m codemesh_agent_access eval agent ` --suite youtube-downloader-agent ` --repository-root ..\references\YoutubeDownloader ` - --model + --model ` + --max-reported-tokens ``` The three task families are implementation discovery, likely change impact, @@ -135,14 +177,23 @@ uv run python -m codemesh_agent_access eval live ` --suite youtube-downloader-impact-live ` --output ..\.codemesh-evals\youtube-downloader-impact-live\report.json +uv run python -m codemesh_agent_access eval live ` + --suite youtube-downloader-impact-adaptive-live ` + --output ..\.codemesh-evals\youtube-downloader-impact-adaptive-live\report.json + uv run python -m codemesh_agent_access eval agent ` --suite youtube-downloader-impact-agent ` --repository-root ..\references\YoutubeDownloader ` - --model + --model ` + --max-reported-tokens ``` Keep these suites separate from the original live and three-task agent suites so -their outcomes do not rewrite the frozen baseline. +their outcomes do not rewrite the frozen baseline. The adaptive live suite is +provider-free: it compares the frozen task-shaped package with three bounded +focused facets while preserving the same repository commit, six targets, and +retrieval thresholds. It is a retrieval and package-budget gate, not a paired +agent campaign. If the spontaneous complex campaign has zero adoption, the separate `youtube-downloader-impact-assisted-agent` suite adds transparent treatment-only @@ -156,12 +207,21 @@ substitute for spontaneous-discovery evidence: uv run python -m codemesh_agent_access eval agent ` --suite youtube-downloader-impact-assisted-agent ` --repository-root ..\references\YoutubeDownloader ` - --model + --model ` + --max-reported-tokens ``` +The `youtube-downloader-impact-configured-agent` suite preserves the same +frozen prompt, targets, thresholds, and source commit while setting +`integration_mode = "configured"`. It uses the reviewed repository-owned +normal installation and guidance, keeps control and treatment prompts equal, +and forbids evaluator-only treatment guidance. Run its configured preflight and +retrieval gates before any separately authorized model campaign. + ## Independent Replication And Discoverability -The `config-net-cache-live` and `config-net-cache-assisted-agent` suites pin the +The `config-net-cache-live`, `config-net-cache-assisted-agent`, +`config-net-cache-agent`, and `config-net-cache-configured-agent` suites pin the MIT-licensed Config.Net repository at commit `ae0af7d4b3781c66233bbb677b6e11df1f5b416b`. This is a separate repository, author, architecture, and task family from YoutubeDownloader. The task traces @@ -192,7 +252,8 @@ uv run python -m codemesh_agent_access eval live ` uv run python -m codemesh_agent_access eval agent ` --suite config-net-cache-assisted-agent ` --repository-root ..\references\Config.Net ` - --model + --model ` + --max-reported-tokens ``` The assisted suite records `prompt_parity = false`; it can replicate utility @@ -217,7 +278,8 @@ including its canonical initialization instructions and tool contracts: uv run python -m codemesh_agent_access eval agent ` --suite config-net-cache-agent ` --repository-root ..\references\Config.Net ` - --model + --model ` + --max-reported-tokens ``` This campaign is model-cost-bearing. Run it only with explicit authorization @@ -234,12 +296,233 @@ to CodeMesh. Preserve this as negative discoverability evidence. The sanitized transcription is [`docs/evaluation/evidence/config-net-prompt-parity-647d83d.json`](evidence/config-net-prompt-parity-647d83d.json). -Before another paid run, implement and verify the selected explicit-onboarding -contract. The treatment must use the same repository guidance and normal-user -tool profile shipped outside the evaluator; do not create an evaluator-only -tool-selection advantage. Record the result as configured/onboarded, not -spontaneous or evaluator-assisted, and retain the prompt-parity failure -separately. +Exact clean candidate `b16eee7` adds the configured suite while preserving that +prompt, target set, threshold, and source commit. Its reviewed normal-profile +plan and ignored repository guidance passed the configured preflight with +prompt parity, a fresh exact binding, positive context retrieval, and +fail-closed rejection. The full six-case live gate then passed all 18 calls +with recall, MRR, and nDCG of 1.0 and zero secret leaks. No model provider was +invoked. The sanitized record is +[configured Config.Net provider-free evidence](evidence/configured-config-net-provider-free-b16eee7.json). +That gate also retained an earlier broad task-query diagnostic: it returned +tests but missed requested implementation paths, whereas the canonical +decomposed live cases all passed. This did not establish a root cause. + +The separately authorized configured campaign completed all three pairs with +context-package adoption in every treatment and no safety, MCP transport, or +command-policy failure. All three controls passed. Treatments passed one of +three: repetition one omitted `ConfigurationBuilder.cs`, and repetition two +omitted `LazyVar.cs`, `ConfigurationBuilder.cs`, `LogicTest.cs`, and +`ConfigurableMethodsTest.cs`. The verdict was `regressed`, with no attributable +wins. On the only jointly successful pair, treatment duration was 21.31% higher +and token use was 117.18% higher. The sanitized record is +[configured Config.Net agent evidence](evidence/configured-config-net-agent-b16eee7.json). +Preserve this consumed negative configured-product artifact and do not retry it +automatically. This result activated provider-free reproduction of full-task +context-package coverage against every required target, correlated with the +retained broad-query miss but without claiming a cause that raw campaign +packages and final messages could not establish. + +That provider-free diagnosis is now retained separately. The exact frozen +58-term task query returned eight test paths and only `LogicTest.cs` from the +seven required targets. Four concise task-derived queries covering cached +writes, setter routing, proxy interception, and cache expiration recovered all +seven targets. The separate `config-net-cache-focused-live` diagnostic passed +five cases and all 15 calls with recall 1.0, MRR 0.660714, nDCG 0.76259, and +zero secret leaks against clean `b16eee7`; the unchanged canonical suite also +passed all 18 calls. Its suite definition was uncommitted, so this validates the +retrieval strategy rather than an exact correction candidate. The sanitized +record is +[focused-query diagnosis](evidence/config-net-focused-query-diagnosis-b16eee7.json). + +Exact clean detached candidate `3804028` retains the context package as the +one-step entry tool but replaces one-long-full-task guidance with a small set of +focused implementation and test queries for multi-part work. Its 125 Python +tests, Ruff checks, and five fixture cases pass. The reviewed configuration-only +plan and ignored guidance passed configured preflight, a positive package +probe, and wrong-checkout rejection with provider mode `none`. The committed +focused suite passed five cases and 15 calls with recall 1.0, MRR 0.660714, +nDCG 0.76259, and zero leaks; `LazyVar.cs` remains at rank eight. The unchanged +canonical suite passed six cases and 18 calls with recall, MRR, and nDCG of 1.0 +and zero leaks. The sanitized exact-candidate record is +[focused-query provider-free evidence](evidence/config-net-focused-query-provider-free-3804028.json). +This is provider-free correction readiness, not normal-agent compliance or +product-benefit evidence. + +The separately authorized exact-candidate campaign retained the frozen task, +grading, repository, model, reasoning effort, seed, repetitions, normal profile, +and prompt parity. It completed all three pairs with an `improved` verdict. +Every treatment adopted the context package, made two or three package attempts, +and passed; controls passed two of three because repetition two missed +`ConfigurationBuilder.cs`. The report therefore records one +CodeMesh-attributable win, zero treatment regressions, and no safety, MCP, or +command-policy failures. Across the two jointly successful pairs, treatment +median duration was 13.69% lower and median tokens were 3.59% lower. The +sanitized record is +[focused-query configured campaign evidence](evidence/configured-config-net-agent-3804028.json). +This is positive configured evidence for one bounded campaign. Raw package +arguments and final messages were not retained, so it neither proves exact +query wording nor establishes causal or repeatable product benefit. + +Independent clean candidate `64d4052` added the configured YoutubeDownloader +impact suite while preserving the frozen task and repository identity. Its full +deterministic checks, reviewed configuration-only plan, configured preflight, +positive and wrong-checkout probes, two-case impact live gate, and unchanged +six-case live gate passed with provider mode `none`. The impact gate returned +all six targets with recall and MRR 1.0, nDCG 0.891323, and zero leaks; the +canonical gate retained recall 1.0, MRR 0.9, nDCG 0.926186, and zero leaks. The +sanitized record is +[configured YoutubeDownloader provider-free evidence](evidence/configured-youtube-downloader-provider-free-64d4052.json). + +The separately authorized campaign completed every control and treatment +correctly and safely, with context-package adoption in every treatment and no +failed MCP calls. Treatments attempted two, two, and three packages. There were +no correctness wins or regressions, while treatment median duration was 49.79% +higher and median tokens were 41.00% higher across all three matched pairs. The +efficiency threshold therefore produced a valid `regressed` verdict. The +sanitized record is +[configured YoutubeDownloader campaign evidence](evidence/configured-youtube-downloader-agent-64d4052.json). +This replicates configured adoption and correctness on an independent C# task, +not repeatable product benefit. Because raw query arguments were not retained, +the package-attempt count motivates provider-free adaptive-query diagnosis but +does not establish the regression's cause. + +That provider-free diagnosis preserves the frozen YoutubeDownloader task, six +targets, source identity, and thresholds. One 29-term task-shaped package +covered all six targets in one call with primary rank two and a 12,000-character +agent package. Three focused implementation/test-facet packages also covered +all six, but used 35,991 formatted characters across three calls and repeated +five of 24 ranked item occurrences. The observed one-call latency was +17,177.121 ms versus 18,669.373 ms summed across the three focused calls. The +one-repetition comparison ran while the new suite and guidance were uncommitted, +so it establishes the retrieval-shape decision rather than exact-candidate +readiness. The sanitized record is the +[task-adaptive query diagnosis](evidence/youtube-downloader-task-adaptive-query-diagnosis-e2e1bca.json). + +Exact clean detached candidate `fbc1433` implements the bounded correction: +start with one concise task-shaped package, stop when every required facet is +resolved, and issue a focused follow-up only for a specific unresolved +implementation or test facet, with at most three non-overlapping packages per +task. Its 125 Python tests, Ruff checks, five fixture cases, reviewed +normal-profile preflight, positive probe, and wrong-checkout rejection passed. +The committed adaptive suite passed all 15 calls with recall 1.0, MRR 0.875, +nDCG 0.862253, and zero leaks. The unchanged impact suite passed all six calls +with recall and MRR 1.0 and nDCG 0.891323; the unchanged canonical suite passed +all 18 calls with recall 1.0, MRR 0.9, and nDCG 0.926186. No model provider was +invoked. The sanitized record is +[task-adaptive provider-free evidence](evidence/youtube-downloader-task-adaptive-provider-free-fbc1433.json). + +The separately authorized configured campaign at the same exact candidate then +completed all three pairs and all six executions correctly and safely. Every +treatment made exactly one successful context-package call, with no MCP or +command-policy failure. There were no treatment wins, attributable wins, or +regressions. Median treatment tokens fell 6.63%, while median duration rose +10.0192%, exceeding the frozen 10% threshold by 0.0192 percentage points and +producing a `regressed` verdict. The six runs reported 2,596,056 aggregate +tokens, including 2,219,264 cached-input tokens, and exceeded the authorized +1.6M reported-token ceiling. The historical evaluator had no hard +aggregate-token stop; record that control failure and do not retry the consumed +campaign. Raw traces and exact query arguments were not retained. The sanitized record is +[task-adaptive configured campaign evidence](evidence/configured-youtube-downloader-agent-fbc1433.json). + +This establishes configured single-package adoption and correct/safe completion +for the frozen task. It does not establish overall efficiency benefit, +causality, repeatability, release readiness, or production fitness. The current +evaluator now has a tested fail-closed reported-token contract. The default +`codex` runner is refused before model calls; explicit `capped-codex` implements +the contract, and its consumed incomplete campaign is retained separately. +Any future paid campaign needs fresh candidate gates and separate authority. +The provider-free search correction is performance evidence only. + +Candidate `0bde606` passed the configured preflight for the selected explicit- +onboarding contract against exact Primary commit `fe2f761`. Its first paid run +completed three correct and safe pairs, with repository-status adoption in two +treatments, no context-package calls, no attributable wins, and verdict +`insufficient`. The sanitized record is +[configured one|nine agent evidence](evidence/configured-onenine-agent-0bde606.json). +Preserve that frozen identity and result. The corrected run used a new exact +clean candidate while keeping repository guidance, task prompt, grading, +normal-user tool profile, model settings, and control behavior fixed; it did not +create an evaluator-only tool-selection advantage. + +Clean detached candidate `0378740` contains the bounded status-to-context +correction and passed the full deterministic checks, configured preflight, all +eight frozen one|nine retrieval and rejection gates, all three retained feedback +packet validations, and the pinned YoutubeDownloader live gate with provider +mode `none`. The sanitized record is +[corrected provider-free evidence](evidence/configured-onenine-provider-free-0378740.json). +This established readiness for the separately authorized model campaign; it did +not establish agent context adoption or product benefit. + +The authorized `0378740` campaign completed three correct and safe pairs with +prompt parity and the exact reviewed normal-profile installation. Only one +treatment called repository status, no treatment called the context-package +tool, and there were no attributable wins. Its verdict was `insufficient`. +Overall treatment medians were 6.07% slower and used 4.17% more tokens; those +differences are diagnostic because context retrieval never occurred. The +sanitized record is +[corrected configured campaign evidence](evidence/configured-onenine-agent-0378740.json). +Preserve this consumed artifact and do not retry it automatically. + +The subsequent provider-free implementation removes the measured two-step +dependency by making the fail-closed context package the single normal-host +entry action across MCP instructions, tool descriptions, and generated +repository onboarding. Status remains an explicit diagnostic tool. Exact clean +candidate `8844c21` passed the full deterministic checks, configured preflight, +all eight frozen one|nine gates, all three retained feedback packet validations, +and the pinned YoutubeDownloader live gate with provider mode `none`. The +sanitized record is +[one-step provider-free evidence](evidence/configured-onenine-provider-free-8844c21.json). +The authorized campaign from those identities retrieved a context package in +all three treatments and passed every control and treatment safely. It produced +no attributable win or regression, so the verdict was `neutral`; both median +efficiency deltas remained inside the 10% threshold. The sanitized record is +[one-step configured campaign evidence](evidence/configured-onenine-agent-8844c21.json). +This establishes adoption for the selected one|nine task, not product benefit. +The later Config.Net configured campaign also established adoption but +regressed correctness, so adoption now spans two repositories without +establishing repeatable product benefit. + +## Configured Agent Preflight And Campaign + +A configured suite sets `integration_mode` to `configured`, pins +`repository_commit`, and may list `baseline_mcp_servers`. Named baseline servers +must be present in the reviewed checkout-local `.codex/config.toml`; control and +treatment receive the same selected baseline, while treatment additionally +receives the reviewed CodeMesh normal-profile launch. Embedded environment +values are rejected for baseline servers; use declared `env_vars` names. + +From `agent-access`, run the provider-free gate first: + +```powershell +uv run python -m codemesh_agent_access eval agent ` + --suite ` + --repository-root ` + --max-reported-tokens ` + --configured-plan ` + --configured-guidance-path AGENTS.override.md ` + --configured-probe-query "" ` + --configured-expected-path ` + --preflight-only ` + --preflight-output +``` + +The guidance path may be a committed repository file or an explicitly installed +ignored file such as instance-generated `AGENTS.override.md`. The evaluator +copies the same reviewed bytes into both temporary conditions and records the +path, source classification, and SHA-256 hash rather than raw guidance. + +The preflight rejects dirty or stale source state, a dirty CodeMesh tool checkout, +commit, checkout, root, or source-view mismatch, changed installation files, a +non-normal tool surface, uncommitted guidance, a failed context probe, or a +failed wrong-checkout rejection probe. The report records stable identities and +hashes, not raw guidance or MCP payloads. + +Only after the preflight and repository-specific frozen retrieval gates pass, +and only with explicit model-spend authority, remove `--preflight-only`, add +`--model ` and `--output-dir `, and run the paired +campaign. The task prompt, model, reasoning effort, temporary checkout, and +declared baseline MCP servers stay identical across conditions. ## Deterministic Fixture Suite @@ -273,6 +556,7 @@ uv run python -m codemesh_agent_access eval live --repository-id code_mesh --rep uv run python -m codemesh_agent_access eval live --suite path/to/live-suite.json uv run python -m codemesh_agent_access eval live --allow-stale uv run python -m codemesh_agent_access eval live --keep-raw-traces +uv run python -m codemesh_agent_access eval live --capture-context-package-timings ``` Without `--allow-stale`, the indexed commit must match a clean local checkout @@ -293,6 +577,29 @@ The report contains: It intentionally omits snippets, credentials, environment values, and full MCP responses unless raw traces are requested. +`--capture-context-package-timings` enables opt-in sideband timing for context +packages. The MCP process emits only fixed stage and operation names, elapsed +times, counts, and success state to its captured standard error. It does not +emit queries, repository or node identifiers, paths, content, credentials, or +environment values, and it does not change MCP JSON, agent-formatted content, +ranking, or character budgets. The live report adds per-case p50 timing for +repository and binding resolution, search, repository metadata, item hydration, +assembly, and total time, with node, content-fallback, summary, and relationship +operation totals inside hydration. Warm-up calls are captured for record +alignment but excluded from the per-case summaries. + +Search attribution additionally records `search_` operation totals and call +counts for repository resolution, optional embedding, lexical/vector/summary +pipelines and their store fetches, lexical candidate selection, context +expansion, primary declarations, source content, optional relationships, and +final ranking. Timing belongs to the individual package request, including +concurrent source tasks; overlapping packages cannot share accumulators. +Pipeline totals include their child operations, and concurrently executing +operations overlap in elapsed time. Do not add these totals together or treat +their sum as package wall-clock time. An operation count identifies repeated +candidate work without retaining a node id or query. Direct search calls +outside a package do not emit package timing records. + ## Agent A/B Benchmark Agent evaluation requires: @@ -302,32 +609,43 @@ Agent evaluation requires: commit and branch separately from the benchmark repository identity. - A working `codex` installation and existing authentication. - An explicit model selection. +- A positive `--max-reported-tokens` value and a selected runner that guarantees + that exact hard-cap accounting contract. - Enough time and model budget for three runs per task and condition by default. Run from the repository root or pass `--repository-root`: ```powershell cd agent-access -uv run python -m codemesh_agent_access eval agent --model +uv run python -m codemesh_agent_access eval agent ` + --runner capped-codex ` + --model ` + --max-reported-tokens ``` Optional controls: ```powershell uv run python -m codemesh_agent_access eval agent ` + --runner capped-codex ` --model ` + --max-reported-tokens ` --reasoning-effort medium ` --repetitions 3 ` --seed 42 ` --output-dir ..\.codemesh-evals\manual-run ``` -To evaluate a model served locally by LM Studio or Ollama, select the local -provider explicitly. The same provider and model are used for both conditions: +The capped runner uses the OpenAI Responses and input-token endpoints and does +not support LM Studio or Ollama. The uncapped Codex runner retains its local +provider interface for non-agent harnesses, but it cannot pass the agent +campaign's hard-cap capability gate: ```powershell uv run python -m codemesh_agent_access eval agent ` + --runner codex ` --model ` + --max-reported-tokens ` --local-provider lmstudio ``` @@ -338,7 +656,46 @@ that combination. The runner detects older builds that lack `--ignore-user-config` and gives local-provider runs a temporary isolated `CODEX_HOME`; it never reads the user's Codex configuration in that mode. -Each control and treatment run receives the same task prompt, model, reasoning effort, sandbox, and isolated temporary clone. Both run non-interactively with `--ask-for-approval never`. The control has an empty MCP server map. The treatment adds CodeMesh as a required stdio MCP server, explicitly selects the non-destructive `diagnostic` profile to preserve the frozen historical tool surface, and approves its allowlisted tools so non-interactive calls execute instead of being canceled at the client approval boundary. Under the current report schema, suites without `treatment_guidance` preserve prompt parity and measure spontaneous adoption. Suites with guidance are reported as assisted, preserve the guidance in report metadata, and isolate best-case utility from discoverability. Configured/onboarded mode must be implemented before the runner can represent the selected v1 contract; that future mode must use and record the shipped `normal` profile rather than inheriting the evaluator's diagnostic surface. +Reported-token accounting is `input_tokens + output_tokens + +reasoning_output_tokens`. Cached input remains part of `input_tokens` and is +not added again. Before repository preflight or any model call, the runner must +declare that it hard-limits each complete invocation to the remaining campaign +allowance using this accounting. The campaign stops before the next run at zero +and stops after an incomplete run without retry. Missing or inconsistent usage +and any declared-cap violation fail as infrastructure errors. + +`--runner codex` remains the default and fails capability preflight because the +Codex CLI exposes usage only after completion and has no compatible hard-limit +option. `--runner capped-codex` is the explicitly selected compatible mode. For +each agent execution it starts an ephemeral loopback Responses proxy, uses the +official input-token endpoint to count the complete submitted input before each +model request, reserves that input plus twice the allowed generated-token +envelope, and clamps `max_output_tokens` to the remaining allowance. The factor +of two is the worst case under the committed accounting because Codex's +`output_tokens` includes reasoning and `reasoning_output_tokens` is then added +again. The Codex custom provider has request and stream retries set to zero. + +The proxy accepts only foreground `/responses` calls and local function/custom +tools. It rejects background mode, context compaction, other provider +endpoints, cost-bearing built-in tools, duplicate or declared retry activity, +insufficient allowance, missing usage, and accounting disagreement. A completed +response releases only the proven-unused part of its reservation. An interrupted +or unknown response retains the entire reservation and makes the execution +incomplete, so it cannot fund another attempt. Codex final cumulative usage must +match the proxy ledger before completion is accepted. Reports retain only the +sanitized ceiling, committed and reported totals, remainder, request counts, +retry configuration, normalized usage, and fixed failure messages; neither +credential values nor request content enters the ledger. + +Before starting the proxy, the runner loads the selected model's exact bundled +catalog entry from the selected Codex executable and supplies only that entry +through an ephemeral `model_catalog_json`. This preserves Codex's model-specific +shell and MCP tool contract without persisting instructions or trusting a +remote custom-provider catalog. An absent or malformed bundled entry fails +before any provider call. Codex's cost-free `/models` discovery receives an +empty local catalog and is never forwarded upstream. + +Each control and treatment run receives the same task prompt, model, reasoning effort, sandbox, and isolated temporary clone. Both run non-interactively with `--ask-for-approval never`. Historical spontaneous and assisted suites preserve their frozen behavior: control has an empty MCP map and treatment adds the unbound `diagnostic` profile. Configured suites instead give both conditions the same explicitly selected baseline MCP servers and add only the exact reviewed, bound `normal` CodeMesh installation to treatment. Suites without explicit mode preserve compatibility by deriving spontaneous or assisted classification from `treatment_guidance`; configured suites forbid evaluator-only guidance and preserve prompt parity. The treatment configuration allowlists CodeMesh read tools and does not expose repository deletion to the benchmark agent. @@ -353,8 +710,31 @@ tied, a 10% time or token improvement counts only when the other efficiency metric does not regress by more than 10%. Any treatment safety violation is a regression. +Suites and the selected executable must be trusted local inputs: validation +commands execute with the operator's privileges through shell-free argv. Agent +and model suite names and task/question/scenario identifiers must be safe ASCII +path segments and unique ignoring case; setup patches must be existing files +contained within the suite directory, using relative forward-slash paths. The +entire suite is checked before evaluation preparation or model execution. +Generated artifact descendants cannot resolve outside the selected output +root. See [Security and Redaction](../current/security-and-redaction.md#local-evaluation-and-inspection-boundaries) +for the exact constraints and remaining local trust boundary. + Default artifacts are written under `.codemesh-evals/` and contain the report, validation summaries, changed paths, and secret-scrubbed diffs. `--keep-raw-traces` additionally retains exact Codex JSONL, final messages, full diffs, and raw MCP output; treat that option as sensitive. +### Retained capped-runner campaign + +Exact clean candidate `0f18071` passed the deterministic, configured +positive/rejection, adaptive, impact, canonical, and synthetic capped-runner +gates provider-free. Its one authorized `gpt-5.6-sol`, medium-reasoning, +three-repetition campaign stopped after the first control execution because the +provider reported unavailable credits. No pair completed and no retry or top-up +occurred. The interrupted response retained its full 267,528-token reservation, +and the ledger recorded one request, zero completed requests, and zero configured +request or stream retries. This is retained fail-closed runner evidence, not a +product verdict or a claim about provider spend. The campaign is consumed; see +the [sanitized evidence record](evidence/configured-youtube-downloader-agent-0f18071.json). + ## Model Benchmark The built-in `codemesh-model` suite compares models on repository work rather diff --git a/docs/evaluation/review-packet-a158bc0.md b/docs/evaluation/review-packet-a158bc0.md new file mode 100644 index 0000000..de47e0f --- /dev/null +++ b/docs/evaluation/review-packet-a158bc0.md @@ -0,0 +1,221 @@ +# CodeMesh Source Review: a158bc0 + +Document type: local review packet and draft release notes + +Prepared: 2026-09-05 + +This packet reviews source version `0.1.0` at immutable source commit +`a158bc0308d9fdb8f373ec0d9d6f03233b984f9f`. It is ready for review of the local +work. Release acceptance is incomplete: no release version, tag, publication, +deployment target, or supported profile has been approved. Current implementation +and execution order remain owned by [Project Status](../current/project-status.md) +and [Next Steps](../planning/next-steps.md). + +## Candidate And Artifact Identity + +The previous published development commit is +`8e9c0da7b497fe51126e3ef66509a8587cfe8d83`. No GitHub release or remote tag exists +at the 2026-09-05 read-only check. This is the first local distribution review, +not an upgrade from a supported released version. + +The [package record](evidence/local-package-verification-a158bc0.json) binds the +artifacts below. Their bytes and SHA-256 values were reverified for this packet. +The [review manifest](evidence/review-manifest-a158bc0.json) records all 11 .NET +lock files, the Python lockfile, the inspected local toolchain, and hashes of +retained application-test logs. Raw archives, installation environments, and +logs remain under ignored `.codemesh-evals/` directories in the main checkout. + +| Artifact | Bytes | SHA-256 | +| --- | ---: | --- | +| `python/codemesh_agent_access-0.1.0-py3-none-any.whl` | 157,731 | `b07d638a88c761fc546bcd4ad46bb28a508a3590251350b5edb6be174c9fce0f` | +| `python/codemesh_agent_access-0.1.0.tar.gz` | 231,686 | `c1341687fe62ce06ddcbcf1b25100d489593663b91d51b7c9e4ca862cf96b072` | +| `codemesh-dotnet-0.1.0-linux-build.tar.gz` | 12,583,162 | `6ce9ff6daf7674cfad4184be4b131b574e12210a43deadf386020fb7ff0dc8b3` | +| `dotnet-file-manifest.json` | 22,251 | `cfdee26f4341750b36e2a701f0bf33f09a7e9f3131dfbebbaf8bcead9b4b81d1` | + +The .NET informational version is +`0.1.0+a158bc0308d9fdb8f373ec0d9d6f03233b984f9f`; Python package and REST OpenAPI +versions are `0.1.0`. Matching version strings alone do not identify identical +artifacts. + +## Verification Gate Ledger + +The numbered gates are owned by [Release Preparation](../engineering/release-preparation.md#candidate-verification-gates). + +| Gate | Evidence inspected | Current conclusion and remaining action | +| --- | --- | --- | +| 1. Immutable candidate and prior baseline | Clean package build at `a158bc0`; artifact checksums reverified; published development baseline `8e9c0da`; no released version or tag. | Local source/artifact identity verified. Owner selection of a release remains pending. | +| 2. Deterministic and standards checks | 71 .NET tests with zero skips at `f314043`; 235 Python tests on 3.12.14, 3.13.15, and 3.14.7 at `b73e19a`; relevant source, tests, locks, and build files unchanged through `a158bc0`; current documentation and publication checks. | Local checks verified for those source surfaces. Hosted `verify` still needs the proposed PR. | +| 3. Install, package, smoke, and cleanup | Installed wheel outside source: version/OpenAPI, normal four-tool manifest, 16 bundled suites and patch loading, five fixture cases. .NET publish includes design-time targets. Full smoke, separate concurrent read-only parser probe, and fresh normal-profile installation/retrieval/rejection probes pass at `a158bc0`. | Local Linux mechanism verified. Supported-profile, Windows, and representative resource/capacity acceptance remain open. | +| 4. Security | Current dependency checks: no known vulnerabilities in 11 CodeMesh .NET projects or 48 Python packages. Exact source-bound CodeQL default and local-threat results and 253 individual boundary reviews retained. | Local analysis reviewed; no alert dismissed. Hosted Python still has 39 findings on `8e9c0da`. Obtain current hosted analysis and review findings before release acceptance. | +| 5. Persistence and recovery | Same-version restore at `f314043`; isolated `8e9c0da` to `ce23f2e` reader/re-ingestion/deletion rehearsal and complete original-software/data rollback. | Local mechanism verified. Supported upgrade acceptance remains incomplete; frozen sample dependency restore failed on its advisory. | +| 6. Product outcome evidence | Configured campaigns include improved, neutral, regressed, and incomplete outcomes, retained separately in canonical status. | Repeatable configured-agent benefit is unproven. A fresh campaign needs new source/model/reasoning/repetitions/seed/ceiling authority and passing provider-free prerequisites. | +| 7. Notes, manifest, and owner review | This packet, draft notes below, artifact hashes, dependency identities, and known limitations. | Local review deliverable prepared. Owner review, release/support choices, resource evidence, and publication authorization remain pending. | + +The [local CodeQL record](evidence/local-codeql-review-b73e19a.json) preserves +invalid cached first attempts and their uncached replacements. Default-threat +results have zero findings; the broader local-input model retains 61 Python and +192 C# results with reviewed boundaries. A zero default result does not clear +those retained results or establish absence of vulnerabilities. + +The [development upgrade record](evidence/local-development-upgrade-ce23f2e.json) +preserves its first failed relationship gate and the failed frozen sample +restore. After dependency assets were emitted, the unchanged six-case/18-call +retrieval gate passed with parser advisory warnings. That later pass does not +make the dependency restore pass. No consumed paid campaign was retried. + +The [fresh configured runtime record](evidence/configured-runtime-verification-a158bc0.json) +verifies the installed wheel's plan/apply/probe commands and the shipped +source-backed MCP launch from a clean `a158bc0` checkout. It returns the expected +source path, rejects source-view mismatch, unknown checkout, and dirty source, +then passes again after exact source restoration. Configuration and onboarding +were applied only to an isolated clone and kept ignored. This is provider-free +runtime evidence, not a configured-agent campaign or product-benefit result. + +The hosted CodeQL configuration covers Actions, C#, and Python with its default +query suite and remote-plus-local threat model. Local retained analysis covers +C# and Python with pinned security-extended suites; it is not an Actions security +scan or an identical hosted query configuration. The current hosted pass remains +a separate requirement. + +## Environment And Resource Boundary + +The observed host is Linux x86_64 with .NET SDK `10.0.111`, Python `3.14.7`, and +uv `0.12.9`. The isolated Python compatibility checks additionally exercised +3.12.14 and 3.13.15. The Agent Access Docker build uses pinned uv `0.12.3`. +The package record retains selected Neo4j, MongoDB, Qdrant, and Agent Access +image IDs, plus the separately observed read-only parser image ID. These are +local Docker image identities, not published registry manifest digests. + +The exercised baseline disables model providers, embeddings, and generated +summaries. Compose exposes services on loopback: Agent Access 8088, C# parser +8091, Neo4j 7474/7687, MongoDB 27017, and Qdrant 6333/6334. Configuration remains +private; this packet contains no credentials or private configuration values. + +Compose configures Neo4j with an initial 512 MiB and maximum 1 GiB heap. That +setting is not a complete host resource requirement. Representative dataset, +concurrency, memory, disk, and latency acceptance criteria have not been +selected and measured for a supported deployment. Linux checks do not establish +Windows acceptance. All task containers are stopped; private artifacts and +recovery volumes remain preserved. + +## Draft Release Notes + +- Source version reviewed: `0.1.0`; release version not selected. +- Status: local artifacts and notes prepared; not tagged, published, deployed, + or promoted to stable. +- Release date: not released. +- Tag: not created. +- Reviewed source commit: `a158bc0308d9fdb8f373ec0d9d6f03233b984f9f`. +- Previous stable release: none. + +### Release highlights + +- Exact checkout binding, reviewable installation, freshness rejection, and a + one-step source-grounded context package for normal MCP clients. +- Bounded Python/Rust ingestion and a provenance-bound feedback path for the + activated one|nine Primary pilot; one selected language set publishes + atomically at the snapshot slot boundary. +- Configured-agent evaluation with explicit hard aggregate-token enforcement, + retained interrupted reservations, and no automatic provider retries. +- Implemented summary-model qualification tooling; no qualified summary + deployment profile or default-summary retrieval claim. + +### Fixes + +- Repository filtering precedes Neo4j lexical candidate scoring. The retained + controlled synthetic-load result improves local retrieval latency; agent + benefit remains separately unproven. +- Read-only parser-container design-time outputs use private temporary + directories, preserving compiler metadata and project relationships. +- Evaluation identifiers and generated paths are validated, dynamic HTML + attributes are escaped, and feedback recording preserves existing packet + destinations, including links. + +### Breaking changes and compatibility review + +The following behavior requires review when moving from older development +source; no released-version compatibility promise exists yet: + +- Normal MCP operation requires the documented exact checkout binding. Use a + reviewed installation plan and fresh probe; diagnostic operation retains its + separate administrative contract. +- Agent evaluation requires `--max-reported-tokens` and a compatible hard-cap + runner. The default uncapped Codex runner rejects that preflight. +- Known secret files are excluded by default. Unsafe evaluation identifiers, + escaping setup patches, linked feedback outboxes, and existing packet + destinations are rejected. +- Path-policy and parser-output changes can produce new snapshot identities. + Re-ingest and verify the selected checkout; do not substitute old binding + or campaign evidence for the new source state. + +### Upgrade + +No supported upgrade is approved by this packet. The rehearsed local sequence +is detailed in [Backup and Recovery](../guides/backup-and-recovery.md): + +1. Identify the exact instance, software, stores, configuration, and indexed + repositories; stop writers. +2. Preserve the complete consistent four-volume set and private configuration. +3. Restore into new isolated volumes and prove original-software digest equality. +4. Exercise the selected new software and re-ingest authorized repositories; + verify identity, freshness, citations, retrieval, and scoped deletion. +5. On failure, stop the candidate and restore the complete original software + and store set. Require original data and retrieval digest equality. + +### Migrations + +The tested development baseline needed no separate offline schema-migration +command. Re-ingestion published new snapshots while preserving project and +checkout identity. This observation is confined to the retained baseline and +same store image identities; other persisted formats need their own review. +Legacy behavior is documented in [Identity and Persistence](../current/identity-and-persistence.md). + +### Required data or configuration + +Configured local stores and private authentication remain required. Normal MCP +clients need their exact reviewed binding. The verified core profile uses no +model provider or generated summaries. Artifact and dependency checksums are +listed above; published artifact URLs and a supported deployment destination +remain unselected. + +### Verification + +Use the exact invocations in [COMMANDS](../../COMMANDS.md), including its +[local package procedure](../../COMMANDS.md#local-package-verification). The +installed checks exercised these commands from outside the source checkout: + +```powershell +dotnet /CodeMesh.Cli.dll --version +/bin/codemesh-agent-access --version +/bin/codemesh-agent-access mcp-manifest --profile normal +/bin/codemesh-agent-access eval +``` + +Expected source version: `0.1.0`, with the exact .NET commit suffix above. +The paths identify the private verified artifacts; these are Linux checks. +For a source checkout, run the canonical full smoke on a separately identified +disposable Compose project. Commands do not authorize changing an existing +user deployment. + +### Known limitations + +- Repeatable configured-agent product benefit remains unproven. +- Hosted current-source checks, Windows/support acceptance, representative + resource criteria, and owner release review remain incomplete. +- Frozen YoutubeDownloader source retains the AngleSharp advisory + `GHSA-pgww-w46g-26qg`; no dependency or warning gate was suppressed. +- Local-input CodeQL results remain reviewed and retained, without dismissal. +- Existing .NET nullable and Python dependency warnings are retained in the + source verification records. +- C# project loading executes MSBuild design-time logic. Private outputs do not + sandbox arbitrary project targets; custom-target acceptance remains separate. +- Shared-service isolation, broader deployment/parser coverage, outcome memory, + portable snapshots, and other deferred roadmap scope remain unactivated. + +## Proposed External Review Step + +Publish the reviewed `workstream/completion` branch and open a draft pull request +against `main` only after explicit owner authorization. The observed branch +rules require a PR, the `verify` status check, resolved review threads, and +squash merging. This proposal requests no merge, tag, package publication, +deployment, provider invocation, or one|nine instance expansion. diff --git a/docs/evaluation/testing.md b/docs/evaluation/testing.md index 9dfe23a..91f6de5 100644 --- a/docs/evaluation/testing.md +++ b/docs/evaluation/testing.md @@ -3,31 +3,24 @@ Document type: current operating guide Use this guide to choose the right verification path after CodeMesh changes. +The supported invocations are indexed in [`COMMANDS.md`](../../COMMANDS.md); +this guide defines when to use them, plus the gates around end-to-end and +evaluation procedures. ## Default Verification Commands -From the repository root: - -```powershell -dotnet restore CodeMesh.sln --locked-mode -dotnet build CodeMesh.sln --no-restore -dotnet run --project tests/CodeMesh.Tests --no-restore -dotnet format CodeMesh.sln --verify-no-changes --no-restore -``` +Run the [deterministic .NET commands](../../COMMANDS.md#net) from the repository +root. Run these .NET commands serially because projects share intermediate output directories. -From `agent-access`: +Run the [Python Agent Access commands](../../COMMANDS.md#python-agent-access) +from `agent-access`. -```powershell -uv sync --locked -uv run --no-sync pytest -uv run --no-sync ruff check . ../tools -uv run --no-sync ruff format --check . ../tools -``` - -Run both after most implementation changes. +Run both after changes that cross .NET/Python contracts, serialization, +lifecycle, integration, or end-to-end boundaries. For a focused change, run +the affected surface and report why the other suite was not required. ## .NET Tests @@ -38,11 +31,20 @@ The .NET test harness covers: - Agent Access contract mapping. - Agent Access .NET client behavior. - Ingestion orchestration. +- Default secret-file exclusion, linked-path containment, configurable path + filters, watch filtering, and path-policy snapshot identity. - Incremental writes, deterministic project/checkout/snapshot identity, compare-and-swap slot retention, pins, and cleanup. - Embedding and summary providers. +- Summary qualification input redaction, held-out partition enforcement, + immutable evidence binding, typed reviewer aggregation, mandatory + query-level retrieval assessment, resource/cost evidence, gate enforcement, + and fail-closed report comparison. - MongoDB, Neo4j, and Qdrant storage adapters. -- C# parser behavior. +- C#, Python, Rust, and atomic multi-language parser behavior. +- Parser-container design-time output isolation, compiler-visible metadata, + concurrent same-named projects, project failure cleanup, and cross-project + invocation preservation. - Optional configured sample parsing. Some tests skip when required live services or sample paths are unavailable. @@ -56,6 +58,8 @@ The Python tests under `agent-access/tests` cover: - MCP evaluation scenarios. - Formatting. - MCP contracts. +- Exact checkout binding, reviewable installation, provider-free probing, and + versioned feedback recording/summarization. - REST contracts. - Store read components. @@ -93,11 +97,56 @@ cd agent-access uv run python -m codemesh_agent_access eval live ``` +Add `--capture-context-package-timings` when attributing context-package +latency. The opt-in report contains sanitized per-case stage p50s from a +sideband stream; tool responses and character budgets remain unchanged. Treat +those timings as performance evidence separate from retrieval-quality claims. + The retained live and existing spontaneous/evaluator-assisted agent suites select the `diagnostic` profile explicitly to preserve their frozen tool surface. -That does not make them configured/onboarded product evidence. The future -configured/onboarded mode must select the shipped `normal` profile and record -that identity separately. +That does not make them configured/onboarded product evidence. The implemented +configured mode selects the shipped `normal` profile and records that identity +separately in its installation plan and runtime probe. It also runs an +automatic wrong-checkout rejection probe before a paired run. + +For an installed configured checkout, run the reviewed plan's normal-profile +probe instead. It verifies the exact launch and binding and can gate real target +paths: + +```powershell +uv run python -m codemesh_agent_access mcp-probe ` + --plan C:\temp\codemesh-install-plan.json ` + --query "Python to Rust native replay implementation and tests" ` + --expected-path native/feed_hub_replay/src/lib.rs +``` + +This is configured retrieval evidence, not a model-backed agent comparison. +Record misses through the feedback workflow and retain passing rechecks with an +explicit superseded feedback id. + +Before requesting model-spend authority, run the configured agent preflight +with the pinned configured suite and reviewed plan. It invokes no model: + +```powershell +uv run python -m codemesh_agent_access eval agent ` + --runner capped-codex ` + --suite ` + --repository-root ` + --max-reported-tokens ` + --configured-plan ` + --configured-guidance-path AGENTS.override.md ` + --configured-expected-path ` + --preflight-only ` + --preflight-output +``` + +The suite must declare `integration_mode` as `configured`, pin +`repository_commit`, and may name reviewed `baseline_mcp_servers` that both +conditions receive. Guidance may be committed or an explicitly installed +ignored file such as instance-generated `AGENTS.override.md`; both temporary +conditions receive identical reviewed bytes, while the report retains its path, +source classification, and hash rather than raw content. A preflight pass is +provider-free configuration and runtime evidence only. The pinned YoutubeDownloader suite is the representative external C# baseline. Prepare and ingest the exact checkout as described in @@ -113,9 +162,40 @@ Run the opt-in Codex A/B benchmark only when live stores, a fresh index, Codex authentication, time, and model budget are available: ```powershell -uv run python -m codemesh_agent_access eval agent --model +uv run python -m codemesh_agent_access eval agent ` + --runner capped-codex ` + --model ` + --max-reported-tokens ``` +The agent evaluator now requires a positive hard cap and a selected runner that +declares compatible hard-cap enforcement before repository preflight or any +model call. Reported tokens are input plus output plus reasoning-output tokens; +cached input is already included in input and is not counted twice. Every run +receives only the remaining campaign allowance. The campaign stops at zero or +after an incomplete run without retry. Missing or inconsistent usage and a +runner cap violation fail as infrastructure errors. + +The default `--runner codex` subprocess mode cannot guarantee this cap because +the Codex CLI reports usage only after a turn and exposes no compatible +hard-limit option, so provider-free capability preflight refuses it. Select +`--runner capped-codex` explicitly for the implemented loopback Responses proxy. +Provider-free preflight verifies its declared accounting without using a model +or requiring a credential. A model-backed run additionally requires an +`OPENAI_API_KEY`; its value is forwarded only to the upstream API and is never +included in reports or logs. + +The capped runner counts each complete input before generation, reserves input +plus twice the admitted generated-token envelope as the worst case under the +committed output-plus-reasoning accounting, clamps generated output to the +smaller of the remaining allowance and the model's documented output maximum, +and configures zero request and stream retries. It rejects background, +compaction, unreviewed parameters outside the input-counting contract, +unsupported cost-bearing endpoints and tools, retry activity, missing usage, +and accounting disagreement. Interrupted calls retain their reservation and +stop the campaign as incomplete. Per-execution reports contain a sanitized +hard-cap ledger for auditing this behavior. + Use `--suite youtube-downloader-agent` and point `--repository-root` at the pinned external checkout for its implementation-discovery, change-impact, and validation-selection comparison. @@ -126,6 +206,14 @@ the default three repetitions and does not alter the original frozen suite. Run `eval live --suite youtube-downloader-impact-live` first to retain its retrieval and relationship baseline. +Use `eval live --suite youtube-downloader-impact-adaptive-live` for the +provider-free task-adaptive gate. It preserves the same repository commit, six +targets, and thresholds while comparing one task-shaped package with three +bounded focused facets. Run it with the configured preflight, unchanged impact +suite, and unchanged canonical suite before requesting authority for another +paired campaign. A passing adaptive suite proves retrieval and package-budget +readiness only; it does not prove agent compliance or benefit. + Use `--suite youtube-downloader-impact-assisted-agent` only after a spontaneous campaign records zero adoption. Its treatment-only onboarding deliberately breaks prompt parity and is reported as assisted utility evidence. @@ -164,13 +252,7 @@ Common variants: ## Documentation-Only Changes Tests are usually not required for documentation-only changes. Run: - -```powershell -npx --yes markdownlint-cli2@0.23.1 "**/*.md" -python tools/check_markdown_links.py -python tools/check_publication_safety.py -git diff --check -``` +the [repository standards checks](../../COMMANDS.md#repository-standards). The link and publication-safety checks are local-only and do not print secret values. @@ -196,11 +278,87 @@ deterministic baseline workflow. - Skipped integration tests usually mean a store or sample repository is unavailable. - Failed integration tests may be service setup issues or code regressions; check service health before changing code. +## Harness Implementation and Invariant Map + +For the human reading path, see the +[Developer handbook](../guides/developer-handbook.md). The following map helps +maintainers locate relevant test machinery without replacing the risk-based +check selection above. + +### How the harnesses run + +The [.NET harness](../../tests/CodeMesh.Tests/Program.cs) is a console program +with a table of `(Name, Func)` entries. It awaits them sequentially, +prints pass/skip/fail records, catches `SkipTestException` separately, and exits +nonzero for failures. New harness tests must be registered in that table. +`dotnet test` is not a substitute for its supported `dotnet run` invocation. + +Many fixtures are embedded source strings written to temporary directories; +some initialize Git repositories to exercise commits, worktrees, markers, and +retention. Fake parser/store/provider implementations isolate orchestration. +Live adapter tests probe configured stores and use disposable database, +collection, or repository identities; they skip when the required service is +unavailable. Therefore a successful harness exit with skips does not establish +live-store correctness. The optional YoutubeDownloader parser test also needs +its configured sample path. + +The [Python suite](../../agent-access/tests/) uses pytest. Async paths are often +driven with `asyncio.run`; monkeypatches, stub facades, controlled clocks, mock +transports, temporary checkouts, and in-process ASGI/MCP calls keep normal +tests deterministic. The separate +[`run_mcp_evaluations`](../../agent-access/codemesh_agent_access/evaluation/fixture.py) +temporarily replaces the store used by tools and invokes the diagnostic MCP +surface in process. It does not test a fresh external stdio launch, real database +state, or a model's decision to adopt a tool. + +[`e2e-smoke.ps1`](../../scripts/e2e-smoke.ps1) creates a small committed C# fixture, +starts services unless told otherwise, ingests it, checks REST/client-visible +context and lifecycle behavior, and cleans its generated fixture/index when +configured to do so. Its `-FixtureOnly` branch returns before service startup +and ingestion; that branch is fixture preparation evidence only. The +[live evaluator](../../agent-access/codemesh_agent_access/evaluation/live.py) +separately owns real MCP stdio retrieval checks. + +### Invariants and representative tests + +All C# test symbols in this table are in +[Program.cs](../../tests/CodeMesh.Tests/Program.cs). + +| Invariant or risk | Representative test or file | Evidence boundary | +| --- | --- | --- | +| Files cannot escape selection via linked paths | `TestRepositoryPathPolicyRejectsLinkedPaths`, `TestCSharpParserAppliesRepositoryPathFilters` | Synthetic path/parser cases; not an exhaustive secret detector. | +| Redaction precedes persistence and models | `TestIngestionRedactsSensitiveContentBeforeStoresAndEmbeddings` | Asserts controlled values and provider input. | +| Multi-language output publishes together | `TestMultiLanguageIngestionPublishesOneDeterministicSnapshot` | Composite success and parser-error behavior with controlled collaborators. | +| Dirty source does not move clean branch-head | `TestSnapshotSlotsRetainCleanHeadsAndBoundDirtyHistory` | Temporary Git plus in-memory registry; not every Mongo crash interleaving. | +| Failed writes preserve previous slots | `TestFailedSnapshotGenerationPreservesActiveSlots` | Injected graph failure after staging. | +| Pins prevent collection until released | `TestExplicitPinsProtectSnapshotsUntilReleased` | Sequential retention behavior; no shared-service coordinator proof. | +| C# logical symbols retain separate source declarations | `TestCSharpParserSeparatesPartialDeclarations` | Representative partial-source fixture. | +| Local semantic calls cross projects | `TestCSharpParserEmitsCrossProjectInvocationRelationships` | Loaded test projects, not all runtime dispatch. | +| Python/Rust native edges remain bounded | `TestCompositeParserLinksPythonCallsToRustPyO3Exports` | Static import/export names, not a running native module. | +| Store schema and payloads round-trip | `TestMongoContentStoreRoundTripsContent`, `TestMongoRegistryStoreRoundTripsRepositoriesAndRuns`, `TestNeo4jGraphStoreRoundTripsGraph`, `TestQdrantVectorStoreRoundTripsEmbeddings` | Live only when prerequisites are available; inspect skips. | +| Binding rejects before retrieval | [test_store_components.py](../../agent-access/tests/test_store_components.py), `test_context_package_rejects_unaccepted_binding_before_search` | Controlled binding/store failure, complemented by the live rejection probe. | +| Ranking covers files and resolves saturated ties | [test_context_ranking.py](../../agent-access/tests/test_context_ranking.py) | Deterministic synthetic relevance invariants. | +| Package namespaces and budgets stay coherent | [test_context_ranking.py](../../agent-access/tests/test_context_ranking.py), [test_formatting.py](../../agent-access/tests/test_formatting.py), [test_cli_output.py](../../agent-access/tests/test_cli_output.py) | Snippet/format contracts; not a tokenizer or product-benefit benchmark. | +| REST/MCP/.NET schemas align | [test_rest_contract.py](../../agent-access/tests/test_rest_contract.py), [test_mcp_contract.py](../../agent-access/tests/test_mcp_contract.py), `TestAgentAccessContractsMapRestJson`, `TestAgentAccessClientCallsRestEndpoints` | Serialization and transport mapping. | +| Installation rejects drift; feedback preserves provenance | [test_installer_probe.py](../../agent-access/tests/test_installer_probe.py), [test_feedback.py](../../agent-access/tests/test_feedback.py) | Plan/packet checks with controlled inputs. | +| Evaluation stops and reports truthfully | [test_effectiveness_evaluation.py](../../agent-access/tests/test_effectiveness_evaluation.py), [test_model_benchmark.py](../../agent-access/tests/test_model_benchmark.py), [test_capped_codex.py](../../agent-access/tests/test_capped_codex.py) | Harness/accounting behavior, not a completed live model campaign. | + +### Select checks by changed boundary + +| Change | Required direction | +| --- | --- | +| Documentation/navigation only | Repository standards; validate referenced examples and diagrams without starting services. | +| Parser-only implementation | .NET restore/build/harness/format; add a source fixture for the actual semantic or structural failure. | +| Python ranking/formatting only | Python tests and lint/format; deterministic MCP fixtures for search/package/guidance changes. | +| Shared request/response or persisted serialization | Both language suites; verify producer and consumer field/default/null behavior. | +| Lifecycle, cleanup, Docker, live wiring, or REST integration | Both language suites and the authorized PowerShell smoke path described above. | +| Product-benefit claim | The separately authorized, identity-bound evaluation procedure; unit and smoke tests cannot supply this evidence. | + +The [change walkthroughs](../guides/making-changes.md) give concrete examples of +these selections. The [evaluation internals reference](../current/enrichment-and-evaluation.md) +explains how fixtures, live retrieval, configured campaigns, and model benchmarks +differ. Historical results do not automatically transfer to the current source. + ## Useful Health Checks -```powershell -dotnet run --project src/CodeMesh.Cli -- doctor -dotnet run --project src/CodeMesh.Cli -- status -docker compose ps -Invoke-RestMethod http://127.0.0.1:8088/health -``` +Use the [local product health commands](../../COMMANDS.md#local-product-health). diff --git a/docs/guides/agent-quickstart.md b/docs/guides/agent-quickstart.md index 793f003..06ae568 100644 --- a/docs/guides/agent-quickstart.md +++ b/docs/guides/agent-quickstart.md @@ -4,14 +4,20 @@ Document type: repository navigation guide This is the preferred starting point for coding agents working in CodeMesh. +The [Developer Handbook](developer-handbook.md) provides the human-oriented +implementation explanation, glossary, source walkthroughs, and debugging guides. + ## Read Order 1. [`AGENTS.md`](../../AGENTS.md) for operating rules and authority boundaries. 2. [Documentation Hub](../README.md) for document ownership and classification. -3. [Project Status](../current/project-status.md) for current implementation and evidence. -4. [Architecture](../current/architecture.md) and +3. [`COMMANDS.md`](../../COMMANDS.md) for supported repository commands and + [Software Engineering Methodology](../engineering/methodology.md) for + evidence and authority boundaries. +4. [Project Status](../current/project-status.md) for current implementation and evidence. +5. [Architecture](../current/architecture.md) and [Agent Access Contracts](../current/agent-access-contracts.md) for current behavior. -5. [Next Steps](../planning/next-steps.md) only when selecting future product work. +6. [Next Steps](../planning/next-steps.md) only when selecting future product work. Use [MCP Setup](mcp-setup.md) and [Self-Analysis](self-analysis.md) when configuring or dogfooding CodeMesh. Use [Testing](../evaluation/testing.md) to select @@ -30,10 +36,11 @@ execution order or the user activates it. | Task | Start in | | --- | --- | | REST, MCP, Python CLI, ranking, formatting, or context packages | `agent-access/codemesh_agent_access` and `docs/current/agent-access-contracts.md` | +| Development feedback sessions, MCP packet recording, or maintainer intake | `agent-access/codemesh_agent_access/feedback_session.py`, `feedback.py`, `mcp.py`, and `docs/guides/mcp-setup.md` | | .NET Agent Access client contracts | `src/CodeMesh.Domain/Contracts` and `src/CodeMesh.Control/AgentAccess` | -| Ingestion, cleanup, embeddings, or summaries | `src/CodeMesh.Ingestion` | +| Ingestion, cleanup, embeddings, summaries, or summary qualification | `src/CodeMesh.Ingestion` and `COMMANDS.md` | | C# semantic graph behavior | `src/CodeMesh.Parser.CSharp` | -| Python, Markdown, or deployment parsing | The matching `src/CodeMesh.Parser.*` project | +| Python, Rust, Markdown, or deployment parsing | The matching `src/CodeMesh.Parser.*` project | | Store reads, writes, or cleanup | `src/CodeMesh.Storage` | | CLI workflow or repository discovery | `src/CodeMesh.Cli` and `src/CodeMesh.Control` | | Product direction | `docs/current/project-status.md`, then `docs/planning/next-steps.md` | @@ -48,5 +55,6 @@ execution order or the user activates it. - Never print `.env` values, credentials, tokens, keys, certificates, or local secrets. - Report passed, failed, skipped, and unavailable checks separately. -- Use the smallest risk-appropriate verification path in +- Use supported invocations from [`COMMANDS.md`](../../COMMANDS.md) and select + the smallest risk-appropriate verification path in [Testing](../evaluation/testing.md). diff --git a/docs/guides/backup-and-recovery.md b/docs/guides/backup-and-recovery.md new file mode 100644 index 0000000..2fed9ac --- /dev/null +++ b/docs/guides/backup-and-recovery.md @@ -0,0 +1,133 @@ +# Local Backup And Recovery Rehearsal + +Document type: verified local operating guide + +This procedure was exercised with disposable, single-node Linux Docker stores +at application candidate `f314043`. It restored a summary-free indexed +repository and synthetic vector fixtures into new volumes using the same image +identities. A separate development-baseline upgrade rehearsal is recorded below. Neither +rehearsal establishes supported-version upgrades, Windows acceptance, online backups, +cluster recovery, or permission to stop or replace an existing user environment. + +## Preconditions And Evidence + +Record the exact source and restore Compose project names, source SHA, store +and Agent Access image IDs, logical volume names, and selected repository +identity. Inspect volume ownership labels. Refuse a destination project or +volume that already exists. Keep the source volumes intact. + +Capture complete graph-node and relationship digests, Neo4j index definitions, +MongoDB document digests for every CodeMesh collection, vector points and +payloads, repository metadata, and representative context-package items. +Require actual data in every store being claimed as verified; the vector +fixture here contains three explicitly synthetic points and uses no model. + +The private archive directory must restrict access to its owner. Volume backups +contain repository content and may contain database authentication material. +Keep archives and raw manifests outside Git. Publish only sanitized evidence. + +## Quiesce And Copy + +From the selected application checkout, with its authorized local configuration: + +```powershell +docker compose -p stop +docker ps -q --filter label=com.docker.compose.project= +``` + +The second command must return no running containers. Also wait for any +independent ingestor or writer to stop. Do not archive an online store or mix +volumes copied across different write states. + +Copy all four volumes as one stopped set: `neo4j_data`, `neo4j_logs`, +`mongodb_data`, and `qdrant_data`. For each verified source volume: + +```powershell +docker run --rm --network none --read-only ` + --mount type=volume,src=,dst=/data,readonly ` + --mount type=bind,src=,dst=/backup ` + --entrypoint /bin/tar ` + -czf /backup/.tar.gz -C /data . +``` + +Record each archive's size and SHA-256 checksum, then verify the checksum before +restoration. This uses Docker's documented +[volume copy mechanism](https://docs.docker.com/engine/storage/volumes/#back-up-restore-or-migrate-data-volumes). +The database consistency claim comes from the stopped set and the specific +restoration evidence, not from `tar` alone. + +## Restore Into New Volumes + +Create distinct, initially absent volumes with the restore project's ownership +labels. For each volume: + +```powershell +docker volume create ` + --label com.docker.compose.project= ` + --label com.docker.compose.volume= ` + + +docker run --rm --network none --read-only ` + --mount type=volume,src=,dst=/data ` + --mount type=bind,src=,dst=/backup,readonly ` + --entrypoint /bin/tar ` + -xzf /backup/.tar.gz -C /data +``` + +Create a private Compose override that binds each store and Agent Access to its +recorded source image ID and sets `pull_policy: never`. Start the restored +project with `--no-build`, using free loopback ports or keeping the source +project stopped. Recheck actual image IDs after startup. Do not use mutable tags +as proof of version equality. + +## Validate And Retain + +Wait for store and Agent Access health, then reproduce the complete pre-backup +digests. Require identical graph nodes, edges, index definitions, MongoDB +records, vector points/payloads, repository metadata, and ordered context items. +A startup success or matching counts alone is insufficient. + +The [retained rehearsal](../evaluation/evidence/local-recovery-rehearsal-f314043.json) +matched all of these checks for 1,685 nodes, 3,634 relationships, 1,310 content +documents, the snapshot/registry records, three synthetic vectors, and eight +context-package items. No model provider was invoked. The source and restored +volumes remain separate; no existing user store was replaced. + +Stop and remove only the disposable project containers after verification. +Preserve the private archive and evidence; volume deletion is a separate, +explicitly scoped cleanup. A future upgrade must separately name the supported +old and new versions and demonstrate migration and rollback. Select the +appropriate vendor backup method for other versions or deployment profiles. + +## Development-Baseline Upgrade And Rollback + +The [retained upgrade rehearsal](../evaluation/evidence/local-development-upgrade-ce23f2e.json) +uses published development source `8e9c0da` and candidate `ce23f2e`, with a +separate clean clone of the frozen YoutubeDownloader sample. The old software +indexed the sample; its stopped store set was backed up and restored into new +volumes with identical data and retrieval digests. The candidate reader left +stored data unchanged, then re-ingestion preserved project and checkout +identity while publishing new snapshots. Eight candidate citations passed +source-span, snapshot, and content-identity checks. + +The initial retrieval gate failed cross-project relationship coverage because +the fresh sample had no restored dependency assets. Its locked restore failed +on the frozen AngleSharp advisory `GHSA-pgww-w46g-26qg`; no audit or warning gate +was disabled. Assets produced by that restore enabled the missing relationships +on re-ingestion. The unchanged six-case, 18-call retrieval suite then passed, +with two parser advisory warnings retained. The failed restore remains a failed +dependency check, so this result does not establish complete sample or supported +upgrade acceptance. + +Scoped deletion removed the isolated repository's graph, content, and registry +state while preserving the unrelated synthetic vector fixture. Rollback then +restored the original software and complete original backup into another new +volume set. Every original data, metadata, and ordered context-item digest +matched. All rehearsal containers and networks were removed; the private +backups and all volume sets remain preserved. + +Across reader revisions, package order and scoring differed even before +re-ingestion; do not require cross-version response equality as an upgrade +contract. Require grounded citations and the unchanged retrieval gates. Exact +response equality was required and passed for restoration of the same original +software and data. diff --git a/docs/guides/developer-handbook.md b/docs/guides/developer-handbook.md new file mode 100644 index 0000000..93066a5 --- /dev/null +++ b/docs/guides/developer-handbook.md @@ -0,0 +1,138 @@ +# CodeMesh developer handbook + +Document type: human developer entry point + +Source reviewed: `348709de06f971e4ef15fde7f6b7b082799c6777` on `main`. +Review date: 2026-09-05. The working tree was clean before this documentation +change. The chapters describe inspected implementation and representative +tests; their walkthroughs are source traces, not newly executed live runs. +Historical measurements remain bound to the candidates in +[Project Status](../current/project-status.md). + +## What CodeMesh does + +CodeMesh turns a selected repository into a queryable index of source structure, +relationships, and source text. A developer or coding agent can ask where a +behavior is implemented, follow likely callers and dependencies, and obtain a +compact package of relevant source locations with suggested validation. The +package points the reader back to the checkout to verify a conclusion before +changing code. + +The initial product focus is C#/.NET. Its C# parser uses Roslyn compilations to +connect declarations to symbols and resolve local calls and member access. +Python, Rust, Markdown, and deployment parsers cover additional workflows with +more structural analysis. A relationship from one of these parsers is a useful +navigation lead; it is not a runtime execution trace or a complete proof of +change impact. The [parser chapter](../current/ingestion-internals.md) explains +the different evidence each parser produces. + +There are two principal halves. **Ingestion** is the .NET write path: select +files, parse them, redact stored content, calculate identity, write the stores, +and publish a snapshot selection. **Agent Access** is the Python read path: +resolve that selection, retrieve and rank candidates, attach source and +relationships, and serve a response. Normal MCP access also checks that the +selected index belongs to the exact clean checkout. The Python service reads +the stores directly; it does not ask the .NET CLI to ingest on demand. + +This distinction explains a common debugging mistake: a reachable REST service +does not imply an indexed repository, and a successfully indexed dirty checkout +does not imply that normal MCP access will accept it. See +[IngestionOrchestrator.IngestAsync](../../src/CodeMesh.Ingestion/IngestionOrchestrator.cs), +[CodeMeshReadStore.get_context_package](../../agent-access/codemesh_agent_access/store.py), +and the rejection tests in +[test_store_components.py](../../agent-access/tests/test_store_components.py). + +## Vocabulary + +| Term | Meaning in this implementation | +| --- | --- | +| Repository root | The selected filesystem directory. Its path is provenance, not the persistent project identity. | +| Project identity | A persistent `prj_` id for the indexed repository, potentially shared deliberately by multiple checkouts. Distinct from a Roslyn/MSBuild project. | +| Parser project | The `CodeNode.Project` grouping, such as a C# project name. This participates in fingerprints and is not a `prj_` id. | +| Checkout | A particular local working tree, identified by a private `chk_` marker. | +| Alias | A human-readable selector retained with project identity. Aliases can collide; ambiguous reads are rejected. | +| Node | A `CodeNode`: a file, declaration, symbol, document section, or other indexed entity. | +| Stable key | A parser-produced logical key. Its stability depends on the parser; a declaration key can include source position. | +| Relationship | A directed edge with a kind and optional label/metadata, such as `Contains`, `Defines`, or `Invokes`. | +| Source span | Repository-relative file and line/column coordinates. Unknown spans use zero; C# logical symbols often need a declaration for concrete source. | +| Content hash | SHA-256 identity of a stored text item, recalculated if redaction changes that item. It may identify a fragment rather than an entire file. | +| Source view | The canonical fingerprint of effective parser output after redaction, including graph structure and content identities. It is not simply the Git commit. | +| Snapshot | A deterministic `snp_` namespace for that project, source view, ingestion scope, parser profile, and graph schema. | +| Slot | A versioned reference selecting a snapshot: checkout-current, clean branch-head, explicit pin, or evidence slot. | +| Generation | One staging/publication attempt. Identical source can produce multiple generations for the same snapshot. | +| Ingestion run | Execution metadata and write statistics. The successful run record is separate from generation failure tracking. | +| Binding | The configured project, checkout, absolute root, and optional source-view hash used by normal MCP. | +| Context hit | A ranked candidate with source location, identity, score components, and optional expansion information. | +| Context package | Hits hydrated with content metadata, budgeted snippets, optional summaries, grouped relationships, and validation suggestions. | +| Enrichment | Optional embeddings or generated summaries. Core source retrieval can operate without a model provider. | + +The definitions come from +[graph records](../../src/CodeMesh.Domain/Graph/GraphTypes.cs), +[snapshot contracts](../../src/CodeMesh.Domain/Contracts/SnapshotContracts.cs), +[SnapshotIdentity.Compute](../../src/CodeMesh.Ingestion/SnapshotIdentity.cs), and +[Python response models](../../agent-access/codemesh_agent_access/models.py). +The detailed identity chapter explains where the same field name has different +public and storage meanings. + +## Chapter index + +| Chapter | Questions it answers | +| --- | --- | +| [Architecture and code navigation](../current/architecture.md) | Which processes run, what depends on what, and where does a change belong? | +| [Identity and persistence](../current/identity-and-persistence.md) | What are the keys, store responsibilities, publication boundaries, and retention rules? | +| [Ingestion internals](../current/ingestion-internals.md) | How do selection, parsers, redaction, incremental writes, refresh, and watch work? | +| [Retrieval internals](../current/retrieval-internals.md) | How do binding, freshness, search, ranking, expansion, and packaging work? | +| [Enrichment and evaluation internals](../current/enrichment-and-evaluation.md) | How are embeddings, summaries, feedback, installation, and evaluation connected? | +| [Three end-to-end walkthroughs](developer-walkthroughs.md) | What calls what during ingestion, context retrieval, and changed-source refresh? | +| [Development and debugging](development-and-debugging.md) | How do I set up, choose entry points, inspect failures, and recover locally? | +| [Making changes](making-changes.md) | Which producer, consumer, tests, and documents must move together? | +| [Testing](../evaluation/testing.md) | Which harness proves which invariant, and what validation does my change require? | + +These chapters explain implementation. Keep exact interface fields in +[Agent Access Contracts](../current/agent-access-contracts.md), supported +invocations in [COMMANDS.md](../../COMMANDS.md), and security policy in +[Security and Redaction](../current/security-and-redaction.md). The +[documentation hub](../README.md) defines the complete authority order. + +## Reading paths + +For onboarding, read this introduction and glossary, then architecture, the +three walkthroughs, and local development. Return to identity, ingestion, or +retrieval for the detailed mechanisms behind the walkthroughs. Read the testing +guide before your first change. + +For debugging, start with the symptom table in +[Development and debugging](development-and-debugging.md#diagnose-by-symptom). +Establish the process, checkout, selected snapshot, and provider mode before +following the corresponding ingestion or retrieval trace. An empty response +can result from filtering, unavailable stores, or ranking; the response alone +does not identify which one. + +For implementation work, use architecture's +[where to change what](../current/architecture.md#where-to-change-what) table, +then the relevant [change walkthrough](making-changes.md). Read both ends of a +serialized contract. Use the risk map in Testing rather than treating a single +unit test as complete integration evidence. + +## Boundaries and known questions + +The core behavior is repository-scoped parsing, source-backed retrieval, local +snapshot lifecycle, and checkout-bound MCP access. Model generation is optional. +Shared-store tenancy, full language semantics outside the bounded parsers, +general runtime impact proof, and broader SDLC or agent-memory systems remain +proposals unless source and current contracts establish otherwise. +[Next Steps](../planning/next-steps.md) controls activation of future work. + +This handbook does not update product-benefit results. Deterministic fixtures, +live-store integration, configured retrieval, and paired agent evaluation answer +different questions. The [evaluation chapter](../current/enrichment-and-evaluation.md) +and [methodology](../engineering/methodology.md) explain those boundaries. + +Source inspection also found limits worth preserving for maintainers: +publication is not a distributed transaction; cleanup has no shared-service +coordinator; freshness does not reparse the checkout; and some parser/snippet +edge cases need focused regression work. The concrete findings and their +verification limits are recorded in +[Inspection findings](development-and-debugging.md#inspection-findings). +They were documented without changing application code or claiming a live +reproduction. diff --git a/docs/guides/developer-walkthroughs.md b/docs/guides/developer-walkthroughs.md new file mode 100644 index 0000000..d04eee5 --- /dev/null +++ b/docs/guides/developer-walkthroughs.md @@ -0,0 +1,298 @@ +# Three developer walkthroughs + +Document type: source-traced developer guide + +These walkthroughs follow the inspected implementation recorded in the +[handbook](developer-handbook.md). They were not executed as live ingestion or +retrieval during the handbook review. Paths, `P`/`C`/`S` identities, and command +variables below are illustrative placeholders; no counts, exact ranks, hashes, +or provider results are asserted. + +The running source example is checked in inside +[`scripts/e2e-smoke.ps1`](../../scripts/e2e-smoke.ps1), function `New-SmokeFixture`. +It creates a temporary `CodeMeshSmokeFixture.csproj` targeting .NET 10 and a +`Downloader.cs` file. `Downloader.DownloadAsync` assigns +`LastUrl = Normalize(url)` and returns a completed task containing `download:` +followed by `LastUrl`; +`Normalize` trims and lowercases its input. The script initializes and commits +the fixture repository. This source is small enough to follow both a method +call and a property write without requiring a private sample checkout. + +Running the full smoke script starts local services and writes disposable store +state. Use the [Testing guide](../evaluation/testing.md#end-to-end-smoke) when +that work is intended. Reading these traces requires no running service. + +## Walkthrough 1: ingest and publish a small repository + +### Entry and parser selection + +After preparing a disposable copy of the script's fixture, an illustrative +invocation is the following. Shell: PowerShell. Working directory: the CodeMesh +repository root. It uses the supported ingest pattern in +[MCP Setup](mcp-setup.md#ingest-a-repository); the selected path must exist and +store endpoints must point at the intended local environment. + +```powershell +$fixtureRoot = "C:\path\to\prepared-smoke-fixture" +$fixtureProject = Join-Path $fixtureRoot "CodeMeshSmokeFixture.csproj" +dotnet run --project src/CodeMesh.Cli -- ingest ` + --root $fixtureRoot ` + --project $fixtureProject ` + --skip-embeddings ` + --json +``` + +[Program.cs](../../src/CodeMesh.Cli/Program.cs) dispatches `ingest` to +`RunIngestAsync`, resolves the repository and project, normalizes default C# +language, and constructs a local `CSharpParseService` client unless a parser URL +was configured. It constructs Neo4j, Mongo content/registry/summary, and Qdrant +adapters. Embeddings are skipped and summaries are not included. + +`IngestionOrchestrator.IngestAsync` captures Git provenance, resolves persistent +project `P` and checkout `C`, and creates a `ParseRequest`. These markers live +under the fixture's Git metadata, not in `Downloader.cs`. The parser receives +the root, selected project, language, options, and path filter. + +### Source to graph + +[`CSharpParseService.LoadProjectContextsAsync`](../../src/CodeMesh.Parser.CSharp/CSharpParseService.cs) +loads the explicit project through MSBuildWorkspace. The supported path uses +its compile items and compilation; if loading fails and fallback is used, +diagnostics describe the weaker source context. + +`ParseAsync` emits the file, then all logical symbols and declarations, then +relationships. For this fixture, the relevant transformation is: + +```mermaid +flowchart LR + C[Downloader class declaration] -->|Contains| D[DownloadAsync declaration] + D -->|Defines| M[DownloadAsync logical method] + M -->|Invokes| N[Normalize logical method] + M -->|Writes| P[LastUrl logical property] + C -->|Contains| ND[Normalize declaration] + ND[Normalize declaration] -->|Defines| N +``` + +This diagram is a selected subset of the graph, not the full emitted node/edge +count. Above the class, `Contains` connects the file to the namespace declaration +and that namespace to the class declaration. `ExtractDeclarationNodes` gives +source declarations concrete spans and +content hashes, while logical symbols use stable symbol keys and signature +content. `AddBodyReferenceRelationships` resolves the local call to `Normalize` +and the property assignment. Framework calls such as `Task.FromResult` are not +guaranteed to have indexed target nodes. + +### Graph to published selection + +The orchestrator rejects parser error diagnostics, redacts the returned text, +remaps changed hashes, and computes source snapshot `S0`. It stamps +`repositoryId=S0` for storage alongside `projectId=P` and `checkoutId=C` in +metadata. Public `IngestionResult.Repository.Id` remains `P`. + +For a first snapshot, there are no existing records to skip. `BuildPublicationAsync` +captures expected checkout-current and clean branch-head slots and stages +generation `G0`. `Neo4jGraphStore.UpsertGraphAsync` creates endpoint nodes before +relationships using snapshot-prefixed storage keys. Mongo content upserts text +by hash and adds references to `S0`. No model request is made by the skipped +embedding or omitted summary path. + +After namespace cleanup, `MongoRegistryStore.PublishSnapshotAsync` checks slot +expectations and revision-replaces the project's slot document. Successful +publication selects `S0` for checkout `C` and its clean named branch. The final +`WriteRegistryAsync` writes compatibility metadata and the completed ingestion +run/statistics. The exact partial-failure boundaries are in +[Identity and persistence](../current/identity-and-persistence.md#publication-order-and-atomicity). + +### Branches and evidence to inspect + +| Branch | Expected source behavior | Supporting tests | +| --- | --- | --- | +| `--dry-run` | Parse/redact/count without persistent markers, writes, generation, or model calls. | `TestDryRunSkipsStores`. | +| Parser error | Abort before staging/publication; CLI returns an error. | `TestMultiLanguageIngestionPublishesOneDeterministicSnapshot` includes parser-failure coverage. | +| Graph write fails after staging | Attempt to mark generation failed; previous slots survive. | `TestFailedSnapshotGenerationPreservesActiveSlots`. | +| Identical repeat ingest | Same effective snapshot, with unchanged records skipped. | `TestIncrementalIngestionSkipsUnchangedRecords`, `TestSnapshotSlotsRetainCleanHeadsAndBoundDirtyHistory`. | +| Redaction changes text | Remap hashes before content and provider use. | `TestIngestionRedactsSensitiveContentBeforeStoresAndEmbeddings`. | + +These named tests live in the [.NET harness](../../tests/CodeMesh.Tests/Program.cs). +The smoke script adds real adapter and REST coverage when deliberately run; +its `Invoke-JsonCommand` and later relationship/context checks show how it +observes the result. A unit fixture pass alone does not prove store publication. + +## Walkthrough 2: retrieve a context package + +### Start at a supported bound entry point + +Assume a reviewed normal-profile installation binds project `P`, checkout `C`, +the fixture's absolute root, and optionally `S0`'s source-view hash. The source is +still clean at its indexed commit. Installation is performed through +[MCP Setup](mcp-setup.md#repository-agent-onboarding); it is not a side effect of +retrieving context. + +Illustrative MCP call to `codemesh_get_context_package`: + +```json +{ + "query": "DownloadAsync normalization and LastUrl update", + "output_format": "agent" +} +``` + +This example asserts no particular rank. The omitted selector uses the server's +binding; it does not search every repository. + +### Resolve and verify before searching + +The registered wrapper in +[`mcp.build_server`](../../agent-access/codemesh_agent_access/mcp.py) checks the +caller selector, forwards the binding through +[`tools.get_context_package`](../../agent-access/codemesh_agent_access/tools.py), +and constructs a `ContextPackageQuery` for +[`CodeMeshReadStore.get_context_package`](../../agent-access/codemesh_agent_access/store.py). + +The facade calls `get_repository_status` with the binding. Mongo +`get_bound_repository` resolves `C`'s checkout-current slot and observation, +then `_repository_freshness` compares recorded provenance to local Git state. +`assess_binding` requires exact identities/root and acceptable freshness. A +rejected assessment throws before `search_context`. On acceptance, the package +query's `repository_id` becomes the exact snapshot namespace `S0`. + +### Search and transform hits + +With query provider mode `none`, `search_context` starts lexical graph and +stored-summary search without a model request. With no generated summaries, +lexical results provide the candidates. The graph query acquires matching +symbol/path/metadata rows; `_lexical_score` evaluates query-term coverage; +candidate diversification prevents one file from consuming every candidate +when alternatives exist. + +`_build_context_hit` attaches a source declaration to a logical method where +available. For a `DownloadAsync` hit, the logical node identity can remain the +method while the hit's span and hash point to its source declaration. The +selected source path is `Downloader.cs`; its exact line numbers come from the +actual prepared fixture, not this guide. + +`_ranked_unique_hits` deduplicates parser node ids, resolves relevance ties, and +diversifies by file. The tiny fixture has few distinct files, so do not expect +the multi-file diversity examples from larger evaluation suites to appear here. + +### Hydrate, allocate, and format + +For each hit, `_hydrate_context_package_item` calls `get_node` in `S0`, obtains +content, falls back to the hit hash if necessary, reads an optional completed +summary, and reads useful relationships. An outgoing `Invokes` edge to +`Normalize` can become a callee; the write to `LastUrl` can appear in the writes +group. Group/total limits and the selected hit determine what is included. + +The assembly loop allocates remaining snippet characters across remaining +items. The response carries public project provenance, snapshot/content +identity, source span, rank and score components, and heuristic validation +suggestions. `format_context_package_for_agent` turns it into a brief. The MCP +wrapper caps the entire brief and returns a single text block, avoiding a +duplicate structured copy. Choosing `output_format=json` instead returns the +structured package, where `max_characters` constrains snippets rather than +total serialized size. + +Follow the returned source spans into the checkout before concluding that a +change is safe. A bounded package is a selection of evidence, not the full +call graph or a completed test run. + +### Supporting tests and alternate entry points + +`test_context_package_rejects_unaccepted_binding_before_search` in +[test_store_components.py](../../agent-access/tests/test_store_components.py) +protects the gate. `test_context_package_includes_snippet_relationships_and_registry_metadata` +and `test_context_package_uses_snapshot_namespace_for_relationships` in +[test_context_ranking.py](../../agent-access/tests/test_context_ranking.py) +protect assembly and namespace selection. The +[MCP](../../agent-access/tests/test_mcp_contract.py) and +[formatting](../../agent-access/tests/test_formatting.py) tests protect delivery. + +REST `/context/package` and the .NET client's `GetContextPackageAsync` enter the +same facade through HTTP and the canonical request model, but do not supply the +normal MCP binding. They therefore share ranking/assembly while retaining their +unbound selection behavior. The Python CLI also reads stores directly. The +[retrieval chapter](../current/retrieval-internals.md) explains these differences. + +## Walkthrough 3: refresh changed source and handle stale access + +### A dirty edit before refresh + +Starting from `S0`, suppose the developer changes `Normalize`'s implementation. +Until refresh, the stores still contain the older source. A normal bound package +checks Git status, reports stale, and rejects retrieval. It does not invoke +ingestion automatically. The agent can continue direct source inspection while +the developer decides how to update the index. + +Even after a dirty-source refresh publishes a new checkout-current snapshot, +normal MCP still rejects the dirty view. This is deliberate freshness behavior: +the read path cannot verify arbitrary working-tree contents by commit alone. +Administrative status/read surfaces can inspect it with their documented scope. + +### Refresh uses the same pipeline + +Illustrative command, PowerShell from the CodeMesh root, retaining the prepared +fixture variables from walkthrough 1: + +```powershell +dotnet run --project src/CodeMesh.Cli -- refresh ` + --root $fixtureRoot ` + --project $fixtureProject ` + --skip-embeddings ` + --json +``` + +[`RunRefreshAsync`](../../src/CodeMesh.Cli/Program.cs) calls +[`CodeMeshRefreshWorkflow.ResolveTarget/BuildRefreshIngestArgs`](../../src/CodeMesh.Control/Configuration/CodeMeshRefreshWorkflow.cs), +then `RunIngestAsync`. No distinct refresh parser runs. The C# parser reads the +selected project again, the changed file/declaration hashes change the source +view, and the orchestrator stages snapshot `S1` under the same project/checkout. + +For a dirty checkout, publication moves checkout-current `C → S1` and preserves +the clean branch-head `main → S0`. If another dirty refresh produces `S2`, `S1` +becomes collectible only when no pin/evidence/other slot references it; collection +must then wait for its grace period. This does not delete `S0` while branch-head +retains it. + +If the developer commits the exact source that produced `S1` and refreshes, +the source snapshot can be reused while the new observation records the clean commit. +Branch-head can then move to `S1`. Commit identity is observation data rather +than a component of the source snapshot hash. Bound freshness can accept that +clean observation, provided all binding dimensions match. If installation pinned +the old source-view hash, a reviewed binding update and probe are also needed; +refresh does not silently rewrite host configuration. + +### A store failure during refresh + +For a separate failure branch starting from published `S0`, suppose the changed, +committed fixture parses successfully, but a graph write throws after staging. +`IngestAsync` catches the failure inside its persistent +write block, calls `MarkGenerationFailedAsync`, and rethrows to the CLI. The old +slots still point at `S0`; partial data for the new namespace can remain. +Because the checkout commit has changed, normal bound access to the old +observation remains stale. A reachable service and old successful run do not +mean the failed refresh was accepted. + +Inspect CLI diagnostics, the selected slot/observation, and store health. After +the store issue is resolved, an intentional refresh can reuse deterministic +identity and reconcile partial writes. Garbage collection is a separate local +API operation; it is not automatically scheduled by refresh. Concurrent pin/ +publication/cleanup guarantees are limited as described in +[Identity and persistence](../current/identity-and-persistence.md#retention-pins-and-deletion). + +### Watch variant and evidence + +Watch performs the initial ingest, then +[`CodeMeshWatchWorkflow.CreateSnapshot`](../../src/CodeMesh.Control/Configuration/CodeMeshWatchWorkflow.cs) +compares eligible paths, file sizes, and write timestamps. On a change it runs +the same ingestion arguments. Git metadata-only changes may not trigger it; +use explicit refresh after such changes. A thrown ingest error terminates this +watch path rather than retrying forever. + +`TestSnapshotSlotsRetainCleanHeadsAndBoundDirtyHistory` uses a temporary Git +repository and controllable parser output to exercise clean/dirty retention and +snapshot reuse. `TestFailedSnapshotGenerationPreservesActiveSlots` injects a +`FailingGraphStore`. `TestWatchWorkflowDetectsFileChanges` and the argument +normalization tests cover polling and option preservation. All live in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs). Python's binding/freshness +tests cover the rejection side; these are complementary invariants, not a claim +that this illustrative sequence was run end to end during documentation review. diff --git a/docs/guides/development-and-debugging.md b/docs/guides/development-and-debugging.md new file mode 100644 index 0000000..7a87611 --- /dev/null +++ b/docs/guides/development-and-debugging.md @@ -0,0 +1,213 @@ +# Development and debugging + +Document type: human developer operating guide + +Use the [handbook](developer-handbook.md) for the reading path and +[COMMANDS.md](../../COMMANDS.md) for canonical invocations. This guide explains +what to inspect and why. Examples use PowerShell from the CodeMesh root unless +explicitly stated. They are supported command examples checked against source; +they are not a record that local services or application suites were run during +the handbook review. + +## Prepare a development checkout + +CodeMesh's projects target .NET 10, and Agent Access declares Python 3.12 or +newer in [pyproject.toml](../../agent-access/pyproject.toml). The repository's +usual IDE/shell combination is Rider on Windows with PowerShell; the baseline +workflow also defines Linux verification. Install the SDK, Python, `uv`, Git, +and Node/npm for Markdown linting. Docker Compose is needed for live-store work, +not for the in-process Python fixture evaluator. Exact dependency versions come +from the committed NuGet lock files and `agent-access/uv.lock`. + +Establish identity before editing: + +```powershell +git rev-parse --show-toplevel +git status --short --branch +git rev-parse HEAD +``` + +Read [CONTRIBUTING.md](../../CONTRIBUTING.md), the local `AGENTS.md`, and the +[risk-based test guide](../evaluation/testing.md). Run .NET commands one at a +time so concurrent builds do not contend for the same intermediate files: + +```powershell +dotnet restore CodeMesh.sln --locked-mode +dotnet build CodeMesh.sln --no-restore +``` + +Then initialize Python from `agent-access`: + +```powershell +Set-Location agent-access +uv sync --locked +``` + +`uv sync --locked` includes the development dependency group containing pytest +and Ruff. The .NET tests are a console application invoked with `dotnet run`, +not a discovered xUnit/NUnit suite invoked with `dotnet test`. Use the commands +in [COMMANDS.md](../../COMMANDS.md#deterministic-verification) without substituting +a command that exercises no tests. + +The shell-neutral `git`, `dotnet`, `uv`, `python`, and `npx` argument forms also +appear in the checked-in [Linux workflow](../../.github/workflows/baseline-standards.yml). +PowerShell backtick continuations, `$env:` assignments, `Invoke-RestMethod`, and +the smoke script are PowerShell-specific. A workflow definition is evidence of +the intended alternate environment, not proof of an exact-candidate Linux or +Windows run. Use the selected shell explicitly in any validation report. + +## Choose the smallest runtime + +For documentation or pure Python formatting/ranking work, start with repository +standards and the affected deterministic tests. For parser logic, use the .NET +harness and its synthetic temporary projects. The full .NET harness can probe +configured local stores and execute disposable round-trips if reachable; read +the integration setup before treating it as entirely offline. + +For a running product, follow [MCP Setup](mcp-setup.md#start-services) in order: +configure local environment, start stores, ingest the target, then run the +chosen read surface. The CLI's local C# parser does not require the C# HTTP +worker. Normal MCP is a local stdio process with direct store connections; +it does not require a separate REST process. REST is needed for the UI, .NET +client, and REST-based smoke operations. + +[docker-compose.yml](../../docker-compose.yml) binds published ports to loopback. +The C# parser container sees `/workspace`; its path must be meaningful inside +that process. Agent Access's container has no equivalent arbitrary host-checkout +mount, so Git freshness may be unavailable for host paths. Prefer the documented +local stdio setup when exact host checkout access is required. + +## Configuration that actually controls execution + +| Launch | Configuration source | Practical consequence | +| --- | --- | --- | +| .NET CLI | CLI option, then environment, then factory default; `.env` loader fills absent variables from the initial root. | An existing process variable wins over `.env`; changing the file may appear ineffective. | +| Python REST/MCP/CLI | `Settings.from_env` and binding arguments/environment. | No implicit `.env` loading; the parent process must supply the intended environment. | +| Compose | Compose interpolation and service-specific environment mappings. | Container hostnames/ports differ from host-side endpoints. | +| `codemesh.yaml` | Workspace discovery marker and inventory. | Its example keys do not configure runtime factories. | + +Implementation sources are +[EnvironmentFileLoader](../../src/CodeMesh.Control/Configuration/EnvironmentFileLoader.cs), +[CLI factories](../../src/CodeMesh.Cli/Program.cs), and +[Settings.from_env](../../agent-access/codemesh_agent_access/config.py). +`dependencies.get_settings/get_store` cache configuration in a Python process; +restart the affected process when intentionally changing its configuration. +Restarting the REST container does not restart an already-running local MCP +server in an IDE host. + +Use placeholders such as `C:\path\to\checkout`, `prj_EXACT`, and +`` in documentation. Inspect variable **names** and the +configured endpoint/collection identity without dumping `.env`, credentials, +or a whole process environment. Keep the actual values in local configuration. + +The initial .NET environment load recognizes `--root`, before command-specific +normalization. `--repository-root` is accepted later by ingestion but does not +change that initial load. An explicit `--root` is the clearest launch when +target and current directories differ. + +## Useful debugging entry points + +In Rider, open `CodeMesh.sln`. For CLI behavior, set a run configuration for +`CodeMesh.Cli`, working directory at the repository root, and the intended +arguments. Start at `RunIngestAsync`; step into `IngestionOrchestrator.IngestAsync` +to separate parsing from persistence. Use a disposable fixture and the supported +dry-run path before debugging writes. For C# shape problems, break in +`LoadProjectContextsAsync`, `ExtractDeclarationNodes`, or `ExtractRelationships`. + +For the Python facade, use a Python debugger in the `agent-access` environment. +Break at `get_context_package`, `get_repository_status`, `_search_lexical_context`, +`_build_context_hit`, or `_hydrate_context_package_item` in +[store.py](../../agent-access/codemesh_agent_access/store.py). A ranking test's +stub store is a useful starting point because it isolates scoring/assembly from +live adapter state. For surface mapping, start in `rest.py` or the registered +function inside `mcp.build_server` before entering the facade. + +Do not write diagnostic stdout into an MCP stdio server: stdout carries the +protocol. Use debugger state or the existing sanitized timing sideband for +package attribution. Ordinary CLI output includes diagnostics and write/skip +statistics; `--json` uses the serialized ingestion result. The stdout/stderr +paths are visible in [Program.cs](../../src/CodeMesh.Cli/Program.cs) and +[mcp.py](../../agent-access/codemesh_agent_access/mcp.py). + +## Health, logs, and evidence capture + +From the CodeMesh root in PowerShell, the +[local health commands](../../COMMANDS.md#local-product-health) inspect a +configured environment: + +```powershell +dotnet run --project src/CodeMesh.Cli -- doctor +dotnet run --project src/CodeMesh.Cli -- status +docker compose ps +Invoke-RestMethod http://127.0.0.1:8088/health +``` + +`CodeMeshStatusService` reports workspace/component configuration and endpoint +health. `doctor` adds advice and has different exit behavior from `status`; +inspect the structured component states rather than inferring readiness from +an exit code alone. Python health checks graph, content, summaries, vectors, +and query provider, reporting degraded if a component is not healthy/configured. +Neither health call proves freshness or retrieval quality. + +For container errors, use the bounded diagnostic command indexed in +[COMMANDS.md](../../COMMANDS.md#local-product-health): + +```powershell +docker compose logs --tail 100 agent-access csharp-parser +``` + +Review diagnostic text before sharing it: exceptions can contain local paths, +provider response details, or configuration context. Preserve the command, +process/candidate, selected project/checkout/snapshot, first failure, and +sanitized diagnostic codes. Run history is useful but incomplete as an error +log, because early parser failures occur before a persisted failed run exists. + +## Diagnose by symptom + +| Symptom | Likely causes and first inspection | Supported recovery | +| --- | --- | --- | +| Normal MCP refuses startup | Missing/incomplete binding or unknown profile. Inspect `binding_from_values`, `mcp.run`, and the reviewed launch. | Use the exact installation/binding procedure in [MCP Setup](mcp-setup.md#repository-agent-onboarding). | +| `snapshot_not_fresh` | Commit mismatch, dirty current/indexed view, or unverifiable source provenance. Inspect bound repository status and local Git state. | Finish/record the intended source change, refresh the exact clean checkout and scope, then rerun the reviewed probe. | +| `repository_root_mismatch` or `checkout_mismatch` | Moved checkout, copied marker, wrong host process, or wrong installation target. Inspect registered checkout and launch root. | Correct/review the binding for the intended checkout; do not switch to an unbound profile as a product acceptance substitute. | +| `source_view_hash_mismatch` | Plan pins the previous source view after refresh. | Review a new plan for the new identity and probe it; refresh does not edit installation. | +| Repository missing or ambiguous | No successful publication, wrong Mongo database/collection, colliding alias, or ambiguous slot key. | Check exact project/checkout selectors and configured stores; use the intended explicit identity. | +| Freshness unknown in REST container | Host root/Git inaccessible inside that container. | Inspect through the documented local process with checkout access; preserve the container's unknown result. | +| `CMSHARP010` and missing semantic edges | MSBuild project/solution could not load; filesystem fallback used. | Restore the target using its own supported procedure, verify selected manifest, and inspect workspace diagnostics before reingestion. | +| A file never appears | Wrong language, Compile items, generated/secret/deny filters, linked path, or structural parser limitation. | Compare parser selection and policy to the actual file; use parser dry-run and a focused fixture. | +| Search returns no hits | Wrong namespace or exact filters, metadata query miss, bounded candidate pool, adapter degradation. | Inspect health, resolved query/diagnostics, then a known symbol/path through diagnostic tools. | +| Hit has a zero or unexpected span | Logical C# symbol needs its declaration; declaration missing or selected content differs. | Inspect `Defines`, hit expansion, and the declaration/content hash. | +| Snippet or relationship missing | Output/group limits, omitted content, absent record, or parser resolution limit. | Inspect structured package controls and source span; increase only the relevant diagnostic limit when useful. | +| Completed ingest but failed summaries | Per-node provider errors are persisted without failing source ingestion. | Inspect summary coverage and provider metadata; any provider rerun follows its own authorization and qualification procedure. | +| Refresh fails but old results remain | New generation failed before slot publication. | Preserve the failure, repair the relevant dependency, then intentionally refresh; do not erase the previous index to hide the failure. | +| Watch misses a change or exits | Timestamp/size detector, Git-only change, or an uncaught ingest exception. | Explicit refresh with the same scope; inspect the original exception before restarting watch. | +| Vector count/width error | Write/query model mismatch or incompatible existing collection. | Compare exact model, dimensions, and collection; use the documented embedding verification, preserving unrelated collections. | + +The relevant source is linked in +[Ingestion internals](../current/ingestion-internals.md), +[Retrieval internals](../current/retrieval-internals.md), and +[Identity and persistence](../current/identity-and-persistence.md). The table +describes diagnostic paths, not permission to mutate any arbitrary configured +store or to repeat a consumed evaluation campaign. + +## Inspection findings + +The following are concrete source-review findings recorded for future focused +work. No application fix, live reproduction, or new regression test was made +as part of this documentation task. Existing linked tests cover neighboring +behavior; they are not claimed as tests of each uncovered edge case. + +| Finding | Source and consequence | Focused verification needed | +| --- | --- | --- | +| Fragment line slicing can use the wrong coordinate space | [`_snippet_for_hit`](../../agent-access/codemesh_agent_access/store.py) slices whenever the hit's file-absolute line numbers fit within the loaded text's line count. A declaration fragment starting after line 1 can be long enough to satisfy that test, causing it to be sliced as if it were a whole file. | A fragment fixture with non-1 source start and enough lines to enter the slice branch; assert exact emitted source. | +| Markdown headings in fences and repeated titles are not distinguished | [`FindHeadings`/`AnchorFor`](../../src/CodeMesh.Parser.Markdown/MarkdownParseService.cs) use line regexes and unsuffixed anchors. Fenced headings can become sections and repeated anchors can produce duplicate ids. | Fenced-code and repeated-heading parser fixtures, including composite duplicate handling. | +| Refresh path discovery differs from ingest | [`CodeMeshRefreshWorkflow`](../../src/CodeMesh.Control/Configuration/CodeMeshRefreshWorkflow.cs) checks `.git` as a directory, while linked worktrees commonly use a `.git` file; explicit solution paths use caller-relative `Path.GetFullPath`. Ingest uses a different resolver. | Nested-start linked-worktree fixture without another root marker, and a manifest relative only to the selected root. Use explicit root/absolute manifest meanwhile. | +| Publication metadata and cleanup are outside the slot CAS | [`PublishSnapshotAsync`](../../src/CodeMesh.Storage/Mongo/MongoRegistryStore.cs) updates metadata before replacing project slots; [`CollectAsync`](../../src/CodeMesh.Ingestion/SnapshotGarbageCollector.cs) deletes data before its final registry reference check. | Fault injection around revision conflicts and a pin concurrent with collection. The local contract does not supply a shared-store coordinator. | +| Source capture is not a filesystem transaction | [`IngestAsync`](../../src/CodeMesh.Ingestion/IngestionOrchestrator.cs) captures Git before parsing; [`_repository_freshness`](../../agent-access/codemesh_agent_access/store.py) checks Git/recorded provenance without recomputing parser output. | Source edits during parse/read and ignored-but-eligible source changes. Keep source stable during evidence ingestion. | + +Treat these as maintenance leads and stated limits, not an activated backlog. +[Next Steps](../planning/next-steps.md) retains execution-order authority. +Documented design rationale is in +[ADR 0001](../decisions/0001-local-repository-identity-and-snapshot-publication.md), +[ADR 0002](../decisions/0002-summary-qualification-evidence-boundary.md), and +[ADR 0003](../decisions/0003-hard-capped-codex-evaluation-runner.md). The observations +above are source analysis; they do not invent a historical reason for a defect. diff --git a/docs/guides/making-changes.md b/docs/guides/making-changes.md new file mode 100644 index 0000000..c0cd039 --- /dev/null +++ b/docs/guides/making-changes.md @@ -0,0 +1,184 @@ +# Making changes to CodeMesh + +Document type: task-oriented developer guide + +These walkthroughs explain existing extension points without implementing a new +feature. Start with [Architecture](../current/architecture.md#where-to-change-what) +to locate the subsystem, and use [Testing](../evaluation/testing.md) to select +checks. The producer, persisted representation, read consumer, and interface +contract often live in different projects or languages. + +## Change a parser's output + +Suppose an evidenced C# call-graph bug needs a corrected invocation edge. Begin +with a minimal fixture modeled on +`TestCSharpParserEmitsInvocationAndMemberAccessRelationships` in +[tests/CodeMesh.Tests/Program.cs](../../tests/CodeMesh.Tests/Program.cs). Keep the +example small enough to show the source expression, expected source/target +symbols, and any edge that must be absent. Use a multi-project fixture like +`TestCSharpParserEmitsCrossProjectInvocationRelationships` if resolution across +compilations is the actual failure. + +Trace the producer in +[CSharpParseService.cs](../../src/CodeMesh.Parser.CSharp/CSharpParseService.cs). +`ExtractDeclarationNodes` must register target symbol keys before the later +`ExtractRelationships` pass. For an invocation problem, follow +`AddBodyReferenceRelationships`, `ResolveMethodSymbol`, and +`AddSymbolRelationship`; the last resolves only targets that correspond to +indexed symbols. A missing external framework node is different from a broken +local method lookup. + +Before changing ids, decide whether the change affects the logical symbol, +source declaration, or just edge metadata. Review +[CodeMeshFingerprint](../../src/CodeMesh.Domain/Utilities/CodeMeshFingerprint.cs) +and [SnapshotIdentity](../../src/CodeMesh.Ingestion/SnapshotIdentity.cs): stable +metadata and relationships influence source identity, while volatile provenance +does not. Parser binary changes are not automatically fingerprinted. Changing a +schema or profile may therefore require an explicit version decision, with an +ADR for a consequential implemented change. + +Then inspect the consumers. The +[Neo4j adapter](../../src/CodeMesh.Storage/Neo4j/Neo4jGraphStore.cs) persists the +edge, Python [graph_store.py](../../agent-access/codemesh_agent_access/graph_store.py) +retrieves it, and [store.py](../../agent-access/codemesh_agent_access/store.py) +prioritizes/groups it. An edge can be present in Neo4j yet absent from a package +because of direction, grouping, total limits, or candidate selection. Add a +consumer assertion only when the change actually alters that observable behavior. + +| Contract to preserve | Check location | +| --- | --- | +| Every edge endpoint corresponds to the intended local node identity | C# parser fixture and its symbol/declaration assertions. | +| Same effective source remains deterministic | Snapshot/multi-language tests where identity semantics change. | +| New metadata is stored and interpreted consistently | Storage adapter mapping plus Python normalizer tests. | +| Agent-visible relationship remains source-grounded and bounded | Context relationship tests and MCP fixture evaluator. | + +For a parser-only fix, use the affected .NET checks. If graph serialization, +lifecycle, or an Agent Access consumer changes, run both suites. Live wiring or +cleanup changes also select the authorized end-to-end smoke path. Update +[Ingestion internals](../current/ingestion-internals.md), +[Architecture](../current/architecture.md), and any changed canonical contract; +update current status only with the resulting evidence actually obtained. + +The same pattern applies to structural parsers, but their evidence differs. +For a Python/Rust boundary, review both +[Python import metadata](../../src/CodeMesh.Parser.Python/PythonParseService.cs) +and [Rust export metadata](../../src/CodeMesh.Parser.Rust/RustParseService.cs), +then `CompositeParserClient.AddPyO3Relationships`. Preserve ambiguity rejection +and one combined generation. Adding a parser client also requires CLI selection, +capability/version, path policy, composition, tests, and documentation; the +interface seam alone is not an authorized parser-expansion task. + +## Change context ranking + +Suppose a recorded multi-term query returns repeated declarations from one file +and misses a relevant implementation file. First distinguish candidate acquisition +from final order. Inspect `search_context_nodes` in +[graph_store.py](../../agent-access/codemesh_agent_access/graph_store.py), then +`_search_lexical_context`, `_lexical_score`, `_diversify_lexical_nodes`, +`_build_context_hit`, and `_ranked_unique_hits` in +[store.py](../../agent-access/codemesh_agent_access/store.py). A final-sort change +cannot improve recall for a node excluded from the raw candidate pool. + +Create a regression example using the stub patterns in +[test_context_ranking.py](../../agent-access/tests/test_context_ranking.py). +Express the observable ranking or file-coverage requirement, rather than +asserting every intermediate numeric constant. Preserve existing tests for +parameter expansion, multi-term coverage, saturated-score tie-breaking, and +distinct-file selection. A score change may also change which declaration and +content hash supply the package's citation; verify that the source remains +coherent, not just that the expected filename appears. + +Use the Python tests, Ruff checks, and deterministic MCP fixture evaluator from +[COMMANDS.md](../../COMMANDS.md#python-agent-access). .NET checks are unnecessary +for a purely Python ranking implementation unless a shared contract changes. +When a task requires live retrieval evidence, use a clean exact candidate and +the existing frozen suite under its controlling procedure. Do not modify frozen +targets or thresholds to make a change appear beneficial. Add a separately +identified diagnostic suite if the measured question is new. + +If the problem is latency, use the existing package timing sideband to identify +the stage before changing concurrency. Search sources already run concurrently; +candidate and item hydration include sequential work. A passing deterministic +ranking test cannot establish a speedup or normal-agent benefit. Keep performance, +retrieval quality, and paired-agent evidence separate in documentation. + +Update [Retrieval internals](../current/retrieval-internals.md) and +[Agent Access Contracts](../current/agent-access-contracts.md) when selection or +diagnostics visibly change. Historical reports stay bound to their original +candidates. + +## Extend an existing interface field + +Suppose an authorized change adds a diagnostic field to a context response. +Start by identifying whether it belongs in the shared structured package or is +specific to one presentation. For example, the existing MCP-only +`codemesh_next_action` status field is presentation guidance, while snapshot +identity belongs in the common response. Avoid copying a presentation-only +field into every storage record. + +Trace this alignment chain: + +1. Define the producer and missing/unknown behavior in + [store.py](../../agent-access/codemesh_agent_access/store.py) or the relevant + adapter. A dictionary field still needs a documented meaning and type. +2. Update the corresponding Pydantic model in + [models.py](../../agent-access/codemesh_agent_access/models.py) if the field + is part of a typed request or response. +3. Review [rest.py](../../agent-access/codemesh_agent_access/rest.py), + [tools.py](../../agent-access/codemesh_agent_access/tools.py), and both the + manifest and registered signatures in + [mcp.py](../../agent-access/codemesh_agent_access/mcp.py). The printable + manifest and actual tool registration are separate representations that tests + keep aligned. +4. Review [cli.py](../../agent-access/codemesh_agent_access/cli.py), + [formatting.py](../../agent-access/codemesh_agent_access/formatting.py), and + [web.py](../../agent-access/codemesh_agent_access/web.py) for presentation. + Preserve the text-only MCP agent form and its whole-response character cap. +5. Update + [AgentAccessContracts.cs](../../src/CodeMesh.Domain/Contracts/AgentAccessContracts.cs) + and [AgentAccessClient](../../src/CodeMesh.Control/AgentAccess/AgentAccessClient.cs) + when the .NET surface consumes it. Check explicit JSON names and nullability; + PascalCase C# names are not the wire schema. + +Run both suites for this serialization/interface boundary. Python REST/MCP/CLI +tests assert forwarding and output; `TestAgentAccessContractsMapRestJson` and +`TestAgentAccessClientCallsRestEndpoints` in the +[.NET harness](../../tests/CodeMesh.Tests/Program.cs) protect the other end. +Use the smoke path if the change affects REST integration or live wiring under +the [risk rules](../evaluation/testing.md). Update the canonical +[Agent Access Contracts](../current/agent-access-contracts.md) with the exact +field and failure behavior, then update handbook explanations that depend on it. + +## Modify storage or lifecycle behavior + +Begin at [StorageContracts.cs](../../src/CodeMesh.Storage/StorageContracts.cs), +then read the .NET adapter, in-memory implementation, Python reader, and any +administrative delete path. An interface such as `IIncrementalContentStore` +promises namespace-aware state/removal; implementing a global delete instead +would break retention even if a one-repository fixture passed. + +For publication work, explicitly identify what is inside the project-document +compare-and-swap and what happens before or after it. Exercise conflict and +partial-failure paths in addition to the success path. For retention work, +protect referenced snapshots and shared content, and verify retry behavior +after one store fails. Keep +[ADR 0001](../decisions/0001-local-repository-identity-and-snapshot-publication.md) +and [Identity and persistence](../current/identity-and-persistence.md) aligned +with the implemented boundary. Both language suites and the end-to-end smoke +path are selected for these changes; a lock-protected in-memory pass is not +equivalent to a real-store race test. + +## Complete the change + +Review the final diff and preserve unrelated edits. Report passed, failed, +skipped, and unavailable checks separately, including integration skips and +unexecuted live/model work. Apply the repository standards checks, use a focused +Conventional Commit, and validate its range as indexed in +[COMMANDS.md](../../COMMANDS.md#repository-standards). A commit does not authorize +publication, a model campaign, or a store mutation outside the selected work. + +Update normative behavior in its canonical reference, verified status in +[Project Status](../current/project-status.md), and future execution order only +in [Next Steps](../planning/next-steps.md) when that order has actually changed. +Use the [methodology](../engineering/methodology.md) to distinguish implemented +behavior from inspected evidence and later acceptance states. diff --git a/docs/guides/mcp-setup.md b/docs/guides/mcp-setup.md index 604391a..30f3ea7 100644 --- a/docs/guides/mcp-setup.md +++ b/docs/guides/mcp-setup.md @@ -55,6 +55,33 @@ dotnet run --project src/CodeMesh.Cli -- ingest ` --skip-embeddings ``` +For one atomic Python/Rust snapshot, select both languages in one ingestion; +do not run separate replacement ingestions: + +```powershell +dotnet run --project src/CodeMesh.Cli -- ingest ` + --root $repositoryRoot ` + --languages python,rust ` + --skip-embeddings +``` + +Known secret files are excluded by default. To restrict a repository further, +repeat repository-relative allow or deny globs; deny patterns take precedence: + +```powershell +dotnet run --project src/CodeMesh.Cli -- ingest ` + --root $repositoryRoot ` + --languages python,rust ` + --allow-path "app/**" ` + --allow-path "native/**" ` + --deny-path "app/generated/**" ` + --skip-embeddings +``` + +Use `--include-known-secret-files` only after reviewing the broadened scope. It +does not disable value redaction, configured deny patterns, or generated/cache +directory exclusions. See [Security and Redaction](../current/security-and-redaction.md). + Verify repository discovery: ```powershell @@ -70,6 +97,15 @@ dotnet run --project src/CodeMesh.Cli -- self smoke --skip-embeddings ## Run The MCP Server +The normal runtime requires a complete checkout binding. Prefer the reviewed +[installation plan](#repository-agent-onboarding), which puts the binding in +the launch arguments. The generic launch examples in this section assume the +parent process supplies `CODEMESH_PROJECT_ID`, `CODEMESH_CHECKOUT_ID`, and +`CODEMESH_REPOSITORY_ROOT` (plus optional `CODEMESH_SOURCE_VIEW_HASH`) from the +intended indexed checkout. Without arguments or those variables, normal startup +fails before serving tools. The Python process also needs the intended store +configuration; it does not load `.env` automatically. + Use the Python package from `agent-access`: ```powershell @@ -144,49 +180,173 @@ descriptions and printable stable manifest. The `normal` profile is the default and exposes only the context-package, repository-status, repository-listing, and node-read tools. Use -`--profile diagnostic` for non-destructive operational investigation. Unknown -profile names fail before server startup, and neither profile exposes repository -deletion. +`--profile diagnostic` for non-destructive operational investigation. The +`development-feedback` and `feedback-maintainer` profiles are available only +through the explicit session procedure below. Unknown profile names fail before +server startup, and no profile exposes repository deletion. ## Repository Agent Onboarding Explicit repository onboarding is the selected v1 product contract. MCP server instructions remain defense in depth because hosts differ in whether and how -they present those instructions to an agent. The tool profiles are implemented; -the future live runtime probe, checkout binding, fail-closed package freshness, -and installation behavior are specified in -[Agent Integration Contract](../planning/agent-integration-contract.md). - -Until reviewable installation support exists, repository owners may manually -add this canonical block to the repository's `AGENTS.md` or equivalent durable -agent-instruction file: - -```markdown - -## CodeMesh repository context - -For C#/.NET implementation discovery, likely change-impact analysis, and -validation selection in this repository: - -1. Start with `codemesh_get_context_package` using the full task, the configured - repository id or unique alias, `output_format = "agent"`, and default limits. -2. If the repository selector is unknown, use `codemesh_list_repositories`; do - not guess when more than one checkout could match. -3. Before relying on the package, use `codemesh_get_repository_status` to confirm - that the indexed snapshot matches the intended checkout and is fresh. -4. Verify consequential conclusions against the returned source spans with - targeted reads. Treat rankings and relationships as navigation evidence, not - runtime proof. -5. If CodeMesh is unavailable, stale, or ambiguous, report that condition and - continue with normal repository exploration. - +they present those instructions to an agent. Generate a reviewable exact- +checkout plan after ingestion reports the project, checkout, snapshot, and +source-view identities: + +```powershell +Set-Location agent-access +uv run python -m codemesh_agent_access install plan ` + --target-root C:\path\to\checkout ` + --codemesh-root C:\path\to\CodeMesh ` + --project-id prj_EXACT ` + --checkout-id chk_EXACT ` + --source-view-hash HASH_FROM_INGESTION ` + --output C:\temp\codemesh-install-plan.json +``` + +Review the plan's exact launch, binding, complete file contents, diffs, target +hashes, forwarded environment names, and `plan_hash`. Apply only that unchanged +plan and hash, then run the live provider-free probe: + +```powershell +uv run python -m codemesh_agent_access install apply ` + C:\temp\codemesh-install-plan.json ` + --approve-plan-hash REVIEWED_HASH +uv run python -m codemesh_agent_access mcp-probe ` + --plan C:\temp\codemesh-install-plan.json +``` + +The default plan manages `.codex/config.toml` and a marked onboarding block in +`AGENTS.md` while preserving other content. Use `--configuration-only` only when +an external instance manager owns repository guidance. Apply rejects a changed +target or plan. The generated config forwards secret variable names, never +their values, and forces provider mode to `none`. + +The bound normal server requires an exact project, checkout, and root. Its tools +inject that project selector and reject caller overrides. Start with one +`codemesh_get_context_package` request using a concise task-shaped query, +agent output, and default limits. Stop retrieving when it covers every required +facet. Only issue one focused follow-up for a specific unresolved implementation +or test facet, use at most three packages for the task, and avoid overlapping +queries. The package validates the binding +and freshness before search; dirty, moved, stale, missing, or source-view- +mismatched checkouts fail closed. Use repository status only for explicit +ingestion, count, or freshness diagnostics. + +## Agent Feedback + +### Explicit development session + +Use this path only when a human is deliberately running a CodeMesh development +session and wants client agents to submit local diagnostic feedback for review +by an agent in the CodeMesh checkout. It does not authorize a CodeMesh edit. + +First add `/.codemesh-feedback/` to each participating client repository's +managed ignore rules. Session planning validates this without editing ignore +files. From `agent-access`, create a proposed manifest and activation plan. Each +`--client-json` value names one exact participant: + +```powershell +uv run python -m codemesh_agent_access feedback session plan ` + --manifest-path C:\temp\codemesh-feedback-session.json ` + --codemesh-root C:\path\to\CodeMesh ` + --codemesh-project-id prj_CODEMESH ` + --codemesh-checkout-id chk_CODEMESH ` + --client-json '{"project_id":"prj_CLIENT","checkout_id":"chk_CLIENT","repository_root":"C:\\path\\to\\client","reporter_role":"client-agent"}' ` + --expires-at 2026-09-10T18:00:00+02:00 ` + --output C:\temp\codemesh-feedback-session-plan.json +``` + +Review the exact roots, identities, expiry, bounds, tools, manifest-file hash, +and `plan_hash`. Activate only the unchanged plan: + +```powershell +uv run python -m codemesh_agent_access feedback session activate ` + C:\temp\codemesh-feedback-session-plan.json ` + --approve-plan-hash REVIEWED_PLAN_HASH ``` -Review the block and repository selector before committing it. Do not replace -unrelated instructions or assume that adding the text proves the MCP process is -healthy. The current `mcp-manifest --profile normal|diagnostic` command validates -static profile-specific contract output; it is not the planned live startup and -freshness probe. +Create separate reviewed installations for the client checkout and the CodeMesh +maintainer checkout. Both pin the exact manifest path and file hash: + +```powershell +uv run python -m codemesh_agent_access install plan ` + --target-root C:\path\to\client ` + --codemesh-root C:\path\to\CodeMesh ` + --project-id prj_CLIENT --checkout-id chk_CLIENT ` + --profile development-feedback ` + --feedback-session C:\temp\codemesh-feedback-session.json ` + --feedback-session-sha256 REVIEWED_MANIFEST_HASH ` + --output C:\temp\client-feedback-install.json + +uv run python -m codemesh_agent_access install plan ` + --target-root C:\path\to\CodeMesh ` + --codemesh-root C:\path\to\CodeMesh ` + --project-id prj_CODEMESH --checkout-id chk_CODEMESH ` + --profile feedback-maintainer ` + --feedback-session C:\temp\codemesh-feedback-session.json ` + --feedback-session-sha256 REVIEWED_MANIFEST_HASH ` + --output C:\temp\maintainer-feedback-install.json +``` + +Review and apply each installation with `install apply`, then run +`mcp-probe --plan PLAN` for each one. The client probe checks session discovery +without writing feedback. The maintainer probe lists only allowlisted outboxes. + +During the session, the client agent calls +`codemesh_get_feedback_session`, source-verifies consequential CodeMesh output, +and uses `codemesh_record_feedback` only for a bounded sanitized observation. +The maintainer agent starts with `codemesh_list_feedback`, retrieves an exact id, +reproduces it read-only, and calls `codemesh_prepare_feedback_resolution`. A +human must approve the exact feedback ids and returned plan hash before the +maintainer agent edits CodeMesh. An agent-supplied confirmation is not human +approval. + +After reconfiguration and recheck, the client records a passing packet with +`supersedes_feedback_ids` naming the exact failure. Resolution is derived only +for a passing recheck from the same checkout with the same issue category and +task family. Close the session without deleting evidence: + +```powershell +uv run python -m codemesh_agent_access feedback session revoke ` + --manifest C:\temp\codemesh-feedback-session.json ` + --manifest-sha256 REVIEWED_MANIFEST_HASH ` + --reason "Development session complete" +``` + +Expiry or revocation blocks subsequent calls. No feedback endpoint edits +CodeMesh, starts ingestion, invokes a provider, creates an external issue, or +transmits a packet. + +### Direct v1 recorder + +The existing operator/evaluator CLI remains available. Participating +repositories must ignore `/.codemesh-feedback/` before recording. It writes a +bounded v1 packet with caller-supplied accepted provenance: + +```powershell +uv run python -m codemesh_agent_access feedback record ` + --repository-root C:\path\to\checkout ` + --role primary ` + --classification diagnostic ` + --confidence high ` + --project-id prj_EXACT ` + --checkout-id chk_EXACT ` + --snapshot-id snp_EXACT ` + --language python --language rust ` + --parser-profile python-rust-v1 ` + --task-family test-selection ` + --issue-category ranking ` + --validation-outcome failed ` + --missed-path tests/test_expected.py +``` + +Use `feedback validate PACKET` before review. CodeMesh maintainers run +`feedback summarize CHECKOUT... --output SUMMARY.json` to group open failures +without losing commit/snapshot provenance. A passing recheck may use +`--supersedes-feedback-id` to resolve an exact earlier failure. Never store raw +prompts, MCP payloads, source excerpts, secrets, or environment values in a +packet. ## Required Environment @@ -247,6 +407,13 @@ The `diagnostic` profile additionally exposes: - `codemesh_get_summary_coverage`: inspect generated-summary coverage. - `codemesh_list_ingestion_runs`: inspect recent indexing runs. +The session-only profiles add: + +- `development-feedback`: `codemesh_get_feedback_session` and + `codemesh_record_feedback` alongside the normal tools. +- `feedback-maintainer`: `codemesh_list_feedback`, `codemesh_get_feedback`, and + `codemesh_prepare_feedback_resolution` alongside the normal tools. + ## Troubleshooting - If MCP starts but returns no useful context, confirm the repository was ingested with `repos list`. diff --git a/docs/guides/self-analysis.md b/docs/guides/self-analysis.md index 2e1d0b1..94cb4c3 100644 --- a/docs/guides/self-analysis.md +++ b/docs/guides/self-analysis.md @@ -71,9 +71,15 @@ uv run python -m codemesh_agent_access eval live For an opt-in paired Codex comparison using isolated temporary clones: ```powershell -uv run python -m codemesh_agent_access eval agent --model +uv run python -m codemesh_agent_access eval agent ` + --model ` + --max-reported-tokens ``` +The agent command fails closed before any model call unless the selected runner +can guarantee the requested reported-token cap. The checked-in Codex runner +does not currently claim that capability. + To qualify a model against the full graded CodeMesh question set after the live suite passes: diff --git a/docs/planning/agent-integration-contract.md b/docs/planning/agent-integration-contract.md index 4e05313..bdb6ef1 100644 --- a/docs/planning/agent-integration-contract.md +++ b/docs/planning/agent-integration-contract.md @@ -2,14 +2,83 @@ Document type: selected product contract and implementation plan -Reviewed: 2026-08-12 +Reviewed: 2026-09-01 This document selects the v1 integration contract for CodeMesh in coding-agent -hosts. It defines the product behavior that future implementation and evaluation -must target. The MCP server instructions, manual repository guidance in -[MCP Setup](../guides/mcp-setup.md), and normal/diagnostic tool profiles are -implemented. The probe, checkout binding, fail-closed package freshness, and -installation behavior below are not implemented yet. +hosts. The MCP server instructions, manual repository guidance in +[MCP Setup](../guides/mcp-setup.md), normal/diagnostic tool profiles, exact +checkout binding, fail-closed package freshness, reviewable installation, and +provider-free probes are implemented and passed their clean-candidate gates at +`ee1d4d9`. Configured/onboarded evaluator support is implemented and its live +stale-candidate rejection is verified through `9f3216c`. Exact-candidate +preflight, the frozen one|nine retrieval and rejection gates, and the pinned +YoutubeDownloader live gate passed again with the status-to-context correction +at exact clean candidate `0378740`. The first configured normal-host campaign +remains frozen at `0bde606`; it completed safely and correctly but was +insufficient because two treatments stopped after status and none retrieved +task context. The separately authorized `0378740` campaign was also safe and +correct but insufficient: one treatment stopped after status and none retrieved +task context. The status-result next-action field therefore did not establish +the required normal-host transition. The current implementation instead makes +the fail-closed context package the single normal-host entry action and retains +status for explicit diagnostics. Exact clean candidate `8844c21` passed the +full deterministic and provider-free gates. Its authorized paired campaign +then retrieved a context package in all three treatments and passed all six +executions safely. The verdict was `neutral`, with no attributable wins or +regressions, so adoption is established for the selected task but product +benefit is not. + +The separately authorized Config.Net campaign at exact candidate `b16eee7` +also retrieved a context package in every treatment, but two of three +treatments missed required citations while all controls passed. Its `regressed` +verdict establishes configured adoption on a second repository, not product +benefit. Preserve the consumed campaign without an automatic retry. + +The bounded follow-up correction is frozen at exact clean candidate `3804028`. +It keeps the context package as the single entry tool and directs multi-part +work to focused implementation and test queries. The reviewed configured +preflight, positive and fail-closed probes, focused-query live suite, unchanged +canonical live suite, and deterministic Python checks all pass with provider +mode `none`. This establishes correction readiness only; normal-agent guidance +compliance and product benefit still require a separately authorized campaign. + +That separately authorized campaign completed at the same exact candidate with +the frozen Config.Net comparison conditions. All three treatments adopted the +context package and passed safely; controls passed two of three, producing one +CodeMesh-attributable win, zero treatment regressions, and an `improved` +verdict. Efficiency medians were also favorable across the two jointly +successful pairs. This is positive evidence for the bounded campaign, not proof +that focused wording caused the change or that benefit repeats elsewhere. + +The independent configured YoutubeDownloader impact campaign at clean +candidate `64d4052` then preserved a different frozen C# task and repository. +Every control and treatment passed safely, and every treatment adopted the +context package, but treatment medians were 49.79% slower and used 41.00% more +tokens across all three pairs. Its `regressed` verdict is an efficiency result, +not a correctness regression. Configured adoption and correctness now repeat +across repositories, but product benefit does not. Raw query arguments were not +retained, so that campaign does not establish its exact query behavior or the +cause of the regression. + +The provider-free task-adaptive diagnosis and correction are now frozen at +exact clean candidate `fbc1433`. One task-shaped package covered all six frozen +YoutubeDownloader targets, while three proactive focused packages repeated five +ranked items and used three times the formatted package budget. The current +guidance therefore starts with one package, stops when it resolves every facet, +and decomposes only a specific unresolved implementation or test facet, under a +three-package ceiling. Deterministic, configured preflight, positive, rejection, +adaptive, unchanged impact, and unchanged canonical gates all pass without a +model provider. This establishes correction readiness, not agent compliance or +product benefit. + +The separately authorized configured campaign at exact candidate `fbc1433` +then completed all six runs correctly and safely with one context package in +every treatment. Median treatment tokens fell 6.63%, but median duration rose +10.0192%, just beyond the frozen regression threshold. Its `regressed` verdict +is an efficiency result, not a correctness regression. The campaign also +exceeded its authorized 1.6M aggregate reported-token ceiling because the +historical evaluator lacked a hard stop. Preserve the consumed result and +control failure; do not retry it. ## Decision @@ -52,7 +121,10 @@ hold: matching repository snapshot. 5. Agents begin relevant tasks with a context package, verify consequential findings against current source, and fall back to normal repository - exploration when CodeMesh is unavailable, stale, or ambiguous. + exploration when CodeMesh is unavailable, stale, or ambiguous. Agents start + with one concise task-shaped package and stop retrieving when it covers every + required facet. A specific unresolved implementation or test facet permits + one focused follow-up, with at most three non-overlapping packages per task. The supported claim should remain bounded: @@ -67,10 +139,11 @@ product benefit. ## Repository-Owned Onboarding -The canonical manual block lives in [MCP Setup](../guides/mcp-setup.md). A future -installer may add or update only the marked block after showing a reviewable -diff and receiving explicit owner approval. It must not silently create, -replace, or rewrite unrelated repository instructions. +The canonical manual block lives in [MCP Setup](../guides/mcp-setup.md). The +installer generates a complete hashed plan, and applies only that unchanged +reviewed plan after its hash is supplied explicitly. It may manage the marked +guidance block or run in configuration-only mode when another repository owns +generated guidance. It does not silently replace unrelated instructions. Repository guidance is the portable contract. Host-specific hooks may display freshness warnings or reminders, but correctness must not depend on a hook that @@ -98,22 +171,21 @@ only through the explicit administrative REST and Python CLI surfaces. ## Checkout And Freshness Behavior -The integration must bind a host configuration to a stable project and checkout -identity or a unique repository alias. It must fail clearly when selection is -missing or ambiguous rather than using whichever indexed repository looks most -recent. +The normal MCP integration binds host configuration to an exact project, +checkout, repository root, and optional source-view hash. It fails clearly when +selection is missing, wrong, ambiguous, dirty, or stale rather than using +whichever indexed repository looks most recent. -Freshness is a server-side acceptance condition. Before returning task context -as trustworthy, CodeMesh must compare the configured checkout and source state -with the active indexed snapshot and expose the result in the response. A hook -or client warning may improve visibility, but it cannot convert stale data into -accepted evidence. +Freshness is a server-side acceptance condition. Before returning task context, +CodeMesh compares the configured checkout and source state with the active +indexed snapshot and exposes the assessment in the response. Client warnings +cannot convert stale data into accepted evidence. ## Live Runtime Probe Static configuration validation and manifest printing are insufficient because they do not prove that the configured process can initialize in the target -environment. The planned probe must: +environment. The implemented positive probe: 1. launch the exact stdio command and working directory used by the host; 2. complete MCP initialization and inspect the server instructions; @@ -122,8 +194,10 @@ environment. The planned probe must: 5. resolve the configured repository without ambiguity; and 6. confirm current checkout and snapshot freshness. -It must exit nonzero with a specific diagnostic when any required condition -fails. The probe must be provider-free and must not invoke a paid model. +It exits nonzero with a specific diagnostic when any required condition fails. +The companion rejection probe launches the same normal profile with an unsafe +binding and proves that it fails closed. Both probes force provider mode to +`none` and do not invoke a paid model. ## Evaluation Classification @@ -141,23 +215,90 @@ Configured/onboarded evidence supports the selected v1 contract. It must not be relabeled as spontaneous discovery, and older evaluator-assisted runs must not be retroactively relabeled as configured product evidence. -Before another model-cost-bearing campaign, the evaluation harness must record -the integration mode and the identities of the installed guidance and selected -tool profile. The frozen task, grading targets, model settings, clean-checkout -requirements, and summary-free live gate remain unchanged. +The implemented configured mode records integration mode and exact identities +for installed guidance, baseline MCP servers, selected tool profile, binding, +reviewed plan, and positive and rejection probes. It gives control the declared +baseline servers and treatment the identical baseline plus CodeMesh, without +changing the task prompt. The frozen task, grading targets, model settings, +clean-checkout requirements, and summary-free live gate remain unchanged. + +The provider-free preflight must pass against the exact candidate before a +campaign. Across the first two authorized one|nine configured campaigns, three +treatments called repository status but none retrieved a context package or +produced an attributable win. The `8844c21` campaign then retrieved a context +package in every treatment, but its neutral result still produced no +attributable win. The `b16eee7` Config.Net campaign also achieved full adoption +but regressed correctness in two treatments. Any further paired comparison +remains model-cost-bearing and requires new separate authorization. The +separately consumed `3804028` Config.Net campaign later produced one +attributable win and zero regressions, but its single-task result does not make +benefit repeatable. The independent `64d4052` YoutubeDownloader campaign then +preserved full adoption and correctness on a different C# repository, but its +49.79% median duration regression and 41.00% median token regression produced a +`regressed` verdict. The separate `fbc1433` task-adaptive campaign reduced +treatments to one package and lowered median tokens by 6.63%, but its 10.0192% +median duration regression also produced a `regressed` verdict. It exceeded its +authorized aggregate reported-token ceiling because the evaluator lacked a +hard stop. No automatic paid retry is authorized. ## Completion Gates -Complete the integration in this order: +Complete the product-evidence sequence in this order: 1. retain the canonical manual onboarding block and its opt-in boundary; 2. maintain the implemented normal-user and diagnostic tool profiles; -3. implement unambiguous checkout binding and fail-closed freshness behavior; -4. implement the provider-free live runtime probe; -5. pass deterministic contracts and a clean summary-free live gate; and -6. with explicit model-spend authorization, run a configured/onboarded campaign - using the shipped integration unchanged. - -Do not claim the v1 integration complete from documentation, static manifests, -fixtures, configured CI, or an MCP process that starts without passing these -gates. +3. maintain the implemented exact checkout binding and fail-closed freshness; +4. maintain the implemented provider-free positive and rejection probes; +5. maintain the clean-candidate deterministic contracts, summary-free live + gate, and frozen one|nine pilot gate verified for the one-step correction at + `8844c21`; +6. maintain the passing configured/onboarded paired-evaluation preflight so the + implemented runner continues to prove the reviewed normal-profile + installation and its exact integration identities against the candidate; +7. maintain the implemented one-step fail-closed context-package entry action + across MCP instructions, installed onboarding, and tool descriptions, as + provider-free verified at exact candidate `8844c21`; and +8. retain the consumed `8844c21` configured/onboarded campaign that established + context-package adoption and a neutral outcome for the selected one|nine + task; and +9. retain the consumed configured Config.Net campaign at exact candidate + `b16eee7`, which established full context-package adoption but regressed in + two treatments; and +10. retain the provider-free full-task coverage diagnosis against every frozen + Config.Net implementation and test target; and +11. retain the clean-candidate verification of the focused-query guidance + correction at `3804028`; and +12. retain the separately consumed `3804028` Config.Net campaign, which passed + all treatments with one attributable win, zero regressions, and an + `improved` single-campaign verdict; and +13. retain the independently consumed `64d4052` YoutubeDownloader campaign, + which preserved full adoption and correctness but regressed median duration + and token usage on a different frozen C# task; and +14. retain the `fbc1433` provider-free task-adaptive diagnosis and clean- + candidate correction evidence; and +15. retain the consumed `fbc1433` configured campaign, including its + single-package adoption, duration-regressed verdict, and aggregate-token + ceiling breach; retain the implemented fail-closed reported-token accounting, + compatible-runner capability gate, boundary stop, and no-retry behavior + before any future paid evaluation is considered. + +The bounded one|nine path and configured preflight have passed their clean- +candidate provider-free gates. The two status-first campaigns remain valid but +`insufficient`; the one-step campaign is valid and `neutral` with full adoption, +the `b16eee7` Config.Net campaign is valid and `regressed` with full adoption, +and the separate `3804028` Config.Net campaign is valid and `improved` with one +attributable win. The independent `64d4052` YoutubeDownloader campaign is valid +and `regressed` on efficiency despite full adoption and correctness. The +Config.Net improvement remains positive bounded evidence, but the independent +efficiency regressions mean no result yet establishes repeatable product +benefit. The `fbc1433` campaign establishes bounded single-package compliance, +not overall efficiency benefit. Its historical budget-control failure is now +closed in the evaluator. The default `codex` runner cannot guarantee the cap and +fails before model calls; the explicit `capped-codex` runner implements the +contract. Its separately authorized `0f18071` campaign stopped on unavailable +provider credits without a completed pair and is consumed. That failure proves +bounded fail-closed behavior, not product benefit. +Do not claim benefit from documentation, static +manifests, fixtures, +configured CI, adoption alone, or an MCP process that starts without an +attributable agent-outcome result. diff --git a/docs/planning/development-feedback-mcp.md b/docs/planning/development-feedback-mcp.md new file mode 100644 index 0000000..18ed0f4 --- /dev/null +++ b/docs/planning/development-feedback-mcp.md @@ -0,0 +1,569 @@ +# Development-Session MCP Feedback Loop + +Document type: completed owner-activated implementation plan + +Reviewed: 2026-09-10 + +Status: implemented and provider-free verified in the current local checkout; +not attended, published, released, deployed, or product-benefit validated. + +## Decision + +CodeMesh extends its existing local `.codemesh-feedback/` packet workflow +into an explicit development-session MCP loop. Client agents will be able to +record bounded feedback through MCP, and a separate agent working in the +CodeMesh checkout will be able to review that feedback and prepare a proposed +resolution. A human must approve the proposed CodeMesh change before the +maintainer agent edits source. + +This completed priority is tracked in [Next Steps](next-steps.md). It extends +the local recorder, validator, and summarizer. The existing `normal` and +`diagnostic` MCP profiles remain unchanged; the new profiles are separate and +fail closed without an exact active session. + +The first version is local and same-machine. It reads participating checkout +outboxes directly and does not introduce remote feedback transport, shared +multi-user storage, automatic issue creation, or autonomous CodeMesh changes. + +## Goals + +1. Make the feedback workflow discoverable and directly usable by client + agents during an explicitly enabled development session. +2. Preserve trustworthy repository, checkout, snapshot, parser, CodeMesh, and + validation provenance without asking the client agent to supply identities + that the server can determine. +3. Let a CodeMesh maintainer agent in VS Code list, inspect, reproduce, and + prepare a resolution for feedback from session-authorized client checkouts. +4. Require explicit human approval before feedback leads to a CodeMesh source + or documentation change. +5. Close the loop with an append-only passing recheck that supersedes the exact + earlier failure. + +## Non-Goals And Evidence Boundary + +- Feedback does not automatically edit CodeMesh, change ranking, trigger + ingestion, alter priorities, create external issues, or transmit repository + material. +- Session activation authorizes only bounded local feedback artifacts. It does + not authorize a CodeMesh edit, commit, push, pull request, release, + deployment, provider spend, or external mutation. +- Feedback remains diagnostic product input. A packet, fix, or passing recheck + is not controlled agent-benefit evidence unless a separately authorized + evaluation establishes that claim. +- This feature is not the proposed agent outcome-memory system. It does not + train retrieval, prefer nodes, or retain conversations. +- MCP does not enforce write approval for an IDE agent that already has direct + filesystem access. The host approval policy, repository instructions, Git + controls, and explicit human decision remain authoritative for source edits. + +## Roles And Trust Boundaries + +| Role | Allowed behavior | Prohibited implication | +| --- | --- | --- | +| Human owner or maintainer | Review and activate a bounded session; select feedback for investigation; approve an exact proposed change. | Session activation does not pre-approve every resulting CodeMesh change. | +| Client agent | Use CodeMesh and submit sanitized observations through the development-feedback MCP profile. | Client feedback is not trusted ground truth or implementation authority. | +| Client CodeMesh MCP server | Derive provenance, validate and write an append-only packet inside the bound checkout. | It cannot write outside the feedback outbox or modify CodeMesh source. | +| CodeMesh maintainer MCP server | Read and summarize feedback only from session-allowlisted checkout outboxes; prepare a resolution proposal. | It cannot edit source or assert human approval. | +| CodeMesh maintainer agent in VS Code | Reproduce a report, inspect source and tests, propose a bounded change, and implement only after approval. | A feedback packet alone does not authorize an edit. | + +## Explicit Development Session + +Feedback tools must not be enabled by a loose environment boolean. The +implementation must use a reviewable, hashed session manifest with schema +`codemesh-development-feedback-session/v1` containing: + +- a unique session id, creation time, and expiry; +- the exact CodeMesh checkout root; +- each participating client root, project id, checkout id, and reporter role; +- the allowed client and maintainer MCP profiles and tools; +- local-only transport and provider-free declarations; +- packet, text, path, and request-rate bounds; and +- the SHA-256 of the canonical manifest payload, excluding the hash field + itself. + +The installer must include the manifest path and expected hash in its reviewed +launch plan. MCP startup and every feedback call must fail closed when the +manifest is missing, altered, expired, revoked, or does not authorize the exact +bound checkout. Closing a session must prevent new packets without deleting +prior evidence. Because the reviewed manifest is immutable, early closure must +create a separately validated revocation record rather than rewriting it. + +The session lifecycle is exposed under +`codemesh-agent-access feedback session` with `plan`, `activate`, `inspect`, +and `revoke` actions. `plan` produces a reviewable +`codemesh-development-feedback-session-plan/v1` artifact containing the exact +proposed manifest bytes and hashes. `activate` requires the expected plan hash +and creates the immutable session manifest exclusively. `revoke` requires the +expected session hash and creates a separate revocation record exclusively. +Activation must retain the existing reviewed-plan and changed-target +protections. The session preflight must also require the target checkout's +`/.codemesh-feedback/` path to be ignored and local rather than silently +editing tracked ignore rules. + +## MCP Profiles + +The implemented `normal` profile must remain the default read-only product +surface. + +The new profiles are: + +- `development-feedback`: the normal client tools plus bounded feedback + discovery and recording. It exists only for a valid explicit session. +- `feedback-maintainer`: read-only feedback intake and resolution planning for + the CodeMesh checkout. It scans only roots named by the session manifest. + +The installer and runtime probe must validate the exact selected profile and +tool allowlist. A normal installation must not gain write tools merely because +the CodeMesh package supports them. + +## Client MCP Tools + +### `codemesh_get_feedback_session` + +Return the session id, expiry, enabled state, local-only status, allowed +categories, packet limits, and concise instructions describing when feedback +is appropriate. Do not return credentials, environment values, or roots that +the bound client does not need. + +### `codemesh_record_feedback` + +Accept only the agent's bounded observation: + +- task family, issue category, confidence, and validation outcome; +- helpful, incorrect, missed, stale, or ambiguous repository-relative paths; +- an optional bounded fallback reason, minimal reproduction, expected target + set, and proposed correction; and +- optional exact feedback ids superseded by a passing recheck. + +The server, not the client agent, must populate: + +- session id and recording time; +- repository root, branch, commit, and dirty state; +- bound project and checkout ids; +- selected snapshot, language set, parser profile, and binding assessment; and +- CodeMesh commit, dirty state, and working-tree-state hash. + +The tool must allow setup, binding, and freshness failures to be reported even +when context retrieval was rejected. Claims about returned ranking or content +must retain an accepted snapshot and relevant tool-use provenance. The +development-session classification is `diagnostic`; only the controlled +evaluator may create `controlled-evaluation` records. + +Adding the session and MCP-origin fields requires a strict +`codemesh-feedback/v2` packet while preserving validation and summarization of +existing v1 packets. Recording remains append-only and content-addressed. + +A successful response returns only the feedback id, repository-relative packet +path, local-only state, and `human_review_required = true`. It must not imply +that CodeMesh accepted the report as correct or scheduled a change. + +## Maintainer MCP Tools + +### `codemesh_list_feedback` + +Return bounded summaries of validated feedback from session-allowlisted roots, +with filters for open/resolved state, category, task family, language, client +role, and feedback id. Preserve invalid-packet diagnostics without returning +unsafe raw file content. + +### `codemesh_get_feedback` + +Return one validated packet and its derived supersession state by exact +feedback id. Reject ambiguous ids and packets whose claimed root does not match +their allowlisted outbox. + +### `codemesh_prepare_feedback_resolution` + +Produce a read-only proposed resolution containing the selected feedback ids, +exact provenance, reproducibility state, smallest proposed change, risks, +required source and test investigation, validation plan, and a stable plan +hash. It must not modify CodeMesh, write an approval decision, or describe the +plan as human-approved. + +An optional MCP resource such as +`codemesh://development-feedback/open` may support host notifications or +subscriptions. Resource-notification support varies between hosts, so listing +feedback through a tool remains the required portable path. + +## Agent Discovery And Onboarding + +During an active session, the client profile must advertise feedback through: + +1. MCP initialization instructions; +2. feedback tool names, descriptions, and schemas; +3. the generated repository-owned `AGENTS.md` onboarding block; and +4. a concise feedback recommendation when a context package is rejected, + incomplete, truncated, or contradicted by later source verification. + +The instruction must name `codemesh_record_feedback`, state that packets remain +local, and prohibit raw prompts, MCP payloads, source excerpts, secrets, +credentials, and environment values. CodeMesh must not solicit a rating after +every successful call. Passing feedback is primarily a recheck that supersedes +an exact earlier failure. + +The CodeMesh repository instructions must tell the maintainer agent to start a +development-feedback task with `codemesh_list_feedback`, investigate read-only, +and wait for explicit human approval tied to the feedback id and proposed plan +before editing. + +## Human-In-The-Loop Improvement Flow + +1. A human reviews and activates a bounded session for exact client checkouts. +2. A client agent uses CodeMesh and source-verifies consequential results. +3. When verification identifies a missing, stale, incorrect, or unhelpful + result, the agent records a sanitized packet through MCP. +4. A human asks the CodeMesh maintainer agent to review new feedback, or the + host presents a non-authorizing notification. +5. The maintainer agent lists and retrieves the exact packet, checks its + provenance, reproduces it when possible, and prepares a bounded resolution. +6. The human approves, rejects, or defers that exact proposal. An agent-supplied + `confirm = true` is not evidence of human approval. +7. After approval, the maintainer agent changes only the approved CodeMesh + scope, updates tests and documentation, runs the required checks, and reports + the resulting commit and evidence state. +8. The updated CodeMesh candidate is deliberately reconfigured and probed in + the client checkout. No feedback tool silently restarts MCP or rewrites the + client's binding. +9. A client agent performs the same validation and records a passing recheck + with `supersedes_feedback_ids` naming the original failure. +10. The summary derives the original report as resolved; neither packet is + mutated or deleted. + +## Safety Invariants + +- Feedback writes stay inside the exact bound checkout's ignored + `.codemesh-feedback/` directory. +- Linked, redirected, traversing, or non-local outboxes are rejected. Packet + creation is exclusive and never overwrites an existing regular file, + symlink, junction, or hardlink target. +- Text and path limits, secret-pattern rejection, repository-relative path + validation, content ids, and strict schemas remain mandatory. +- Client-supplied repository, binding, snapshot, CodeMesh, session, and + classification identities are rejected rather than trusted. +- Maintainer reads are restricted to exact session-allowlisted roots; the + implementation must not search arbitrary parent directories or the whole + filesystem for outboxes. +- No feedback endpoint invokes a model provider, ingestion, refresh, watch, + repository deletion, garbage collection, external issue tracker, network + transport, or source edit. +- Raw prompts, MCP request or response payloads, source excerpts, credentials, + secret values, and environment values are not retained. +- Session closure and expiry stop new writes but preserve already recorded + evidence and supersession relationships. + +## Selected Implementation Shape + +### Runtime composition + +`build_server()` will receive a validated session runtime explicitly. The +implementation must not use process-global activation state or infer a session +from an environment boolean. The runtime owns the immutable manifest, its +expected hash, revocation and expiry checks, the exact active binding, and a +small injected clock so expiry behavior is deterministic in tests. +Request-rate bounds are enforced per stdio server process with an injected +monotonic clock. They are misuse and runaway-output controls, not a claim of a +cross-process security quota. + +The profile registry must be made explicit before adding tools. In particular, +new feedback tools must not enter the existing `diagnostic` profile merely +because that profile currently includes most registered tools. The profiles +are composed as follows: + +| Profile | Binding | Tool surface | +| --- | --- | --- | +| `normal` | One client checkout | The existing four read tools, unchanged. | +| `diagnostic` | Optional | The existing diagnostic tools, unchanged. | +| `development-feedback` | One allowlisted client checkout | The four normal tools plus session discovery and packet recording. | +| `feedback-maintainer` | The allowlisted CodeMesh checkout | The four normal tools plus feedback list, get, and resolution-plan tools. | + +The maintainer composition lets the VS Code agent inspect CodeMesh through the +normal source-grounded surface while reviewing feedback. Neither new profile +contains repository deletion or other operator controls. + +### Session and installation contracts + +Create `codemesh_agent_access/feedback_session.py` for strict session-plan, +manifest, participant, bounds, and revocation models. It owns canonical JSON +hashing, exclusive creation, allowlist resolution, expiry and revocation +validation, and ignored-local-outbox preflight. Paths are resolved before +comparison; containment is never established by string-prefix matching. +The manifest is an envelope containing a payload and the SHA-256 of the +canonical payload. The activation plan separately pins the exact serialized +manifest-file hash, avoiding a self-referential whole-file hash. + +The installer contract advances to `codemesh-installation-plan-v2`. Its loader +must continue to validate and apply existing v1 normal plans. V2 adds an +optional session block containing the manifest path and expected SHA-256. The +block is required for either feedback profile and rejected for `normal` and +`diagnostic`. Generated launch arguments pin both values: + +```text +--feedback-session +--feedback-session-sha256 +``` + +This keeps the reviewed session identity visible in the same installation plan +that pins the CodeMesh root, profile, binding, launch command, and expected tool +surface. No command silently updates an already installed target when its +reviewed files have drifted. + +### Feedback packet compatibility + +Keep the existing `FeedbackPacket` v1 model and CLI behavior unchanged. Add a +strict v2 model and a version-discriminated loader used by validation, +summarization, and maintainer intake. V2 adds: + +- the server-derived session id, MCP profile, binding assessment, and recording + time; +- server-derived repository, checkout, snapshot, parser, and CodeMesh + provenance; +- a bounded recent-tool trace containing tool name, success or bounded failure + code, and sequence only; and +- an explicit provenance-availability state for reports made before context + could be accepted. + +The MCP server keeps the recent-tool trace in memory for that one stdio server +process. It never records arguments, responses, prompts, source excerpts, or +environment values. The trace supports investigation but is not proof that a +particular result caused the observation. + +Setup, binding, and freshness categories may use a bounded unavailable +provenance state. Ranking, missing-language, missing-relationship, budget, and +validation claims require an accepted snapshot plus the applicable parser and +tool provenance. The development endpoint fixes classification to +`diagnostic`; it rejects any client-supplied identity or classification field. + +### Maintainer resolution plan + +`codemesh_prepare_feedback_resolution` is a deterministic validator and hasher, +not an autonomous fix generator. The maintainer agent supplies a bounded draft +after source inspection: exact feedback ids, reproduction state, proposed +CodeMesh-relative paths, change summary, risks, and checks. The endpoint +validates those fields against the session and packets, orders them +canonically, and returns an in-memory plan plus stable hash. It does not write +the plan, record approval, or edit a file. Human approval must name the exact +feedback ids and returned plan hash before the IDE agent begins implementation. + +### MCP metadata + +Use MCP tool annotations as defense-in-depth metadata: + +- session discovery and all maintainer tools are read-only; +- feedback recording is write-capable, non-destructive, non-idempotent, and + local-only; and +- no feedback tool has an open-world or network side effect. + +The server remains responsible for enforcing these properties; annotations are +not an authorization boundary. + +## Source Change Map + +| Path | Planned responsibility | +| --- | --- | +| `agent-access/codemesh_agent_access/feedback_session.py` | New session plan, manifest, revocation, validation, bounds, and runtime authorization. | +| `agent-access/codemesh_agent_access/feedback.py` | Preserve v1; add strict v2 creation, validation, summaries, allowlisted intake, and deterministic resolution-plan hashing. | +| `agent-access/codemesh_agent_access/mcp.py` | Explicit profile composition, session injection, five new tools, annotations, instructions, and bounded recent-tool trace. | +| `agent-access/codemesh_agent_access/installer.py` | Backward-compatible installation-plan v2 session pinning and feedback onboarding. | +| `agent-access/codemesh_agent_access/probe.py` | Separate feedback-profile probes and altered, expired, revoked, and wrong-root rejection checks. | +| `agent-access/codemesh_agent_access/cli.py` | Session lifecycle commands and feedback-session options for MCP, manifest, install, and probe commands. | +| `agent-access/tests/test_feedback_session.py` | Session schemas, hashing, exclusive activation/revocation, bounds, expiry, path, and outbox safety. | +| `agent-access/tests/test_feedback.py` | V1 compatibility, v2 provenance rules, summarization, supersession, allowlisted intake, and resolution plans. | +| `agent-access/tests/test_mcp_contract.py` | Exact profile surfaces, annotations, instructions, runtime authorization, and provenance rejection. | +| `agent-access/tests/test_installer_probe.py` | V1 compatibility, v2 plans, onboarding, launch pinning, probes, and rejection cases. | +| `agent-access/tests/test_cli_output.py` | Stable JSON and exit behavior for the session lifecycle and new profile arguments. | +| `agent-access/tests/test_development_feedback_integration.py` | Disposable-checkout provider-free failure, intake, planning, revocation, and passing-recheck flow. | +| Current documentation named below | Publish only behavior that its work package has implemented and verified. | + +## Implementation Work Packages + +Each package is independently reviewable and must include its tests and the +documentation for behavior introduced by that package. Later packages depend +on the accepted contracts of earlier packages. + +### WP1: Session authority and immutable lifecycle + +Dependencies: none. + +Implement the new session module and the `feedback session` lifecycle CLI. A +plan consumes an explicit participant configuration, resolves +and validates every checkout, proves each outbox is ignored and local, applies +hard upper bounds, and emits proposed immutable manifest bytes plus hashes. +Activation and revocation use exclusive creation and exact expected hashes. +Inspection returns only sanitized state and reasons such as `active`, +`expired`, `revoked`, `hash_mismatch`, or `wrong_checkout`. + +Required tests cover schema strictness, canonical hashes, malformed or duplicate +participants, missing and non-Git roots, symlink and containment attacks, +tracked outboxes, existing destinations, expiry boundaries, revocation races, +and wrong CodeMesh or client bindings. No MCP profile is exposed in this +package. + +Exit: a human can create, review, activate, inspect, and revoke a bounded local +session artifact, and all failure modes close without modifying a checkout. + +### WP2: Client feedback profile and v2 packets + +Dependencies: WP1. + +Add the explicit `development-feedback` profile, session discovery, packet +recording, bounded in-process tool trace, and v2 packet support. Refactor the +profile registry so exact `normal` and `diagnostic` tool lists are asserted +independently. Resolve repository and CodeMesh provenance inside the server. +Use bounded diagnostic codes when status is unavailable; never persist raw +exceptions. Preserve the existing direct `feedback record` v1 path for +operator and controlled-evaluation use. + +Required tests assert the exact profile tool sets and annotations, absence of +feedback tools from old profiles, missing or invalid session rejection, +client-field overposting rejection, category-specific provenance rules, +sanitization and exclusive creation, v1 validation, mixed v1/v2 summaries, and +append-only passing rechecks. + +Exit: an allowlisted client agent can discover and write one valid local v2 +packet through MCP, while an ordinary normal client sees precisely the prior +read-only contract. + +### WP3: Maintainer intake and approval-ready plans + +Dependencies: WP1 and WP2 packet contracts. + +Add the `feedback-maintainer` profile and deterministic list, get, and prepare +tools. Intake enumerates only the exact outbox of each client participant in +the manifest, validates every packet before projecting safe output, and reports +bounded invalid-packet diagnostics. Exact-id lookup rejects zero matches, +duplicates, claimed-root mismatch, and packets outside the active session. +By default, list and get expose only v2 packets matching the active session. +An explicit `include_legacy = true` may surface valid v1 packets from an +allowlisted outbox as `legacy_unscoped`; that label must survive into any +resolution plan and prevents treating the packet as session-authenticated. + +Resolution preparation accepts only validated session feedback and +CodeMesh-relative proposal paths. It returns reproduction status, unresolved +questions, risks, required tests, evidence limits, and the stable plan hash. +The response always states `human_review_required = true` and +`human_approved = false`. + +Required tests cover every filter, stable ordering and pagination, malformed +and adversarial files, id ambiguity, root escape attempts, mixed packet +versions, supersession state, stable plan hashes, invalid proposal paths, and +proof that no source or feedback file changes during list, get, or prepare. + +Exit: the CodeMesh VS Code agent can turn allowlisted feedback into an exact, +reviewable, non-authorizing proposal without filesystem mutation. + +### WP4: Installer, onboarding, and runtime probes + +Dependencies: WP2 and WP3 tool manifests. + +Add installation-plan v2, pin the session path and hash into feedback-profile +launches, and render profile-specific MCP instructions and `AGENTS.md` +onboarding. Normal onboarding stays byte-for-byte stable unless a deliberate +separate documentation correction is required. Add separate feedback-profile +runtime probes rather than weakening the existing normal probe. + +The client probe verifies the exact profile, tool surface, binding, session +hash, expiry, local-only declaration, instructions, and discovery response +without writing a real packet. The maintainer probe verifies the CodeMesh +binding and participant allowlist without reading outside it. Rejection probes +exercise altered, expired, revoked, wrong-root, wrong-profile, and unexpected +tool-surface cases. Configured-agent evaluation continues to require the exact +`normal` profile. + +Required tests cover v1 installation-plan loading, v2 plan/apply drift +protection, missing and extraneous session blocks, generated launch arguments, +onboarding content, exact manifests, both positive probes, and every rejection +case. + +Exit: a reviewed installation can activate either feedback role reproducibly, +while old normal installations and configured-agent evaluations retain their +existing contract. + +### WP5: Provider-free end-to-end rehearsal and documentation promotion + +Dependencies: WP1 through WP4. + +Create disposable Git checkouts for CodeMesh and one client repository. Run +the exact installed stdio commands with provider mode disabled. Exercise a +client failure packet, maintainer list/get/prepare, explicit confirmation that +no unapproved CodeMesh edit occurred, session revocation rejection, deliberate +reactivation with the updated candidate, and a passing recheck that supersedes +the original packet. Retain only sanitized deterministic test artifacts. + +Update `docs/current/agent-access-contracts.md`, +`docs/current/architecture.md`, `docs/current/security-and-redaction.md`, +`docs/guides/mcp-setup.md`, `docs/guides/agent-quickstart.md`, `COMMANDS.md`, +and `docs/current/project-status.md` to the narrowest verified state. Update +repository agent instructions only with implemented commands and approval +language. Keep product-benefit claims in the evaluation track; this rehearsal +does not satisfy them. + +Exit: the complete local workflow is provider-free verified, canonical docs +match the shipped contracts, and the feature can move out of planned status +without claiming attended utility or CodeMesh outcome improvement. + +## Commit And Review Sequence + +Use one Conventional Commit per accepted work package: + +1. `feat: add development feedback session authority` +2. `feat: add client feedback MCP profile` +3. `feat: add feedback maintainer MCP profile` +4. `feat: integrate feedback profiles with installation probes` +5. `docs: verify development feedback workflow` + +WP5 test code may accompany the fourth commit when it is needed to verify the +integrated implementation; the fifth commit is the final evidence and +documentation promotion. Do not stack all packages into one review. Each +commit must preserve passing earlier contracts, and no package authorizes push, +release, publication, deployment, or use in an external client checkout. + +## Verification Plan + +Run the supported commands from `COMMANDS.md`, reporting passed, failed, +skipped, and unavailable checks separately. At minimum: + +1. run focused tests for the package being implemented; +2. run the complete Agent Access Python suite, Ruff check, and Ruff format + check; +3. run the deterministic MCP fixture evaluation to protect the exact normal + configured-agent surface; +4. run the repository standards and documentation checks for every changed + commit; +5. run the .NET suite before integrated acceptance because the change affects + repository identity, binding, serialization, and lifecycle contracts, even + though the implementation is Python-owned; and +6. run the disposable provider-free end-to-end rehearsal for WP5. + +No authenticated provider, remote transport, issue tracker, external model, +or non-disposable client checkout is required for these gates. Any later +attended VS Code exercise is separately authorized evidence and must identify +the exact commits, session manifest hash, installation-plan hash, client +binding, and result. + +## Acceptance Gates + +The feature is complete only when: + +1. the default normal manifest and provider-free normal runtime behavior remain + unchanged; +2. feedback tools are absent without a valid explicit session and fail closed + for altered, expired, revoked, or wrong-root manifests; +3. client agents cannot override server-derived provenance or classification; +4. setup, binding, and freshness failures remain reportable without treating + rejected context as accepted evidence; +5. v1 packets remain valid and v2 session packets pass strict schema, content-id, + sanitization, containment, and exclusive-creation tests; +6. maintainer tools cannot read a checkout outside the session allowlist; +7. no endpoint can edit CodeMesh, start ingestion, call a provider, transmit + feedback, create an external issue, or assert human approval; +8. installer planning and runtime probes verify the exact profile, session + manifest hash, binding, instructions, and tool surface; +9. the provider-free end-to-end failure-to-recheck rehearsal passes and retains + both append-only packets; and +10. canonical documentation distinguishes planned, implemented, provider-free + verified, human-approved, and product-benefit evidence states. + +Passing deterministic tests proves only the tool and safety contracts. Passing +the local end-to-end rehearsal proves the bounded same-machine workflow. It does +not prove that agents submit useful feedback, that a proposed fix is correct, +or that CodeMesh improves agent outcomes. diff --git a/docs/planning/graph-intelligence-design.md b/docs/planning/graph-intelligence-design.md index 8b337ab..caaa4d7 100644 --- a/docs/planning/graph-intelligence-design.md +++ b/docs/planning/graph-intelligence-design.md @@ -224,8 +224,11 @@ without requiring the full Neo4j, MongoDB, and Qdrant profile. Portable artifact design is distinct from live multi-checkout identity and retention. [Repository Identity and Snapshot Retention](repository-identity-and-snapshot-retention.md) -defines the planning-only requirements for project, checkout, and live index -snapshots, branch and working-state slots, cleanup, and shared stores. +records the implemented local project/checkout/snapshot identity, branch and +working-state slots, retention, and cleanup baseline alongside proposed shared- +store requirements. Portable artifacts and shared coordination remain future +work; see [Identity and Persistence](../current/identity-and-persistence.md) for +the current local implementation. Minimum snapshot contents: diff --git a/docs/planning/method-load-and-retrieval-granularity.md b/docs/planning/method-load-and-retrieval-granularity.md index 7c8ba4e..11d71f4 100644 --- a/docs/planning/method-load-and-retrieval-granularity.md +++ b/docs/planning/method-load-and-retrieval-granularity.md @@ -70,9 +70,10 @@ CodeMesh already contains related mechanisms, but not this capability: - C# parsing emits logical symbol nodes, source declaration nodes, source spans, and semantic relationships. Roslyn provides a suitable base for future control-flow and data-flow measurements. -- Python parsing emits files, classes, functions, async functions, and - containment. It does not yet provide Python call relationships or a - comparable control-flow model. +- Python parsing emits files, classes, functions, async functions, containment, + imports, calls, test relationships, and bounded PyO3 links for the activated + pilot. These heuristic relationships do not provide a comparable semantic + control-flow model or calibrated method-load findings. - Summary generation assigns complexity tiers using node kind, source lines, characters, truncation, and optional complexity metadata. Those tiers select generation budgets; they are not maintainability classifications. diff --git a/docs/planning/next-steps.md b/docs/planning/next-steps.md index 69705b7..6a9ebc9 100644 --- a/docs/planning/next-steps.md +++ b/docs/planning/next-steps.md @@ -2,7 +2,7 @@ Document type: planning and execution-order authority -Reviewed: 2026-08-12 +Reviewed: 2026-09-10 This document is the canonical order for future CodeMesh work. It does not describe implemented behavior; use [Project Status](../current/project-status.md), @@ -17,6 +17,22 @@ change-impact navigation, and validation selection for C#/.NET coding agents. Correctness, safety, freshness, and attributable agent outcomes take precedence over broader feature coverage. +Owner direction has activated a bounded one|nine adoption priority alongside +that wedge. It adds the Rust and Python coverage needed to index one|nine, +repository-specific onboarding across its isolated checkouts, and a durable +feedback path from one|nine agents to CodeMesh development. This activation does +not authorize general parser expansion or make diagnostic feedback product- +benefit evidence. + +Owner direction has also activated the +[Development-Session MCP Feedback Loop](development-feedback-mcp.md) as the +first new implementation priority. The bounded local implementation is now in +source and tests: client agents can record sanitized feedback only during an +explicit hashed session, and a CodeMesh maintainer agent can prepare a +non-authorizing resolution hash. It does not authorize autonomous CodeMesh +edits, external feedback transport, outcome memory, provider spend, +publication, release, or deployment. + Do not select broad parser, SDLC, UI, model-provider, summary, portability, snapshot, shared-store, or agent-memory work merely because it is documented. Activate it only when the ordered work below or explicit owner direction does @@ -24,42 +40,190 @@ so. ## Current Execution Order -1. Keep the clean CodeMesh and pinned YoutubeDownloader summary-free live - baselines passing after material retrieval, packaging, relationship, or MCP - changes. -2. Preserve the spontaneous YoutubeDownloader and prompt-parity Config.Net - zero-adoption campaigns as negative evidence; never attribute their overall - timing or token variance to CodeMesh. -3. Preserve the focused YoutubeDownloader and independent Config.Net assisted - campaigns as replicated utility-after-onboarding evidence, with their - prompt-parity and task-family limitations explicit. -4. Continue the selected explicit-onboarding v1 contract in - [Agent Integration Contract](agent-integration-contract.md). The normal-user - and diagnostic tool profiles are implemented. Next implement unambiguous - checkout binding and fail-closed package freshness, followed by the - provider-free live runtime probe and reviewable installation flow. The - canonical manual repository guidance is already documented. -5. Extend evaluation reporting to distinguish configured/onboarded product use - from spontaneous and evaluator-assisted modes. Then repeat the frozen live - gate and, with explicit model-spend authority, the Config.Net task using the - exact integration shipped to normal users and identical control/treatment - task prompts. Do not relabel this as spontaneous discovery. -6. Repeat the same frozen live and configured/onboarded agent evaluations after - each material tool-selection, ranking, packaging, or relationship change. - Correctness and safety win; efficiency counts only for adopted matched pairs. - Agent runs are model-cost-bearing and require explicit authorization, clean - committed checkouts, and a fresh matching index. -7. Implement the summary-model qualification runner only before summaries are - considered for default retrieval. Require factual, reliability, safety, and - no-summary retrieval comparisons. -8. Complete default secret-file exclusion and configurable ingestion - allow/deny policy before broad artifact ingestion or shared proprietary - workspace storage. -9. Defer method-load and retrieval-granularity analysis, Kubernetes/Helm, - additional parser breadth, UI expansion, outcome memory, portable snapshots, - multi-checkout snapshot retention, shared stores, and broader SDLC ingestion - until configured/onboarded product benefit is repeatable or explicit owner - direction reprioritizes them. +The 2026-09-08 [status checkpoint](../current/project-status.md#review-checkpoint-2026-09-08) +confirms that local preparation through `b2c053c` is already documented and +reviewable. The `a158bc0` package/runtime evidence and dependency-lock hashes +still match; later development-feedback implementation does not update that +historical package evidence. Preserve priorities +1 and 2 and their consumed campaigns. Priority 3 has reached review of the +existing [source packet](../evaluation/review-packet-a158bc0.md), with supported +profiles and representative resource criteria still awaiting selection before +acceptance can be completed. Publishing a review branch/PR, running another paid +campaign, and expanding a deployment each remain separately authority-gated. +Do not repeat completed packaging or recovery rehearsals without a changed +candidate, a new acceptance target, or a concrete evidence gap. + +The checkout-safe integration, atomic Python-plus-Rust ingestion, Rust parser, +feedback loop, instance-owned onboarding, and configured provider-free Primary +gate are implemented. Exact one-step candidate `8844c21`, exact Primary commit +`fe2f761`, the configured preflight, the frozen one|nine pilot, and the +YoutubeDownloader live gate all pass provider-free. The full deterministic +checks also pass at that candidate. The first authorized configured model +campaign remains frozen at historical candidate `0bde606`. The separately +authorized campaign through corrected candidate `0378740` also completed three +correct and safe pairs but remained insufficient: one treatment called status, +no treatment retrieved context, and there were no CodeMesh-attributable wins. +The authorized one-step campaign at `8844c21` then achieved context-package +adoption in all three treatments and passed all six executions safely. Its +verdict was `neutral`: there were no attributable wins or regressions, median +duration was 1.39% lower, and median tokens were 4.13% higher. Both deltas are +inside the 10% neutrality band. +Exact clean candidate `b16eee7` now adds the configured Config.Net suite while +preserving the frozen prompt, targets, thresholds, and source identity. Its +reviewed normal-profile installation, configured preflight, fail-closed probe, +and six-case/18-call live gate pass provider-free with recall, MRR, and nDCG of +1.0 and zero secret leaks. Its separately authorized configured campaign then +achieved context-package adoption in all three treatments but `regressed`: all +three controls passed, while two treatments missed required citations and only +one treatment passed. The sole jointly successful pair was 21.31% slower and +used 117.18% more tokens with CodeMesh. +Exact clean candidate `3804028` replaces one-long-full-task guidance with a +small set of focused implementation and test queries for multi-part work. Its +reviewed Config.Net installation, configured positive and fail-closed probes, +five-case focused suite, unchanged six-case canonical suite, and deterministic +checks all pass provider-free. The focused suite covers all seven required +targets, with `LazyVar.cs` at rank eight. This is correction readiness, not +evidence that a normal agent follows the guidance or benefits from CodeMesh. +The separately authorized exact-candidate campaign then completed all three +pairs with an `improved` verdict. Treatments adopted the context package and +passed in every repetition, controls passed two of three, and the result +contains one CodeMesh-attributable win with no regressions or safety failures. +Median treatment duration was 13.69% lower and tokens were 3.59% lower across +the two jointly successful pairs. This is positive evidence for one bounded +campaign, not repeatable product benefit or proof that the wording change alone +caused the outcome. +Independent clean candidate `64d4052` then preserved the frozen +YoutubeDownloader impact task in a configured suite. Its deterministic checks, +reviewed plan, configured positive and rejection probes, impact retrieval gate, +and unchanged canonical live gate passed provider-free. The authorized paired +campaign achieved 100% control and treatment correctness with full treatment +adoption and no safety failure, but treatment medians were 49.79% slower and +used 41.00% more tokens across all three pairs. Its `regressed` verdict is an +efficiency result, not a correctness regression. Each treatment attempted two +or three context packages, while the frozen provider-free task-shaped query +covered all six targets; raw agent query arguments were not retained. +Exact clean candidate `fbc1433` now replaces proactive decomposition with a +task-adaptive stop rule and a strict three-package ceiling. The frozen +task-shaped package covered all six impact targets in one call, while three +focused packages repeated five of 24 ranked items and used 35,991 formatted +characters versus 12,000. Its deterministic checks, reviewed configured +preflight, positive and rejection probes, adaptive gate, unchanged impact gate, +and unchanged canonical gate all pass provider-free. This is correction +readiness, not proof that agents follow the guidance or benefit from it. +Its separately authorized configured campaign then completed all three pairs +correctly and safely, with exactly one context-package attempt in every +treatment. Median treatment tokens were 6.63% lower, but median duration was +10.0192% higher, just beyond the frozen 10% threshold, so the verdict was +`regressed`. The six executions also reported 2,596,056 aggregate tokens and +exceeded the authorized 1.6M reported-token ceiling because the evaluator had +no hard aggregate-token stop at that candidate. The result is consumed and must +not be retried. +Exact clean candidate `0f18071` implements the explicit fail-closed capped Codex +runner. Its deterministic, configured positive/rejection, adaptive, impact, and +canonical gates all passed provider-free. The one separately authorized capped +campaign stopped on its first control execution when the provider reported no +available credits. No pair completed. The proxy retained the full 267,528-token +reservation for the interrupted response, admitted no subsequent request, and +recorded zero retries. This campaign is incomplete and consumed; it establishes +the fail-closed failure behavior, not agent or product benefit, and must not be +retried or topped up. +The development-session MCP feedback priority is complete in the current local +source. The default normal and diagnostic surfaces remain separate; session +manifests are immutable, expiring, hash-pinned, and separately revocable; v2 +packets derive provenance in the server; maintainer intake is allowlisted and +read-only; and resolution drafts remain human-review-required. A provider-free +real-stdio test covers failure, maintainer intake and planning, passing recheck, +derived resolution, revocation rejection, retained packets, and absence of a +CodeMesh edit. This is implementation and local workflow evidence, not attended +utility, product benefit, publication, release, or deployment evidence. + +The next priorities are: + +1. **Preserve all configured campaign outcomes.** Retain `b16eee7` as the + Config.Net correctness regression, `3804028` as the separately improved + Config.Net campaign, and `64d4052` as the independently replicated + YoutubeDownloader efficiency regression, and `fbc1433` as the separately + duration-regressed adaptive campaign and token-ceiling control failure. Do + not pool or selectively discard them, and do not infer query-guidance + causality from reports without raw query arguments. +2. **Continue the measured overhead diagnosis provider-free.** The evaluator now + defines reported tokens explicitly, requires a compatible runner-declared + hard cap before preflight, passes only the remaining allowance to each run, + stops at the boundary or an incomplete run, and never retries automatically. + The default Codex runner truthfully fails that capability gate; the explicit + `capped-codex` runner implements it. Exact clean instrumentation candidate + `66033ad` measured the + unchanged adaptive suite with one warm-up and five repetitions. The + task-shaped package had 16,830.931 ms total p50: search was the largest stage + at 16,725.370 ms (99.37%), while item hydration was 102.668 ms (0.61%). The + defined hydration-dominance gate therefore failed, so do not apply the + proposed four-task hydration concurrency or guess at another optimization. + Retain the diagnosis separately from retrieval claims. The explicit + `capped-codex` runner is now implemented with pre-request input counting, + worst-case reservation, zero provider request/stream retries, retained + ambiguous reservations, and final Codex/proxy reconciliation. Verify it with + the fresh exact-candidate deterministic, configured, retrieval, and safety + gates before using separately authorized model spend. Those gates passed at + `0f18071`, but the single authorized campaign ended incomplete for unavailable + provider credits and is now consumed. Do not retry or top up it, `64d4052`, + or `fbc1433`. Request-isolated search-operation instrumentation now separates + store fetches, candidate expansion, declaration/content reads, and ranking. + Clean candidate `2879e7f` now identifies a lexical-scoping cost with controlled + unrelated-node growth: the retained query plan loaded searchable properties + before filtering the selected repository. The single correction moves + repository filtering to the initial match. Clean correction `4fb9e81` passed + all three frozen live suites, preserved ordered candidates, and reduced + task-shaped package p50 by 73.96% under the same 50,000-node synthetic load. + Retain this local result without treating it as configured-agent benefit. + Preserve scoring and budgets; any further optimization needs new diagnosis. Nested + and concurrent timing totals are not additive. This diagnosis does not prove + the cause of the historical store's 16.7-second result. +3. **Complete available local security and release prerequisites.** The owner + requested completion of all available CodeMesh work. Address remaining + documentation drift and reviewable release prerequisites while paid product + evaluation remains authority-gated. Local evaluation path hardening, argv + regression tests, adversarial UI coverage, and the 39-finding boundary review + are implemented and verified locally. Documentation drift is reconciled and + a canonical source version and reviewable release procedure are implemented. + Local artifact verification is refreshed at clean `a158bc0`, including the + parser-container, UI, and feedback corrections. The review packet now includes + draft notes, dependency identities, and fresh configured-runtime positive + and rejection probes. A fresh hosted scan, + environment acceptance, supported upgrade/rollback acceptance, and release + approval remain pending. Exact local CodeQL extraction and boundary review + now pass with pinned query packs and separately evaluated threat models; + preserve all 253 reviewed local-input results and the invalid cached attempts. + The HTML correction removes its local finding, and feedback recording now + preserves existing packet destinations. Disposable same-version recovery and + a development-baseline upgrade/re-ingestion/deletion/rollback rehearsal now + pass their local mechanism checks. The frozen sample dependency advisory + remains a failed check; supported-upgrade and environment acceptance stay + open. The standard read-only MSBuild output warning is now corrected and + verified with concurrent container parses; custom build targets and Windows + still need their own acceptance evidence. + Preserve the deferred-feature activation + boundary and do not publish, release, or expand a deployment implicitly. +4. **Require separate authority for expansion.** Expand onboarding to Secondary, Tertiary, + Release, or Hotfix only with separate authority, an exact role-specific + binding, and a passing checkout-specific probe. Keep Run-only excluded + pending a separate acceptance decision. + +Do not run another paid campaign unless explicit authority names its model, +reasoning level, repetitions, seed, and ceiling and every provider-free gate +passes first. No further model spend is currently authorized. Preserve deferred +breadth work unless explicit owner direction changes this order. + +After those priorities, use the implemented summary-model qualification runner +to qualify a complete deployment profile before summaries are considered for +default retrieval. Maintain the implemented default secret-file exclusion and +configurable ingestion path policy before broad artifact ingestion or shared +proprietary workspace storage. + +Defer method-load and retrieval-granularity analysis, Kubernetes/Helm, parser +breadth beyond the activated Python/Rust pilot, UI expansion, outcome memory, +portable snapshots, shared stores, and broader SDLC ingestion until +configured/onboarded product benefit is repeatable or explicit owner direction +reprioritizes them. The evidence supporting this order is summarized in [Project Status](../current/project-status.md). Exact evaluation commands and @@ -76,18 +240,22 @@ execution order above. | Self-ingestion and smoke | Implemented | Maintain source-free smoke and fresh-index behavior. | [Self-Analysis](../guides/self-analysis.md) | | MCP client setup | Implemented | Maintain host examples and troubleshoot environment differences. | [MCP Setup](../guides/mcp-setup.md) | | Repository status and freshness | Implemented locally | Older records remain unknown until re-ingested; shared freshness is future work. | [Agent Access Contracts](../current/agent-access-contracts.md) | -| Redaction and sensitive-file policy | Partial | Add known-secret-file exclusion, configurable allow/deny policy, and broader tests. | [Security and Redaction](../current/security-and-redaction.md) | -| Python parser | Partial | Add imports, calls, routes, MCP handlers, and test relationships after activation. | [Architecture](../current/architecture.md) | +| Redaction and sensitive-file policy | Implemented local baseline | Maintain default known-secret exclusions, reviewed repository-relative allow/deny patterns, pre-persistence redaction, and path-policy identity; extend only for evidenced repository conventions. | [Security and Redaction](../current/security-and-redaction.md) | +| Python parser | Implemented bounded pilot | Maintain imports, calls, tests, and PyO3 boundary relationships; keep unrelated breadth evidence-driven. | [one\|nine Priority](onenine-adoption-and-feedback.md) | +| Rust parser | Implemented bounded pilot | Maintain the heuristic structural, relationship, test, path-exclusion, and PyO3 contract; broaden only from measured gaps. | [one\|nine Priority](onenine-adoption-and-feedback.md) | +| Multi-language ingestion | Implemented | Maintain atomic language-set publication and language preservation during refresh and watch. | [one\|nine Priority](onenine-adoption-and-feedback.md) | | Markdown parser | Partial | Add links from docs to commands, endpoints, projects, and source when justified. | [Architecture](../current/architecture.md) | -| Context packages | Implemented baseline | Maintain compact agent output, citations, rationale, and budget behavior. | [Agent Access Contracts](../current/agent-access-contracts.md) | +| Context packages | Implemented baseline; provider-free timing attributes the retained task-shaped latency to search, not hydration | Maintain compact agent output, citations, rationale, and budget behavior; do not apply the rejected hydration-concurrency optimization. | [Agent Access Contracts](../current/agent-access-contracts.md) | | Local web inspection | Implemented initial slice | Expand only when it serves a measured agent or audit need. | [Agent Access Contracts](../current/agent-access-contracts.md) | -| Tool guidance and onboarding | Server instructions, manual guidance, and normal/diagnostic MCP profiles implemented; v1 contract selected | Implement checkout binding, fail-closed package freshness, live probe, and reviewable installation; then establish configured/onboarded evidence. | [Agent Integration Contract](agent-integration-contract.md) | +| Tool guidance and onboarding | One-step fail-closed context-package entry and task-adaptive stop/decompose guidance verified provider-free; `fbc1433` achieved one-package adoption and correct/safe completion but regressed duration | Diagnose overhead provider-free; do not retry the consumed campaign. | [Agent Integration Contract](agent-integration-contract.md) | +| one\|nine agent feedback | Implemented bounded pilot | Review sanitized, provenance-bound packets and connect recurring findings to controlled evidence without automatic roadmap mutation. | [one\|nine Priority](onenine-adoption-and-feedback.md) | +| Development-session MCP feedback loop | Implemented and provider-free verified locally | Maintain explicit hashed sessions, bounded client recording, allowlisted read-only maintainer intake, human approval before source edits, append-only rechecks, and unchanged normal/diagnostic profiles. Attended usefulness and product benefit remain unproven. | [Development Feedback MCP](development-feedback-mcp.md) | | Refresh and watch | Implemented initial slice | Native file events and broader operational visibility remain optional. | [Self-Analysis](../guides/self-analysis.md) | | Deployment parsing | Partial | Dockerfile and Compose exist; Helm, Kubernetes, CI/CD, and code-to-runtime links are deferred. | [SDLC Roadmap](sdlc-roadmap.md) | | Test and validation context | Partial | Link tests and commands to implementation with source-backed confidence. | [Graph Intelligence Design](graph-intelligence-design.md) | -| Evaluation harnesses | Implemented | Maintain fixtures and live gates; run paid agent/model campaigns only with authority. | [Evaluation](../evaluation/mcp-effectiveness.md) | +| Evaluation harnesses | Implemented for selected v1 contract, including explicit fail-closed capped-Codex mode | Preserve every configured outcome, including both YoutubeDownloader regressions, the `fbc1433` token-ceiling breach, and the consumed incomplete `0f18071` capped campaign. No further model spend is authorized. | [Evaluation](../evaluation/mcp-effectiveness.md) | | Repository identity and snapshots | Local profile implemented | Maintain deterministic identity/slot/retention behavior; shared coordinator, tenant security, and private overlays remain deferred. | [Identity and Snapshot Requirements](repository-identity-and-snapshot-retention.md) | -| Summary-model qualification | Procedure proposed | Implement the runner and qualify a complete deployment profile before default use. | [Summary Model Qualification](summary-model-qualification.md) | +| Summary-model qualification | Runner implemented; no profile qualified | Freeze a reviewed corpus and profile, obtain provider/source/spend authority, complete two blinded reviews, isolated live-retrieval comparison with query-level assessment, and resource/cost measurement, then pass every gate before default use. | [Summary Model Qualification](summary-model-qualification.md) | | Method load and retrieval granularity | Proposal | Calibrate human-maintainability, tokenizer-bound representation, and agent-context thresholds before exposing advisory findings. | [Method Load And Retrieval Granularity](method-load-and-retrieval-granularity.md) | | Graph analysis and outcome memory | Proposal | Activate only when measured relationship or impact limitations justify it. | [Graph Intelligence Design](graph-intelligence-design.md) | | Broader SDLC intelligence | Deferred roadmap | Add contract, persistence, operations, security, and ownership artifacts only after activation. | [SDLC Roadmap](sdlc-roadmap.md) | diff --git a/docs/planning/onenine-adoption-and-feedback.md b/docs/planning/onenine-adoption-and-feedback.md new file mode 100644 index 0000000..b3fd591 --- /dev/null +++ b/docs/planning/onenine-adoption-and-feedback.md @@ -0,0 +1,293 @@ +# one|nine Adoption and Feedback Priority + +Document type: owner-activated implementation plan and pilot gate + +Reviewed: 2026-09-01 + +This document defines the bounded one|nine adoption priority selected in +[Next Steps](next-steps.md). Its initial implementation and provider-free pilot +gate passed from clean candidate `ee1d4d9`; the later configured-evaluator +campaign was finalized at `0bde606`, the bounded status-to-context correction +passed the same provider-free gates at `0378740`, and the one-step entry +correction passed them at `8844c21`. Current behavior and evidence remain +authoritative in +[Project Status](../current/project-status.md); this planning record does not +establish that CodeMesh improves one|nine agent outcomes. + +## Decision + +The selected CodeMesh product work is a safe, measured one|nine pilot that lets +agents use CodeMesh during real repository tasks and return fast, actionable +feedback to CodeMesh development. The pilot activated: + +- unambiguous binding and freshness enforcement for isolated checkouts; +- atomic Python and Rust ingestion; +- a first-class Rust parser module; +- the Python relationships needed to connect application code to PyO3 exports; +- repository-owned MCP onboarding for managed one|nine instances; and +- a sanitized local feedback recorder, validator, and summarizer. + +The existing C#/.NET product wedge and evidence remain valid within their stated +boundaries. Owner direction activates only the Python and Rust breadth needed by +this pilot. JavaScript and broader parser or SDLC expansion remain deferred +unless measured one|nine tasks justify them. + +## Boundaries + +- Checkout correctness and freshness precede rollout. No agent may silently use + the newest, closest, or otherwise implicit snapshot when a binding is missing, + ambiguous, or stale. +- Start provider-free, with generated summaries and embeddings disabled. Model- + backed paired evaluation still requires explicit spend authorization. +- Treat Graphify as an existing one|nine exploration aid and comparative + baseline. The pilot must demonstrate incremental value rather than assuming + replacement value. +- Treat routine agent feedback as diagnostic product input. It is not a fixture + pass, controlled agent comparison, release gate, deployment acceptance, or + production authorization. +- Do not automatically create external issues, edit CodeMesh, change priorities, + or transmit repository material from feedback packets. +- Do not retain raw prompts, MCP payloads, source excerpts, secrets, credentials, + or private environment values in feedback by default. + +## Ordered Delivery + +### 1. Checkout-safe integration + +Complete the outstanding v1 safety work in +[Agent Integration Contract](agent-integration-contract.md): + +1. bind MCP startup to an explicit project, checkout, repository root, and + source view; +2. reject missing, ambiguous, commit-mismatched, or source-view-mismatched + context packages on the server side; +3. add a provider-free probe that launches the exact configured stdio command, + checks server instructions and the tool profile, reaches the stores, resolves + the intended checkout, and proves snapshot freshness; and +4. make installation and repository guidance changes reviewable and opt-in. + +A shared local CodeMesh service may serve multiple developer checkouts only +when these bindings keep their snapshots isolated. A single unbound global MCP +alias is not an acceptable one|nine integration. + +### 2. Atomic Python and Rust ingestion + +Add a language-set ingestion contract, such as `--languages python,rust`, that +publishes one atomic snapshot for a checkout. Snapshot identity, parser-profile +fingerprints, refresh, and watch must preserve the complete selected language +set. A later language-specific ingest must not silently replace a composite +active snapshot with a partial graph. + +Add Python imports, calls, tests, and PyO3 boundary relationships sufficient to +connect one|nine application symbols to the native module. Record JavaScript as +an explicit coverage gap when it appears in a task rather than hiding it behind +partial results. + +### 3. Rust parser module + +Add `src/CodeMesh.Parser.Rust` and expose `rust` through the CLI, ingestion +contracts, parser profiles, tests, and documentation. The initial parser must +emit stable source spans and content hashes for: + +- files and modules; +- structs, enums, traits, functions, and methods; +- `impl` blocks and trait implementations; +- `use` imports, calls, and type references; +- Rust test functions and their attributes; and +- PyO3 modules, classes, functions, and exported names. + +Repository enumeration must exclude Cargo `target` output or honor applicable +ignore rules. Context packages involving Rust must recommend the repository's +relevant formatting, Clippy, and test commands. + +### 4. Durable agent feedback + +Provide a versioned local recorder and validator that writes ignored artifacts +under `.codemesh-feedback/` in each participating checkout. Each packet must be +small enough to review and include: + +- schema version and diagnostic or controlled-evaluation classification; +- one|nine role, repository root, branch, commit, and dirty state; +- CodeMesh commit, project, checkout, snapshot, language set, and parser profile; +- task family and CodeMesh tools attempted or called, without raw payloads; +- helpful, incorrect, missed, stale, or ambiguous returned paths; +- fallback reason and validation outcome; +- issue category, such as setup, binding, freshness, missing language, missing + relationship, ranking, budget, or validation; +- a minimal reproduction or expected target set when safe to retain; and +- a proposed smallest correction and reporter confidence. + +Add a CodeMesh-side summarizer that reads validated packets from the managed +checkout outboxes, groups recurring failures, preserves exact commit and +snapshot provenance, and produces a prioritized review packet. The CodeMesh +agent or maintainer triages whether a packet may warrant implementation, +evaluation, documentation, or no action; a human must approve any resulting +CodeMesh source or documentation change. + +The later owner-activated +[Development-Session MCP Feedback Loop](development-feedback-mcp.md) now +implements explicit local MCP exposure and a human-approved maintainer-agent +workflow over this packet mechanism. It remains separate from the completed +one|nine pilot: no one|nine checkout is enrolled merely because the tools now +exist, and attended usefulness still requires separate authorization and +evidence. + +### 5. Instance-owned rollout + +Extend the one|nine instance configuration repository to generate and validate +reviewable project-scoped MCP configuration and repository guidance for each +participating checkout. Configuration must forward secret names or use a safe +launcher; it must not copy secret values into tracked files or feedback. + +Begin with one pinned, clean checkout. Expand only after the live probe and +feedback path pass for that checkout. Primary, Secondary, Tertiary, Release, and +Hotfix may participate with exact role-specific bindings. Keep Run-only outside +the initial rollout because its operational role and pinned source require a +separate acceptance decision. + +Release-checkout retrieval and feedback remain developer-tool evidence. They do +not establish release readiness or authorize deployment. + +### 6. Evaluation and improvement loop + +Create a provider-free one|nine live suite with real expected files, symbols, +and relationships for: + +- Python implementation discovery, impact navigation, and test selection; +- Rust implementation discovery, call/type navigation, and Rust validation; +- Python-to-PyO3-to-Rust navigation; +- clean and dirty source views; +- wrong-checkout, ambiguous-checkout, and stale-snapshot rejection; and +- compact package behavior under the normal MCP tool profile. + +Compare the pilot with ordinary repository exploration plus the existing +Graphify integration. Use feedback packets to find candidate improvements, but +retain controlled suites, prompts, gold targets, commits, snapshots, and +outcomes separately. The first explicitly authorized configured/onboarded +campaign used identical control and treatment task prompts and completed safely, +but no treatment retrieved context. Treat that as measured handoff evidence, +not product-benefit evidence. + +## Completion Gates + +The initial priority is complete only when: + +1. the checkout binding and provider-free runtime probe fail closed in negative + cases and pass against the selected one|nine pilot checkout; +2. a single active snapshot contains the required Python and Rust graph without + partial-language replacement; +3. Rust fixtures and the selected one|nine crate pass structural, relationship, + PyO3, test-discovery, determinism, and path-exclusion checks; +4. one|nine agents can record sanitized, valid feedback and CodeMesh can produce + a provenance-preserving prioritized summary; +5. the provider-free one|nine live suite passes its frozen safety and retrieval + gates; and +6. canonical current documentation is updated to describe only the behavior and + evidence actually implemented and verified. + +Passing these gates establishes a usable bounded pilot. A broader product- +benefit claim still requires adopted, comparable agent outcomes with +attributable correctness, safety, or efficiency improvement. + +## Pilot Gate Result + +All six completion gates passed on 2026-08-31 from clean CodeMesh candidate +`ee1d4d9` against Primary commit +`e12d0281b687e58c2baac85f771a684df6fa8552` and the provider-free composite +snapshot recorded in [Project Status](../current/project-status.md). The frozen +instance-owned suite passed four positive retrieval cases for Python, Rust, +test selection, and the PyO3 boundary, plus fail-closed wrong-checkout, +wrong-root, wrong-source-view, and dirty-checkout cases. It validated three +feedback packets and produced a provenance-preserving summary. The pinned +YoutubeDownloader live gate also passed after a discovered partial-symbol +ranking regression was corrected and retested. + +This result is clean-candidate provider-free diagnostic evidence. Configured/ +onboarded paired agent tasks, not this pilot gate, can test attributable product +benefit. Exact identities, metrics, report hashes, and the retained failed gate +are in the +[Priority 1 evidence record](../evaluation/evidence/onenine-priority1-ee1d4d9.json). + +Subsequent configured-evaluator corrections preserved the initial gate as +historical evidence and finalized the frozen campaign at CodeMesh candidate +`0bde606` against Primary commit `fe2f761`. That provider-free preflight, all +eight one|nine retrieval and rejection gates, and the pinned YoutubeDownloader +live gate passed. The exact final identities and claim ceiling are retained in +the [configured one|nine evidence record](../evaluation/evidence/configured-onenine-provider-free-0bde606.json). + +The bounded status-to-context correction is frozen at clean candidate +`0378740`. The full deterministic checks, configured preflight, eight one|nine +gates, three retained feedback packets, and pinned YoutubeDownloader live gate +all passed again without invoking a model provider. The exact identities and +claim ceiling are retained in the +[corrected provider-free evidence record](../evaluation/evidence/configured-onenine-provider-free-0378740.json). + +The one-step entry correction is frozen at exact clean candidate `8844c21`. +The full deterministic checks, configured preflight, eight one|nine gates, +three retained feedback packets, and pinned YoutubeDownloader live gate all +passed with provider mode `none`. Instance configuration commit `f9f199d` +repinned only Primary; all managed-instance generation checks passed without +activating another checkout. The exact identities and claim ceiling are in the +[one-step provider-free evidence record](../evaluation/evidence/configured-onenine-provider-free-8844c21.json). + +The first authorized paired model campaign from those exact identities +completed three correct and safe pairs. Two treatments called repository status, +none called the context-package tool, and the verdict was `insufficient` with no +attributable wins. The handoff correction now has full clean-candidate +provider-free verification. The sanitized historical outcome is retained in the +[configured agent evidence record](../evaluation/evidence/configured-onenine-agent-0bde606.json). + +That separately authorized `0378740` campaign also completed three correct and +safe pairs but remained `insufficient`. One treatment called repository status, +none called the context-package tool, and there were no attributable wins. The +status-result action therefore did not produce the required handoff. Preserve +the consumed campaign in the +[corrected configured agent evidence record](../evaluation/evidence/configured-onenine-agent-0378740.json). + +The authorized one-step `8844c21` campaign then completed three correct and safe +pairs with context-package adoption in every treatment. It had no attributable +wins or regressions and received a `neutral` verdict; median duration was 1.39% +lower and median tokens were 4.13% higher, both within the configured 10% +threshold. This establishes the selected one|nine handoff, not product benefit. +Preserve the consumed result in the +[one-step configured agent evidence record](../evaluation/evidence/configured-onenine-agent-8844c21.json). +Configured Config.Net preparation now passes provider-free at exact candidate +`b16eee7`. Its separately authorized paired campaign adopted the context-package +tool in all three treatments but `regressed`: all three controls passed, while +two treatments missed required implementation or test citations. Preserve the +consumed negative result in the +[configured Config.Net agent evidence record](../evaluation/evidence/configured-config-net-agent-b16eee7.json). +The provider-free task-shaped package-coverage diagnosis now shows the full +task misses six of seven targets while four focused task-derived queries cover +all seven. The bounded guidance correction is committed at exact clean +candidate `3804028`; its deterministic checks, reviewed configured preflight, +positive and fail-closed probes, focused-query suite, and unchanged canonical +suite all pass provider-free. This does not establish normal-agent compliance +or product benefit. The separately authorized exact-candidate campaign then +adopted the context package and passed in all three treatments. Controls passed +two of three, producing one attributable win, zero treatment regressions, and +an `improved` verdict; efficiency medians were favorable across the two jointly +successful pairs. This remains single-campaign evidence rather than repeatable +benefit. The retest is consumed, and no further model spend is currently +authorized. + +Independent configured replication on the frozen YoutubeDownloader impact +task then achieved full control and treatment correctness, full treatment +adoption, and no safety failure at exact candidate `64d4052`. Treatment medians +were nevertheless 49.79% slower and used 41.00% more tokens across all three +pairs, producing an efficiency-driven `regressed` verdict. This strengthens +cross-repository handoff evidence but still does not establish repeatable +product benefit. Preserve the consumed campaign. The required task-adaptive +provider-free diagnosis and its smallest guidance correction now pass the +exact-candidate gates at `fbc1433`: one task-shaped package covered all six +frozen targets, while three proactive focused packages repeated content and +consumed three times the formatted package budget. Its separately authorized +configured campaign then completed all six runs correctly and safely with one +package in every treatment. Median tokens fell 6.63%, but median duration rose +10.0192%, producing another efficiency-driven `regressed` verdict. The campaign +also exceeded its authorized 1.6M aggregate reported-token ceiling because the +historical evaluator lacked a hard stop. Preserve the consumed result, diagnose overhead +provider-free, and retain the implemented fail-closed accounting, mandatory +compatible-runner capability gate, boundary stop, and no-retry behavior. The +checked-in Codex runner cannot guarantee the requested cap and remains blocked +before any future model call. diff --git a/docs/planning/summary-model-qualification.md b/docs/planning/summary-model-qualification.md index c717d0f..9e30278 100644 --- a/docs/planning/summary-model-qualification.md +++ b/docs/planning/summary-model-qualification.md @@ -2,14 +2,20 @@ ## Status -This document defines a design and operating procedure. The dedicated -qualification command and report schema described here are not implemented yet. -Existing ingestion and live-evaluation commands can be used to run parts of the -procedure manually. +This document defines the implemented runner contract and its operating +procedure. `codemesh summaries qualify run`, `bind-retrieval`, `compile`, and +`compare` now provide provider-neutral generation, blinded review packets, +live-report binding, gate enforcement, sanitized reports, and fail-closed +comparison compatibility. The production-generation/live-retrieval boundary is +recorded in +[ADR 0002](../decisions/0002-summary-qualification-evidence-boundary.md). CodeMesh does implement versioned, complexity-tiered summary prompts and -completion ceilings. The qualification automation and certification report -remain planned. +completion ceilings. The runner deliberately does not authorize model spend, +approve source transmission, label a corpus, perform human review, or silently +create live stores. Operators must supply those separately, and +`bind-retrieval` requires an explicit assertion that the no-summary and +candidate indexes were isolated. No summary deployment profile is currently CodeMesh-qualified. Generated summaries are optional and must remain outside the core C#/.NET product-proof @@ -348,39 +354,64 @@ report. Store them only in an access-controlled evidence directory when explicitly requested. Reports must never contain credentials or unredacted secret values. -## Proposed Automation +## Implemented Automation -A future implementation should add a versioned suite and commands similar to: +The .NET CLI implements the runner because it can directly reuse the production +summary prompt, parser, budget policy, provider adapters, and redaction path: ```powershell -cd agent-access -uv run python -m codemesh_agent_access eval summaries ` - --suite codemesh-summary-model ` - --provider lmstudio ` - --model ` - --profile local-offline - -uv run python -m codemesh_agent_access eval summaries ` - --suite codemesh-summary-model ` - --provider openai ` - --model ` - --profile online-approved-data - -uv run python -m codemesh_agent_access eval summaries compare ` - candidate-a.json candidate-b.json reference.json +dotnet run --project src/CodeMesh.Cli -- summaries qualify run ` + --suite ` + --profile ` + --private-output ` + --review-output ` + --summary-provider ` + --summary-model ``` -The runner should: - -- Prepare redacted inputs through the production summary prompt. -- Isolate each candidate's summary store or database namespace. -- Capture provider usage without retaining credentials. -- Generate blinded review packets. -- Run the live retrieval suite against each isolated index. -- Enforce predeclared gates and emit machine-readable and human-readable - reports. -- Reject comparisons with incompatible corpus, prompt, repository, or ranking +The remaining commands and exact options are indexed in +[`COMMANDS.md`](../../COMMANDS.md). Checked-in +[`summary-qualification-suite.example.json`](../evaluation/assets/summary-qualification-suite.example.json) +and +[`summary-qualification-profile.example.json`](../evaluation/assets/summary-qualification-profile.example.json) +and +[`summary-qualification-review.example.json`](../evaluation/assets/summary-qualification-review.example.json) +plus the +[`summary-qualification-retrieval-assessment.example.json`](../evaluation/assets/summary-qualification-retrieval-assessment.example.json) +and +[`summary-qualification-deployment-evidence.example.json`](../evaluation/assets/summary-qualification-deployment-evidence.example.json) +show the versioned suite, profile, reviewer, query-level retrieval, and +resource/cost evidence schemas. The suite is intentionally synthetic and +small: it tests runner wiring but cannot support a real qualification claim. +Its deployment budgets are illustrative and must be replaced before a real +run. + +The runner: + +- Prepares redacted inputs through the production summary prompt. +- Requires distinct `calibration` and held-out `qualification` partitions, + uses calibration only for warm-up, and runs at least three measured + repetitions over qualification items. Runner schema v1 supports only an + explicit no-retry policy, so first-attempt and eventual completion rates + remain separately visible but equal. +- Requires separately isolated no-summary and candidate stores, then binds the + resulting live reports and a reviewed query-level assessment to the + qualification identity. +- Requires measured host/accelerator resource evidence for local or private + profiles, or provider cost bound to the recorded pricing snapshot for online + profiles. +- Captures provider usage without retaining credentials. +- Generates blinded review packets. +- Reuses the live retrieval suite rather than duplicating its ranking metrics. +- Enforces predeclared gates, emits a machine-readable report, and prints a + concise human-readable outcome. +- Rejects comparisons with incompatible corpus, prompt, repository, or ranking identities. -Until that automation exists, manual results are research evidence only and -must not be presented as a CodeMesh-qualified deployment profile. +Private archives and blinded packets are create-new, restricted artifacts and +are not publication-safe by default. The compiled report omits source and +generated response text. Both review files must bind the same non-empty hash of +their disagreement-resolution record. A report is `invalid-run` when reviews, +resolution, retrieval assessment, deployment measurement, clean source +identity, suite identity, or profile identity is missing; only a complete +passing report may claim a deployment profile is qualified. diff --git a/src/CodeMesh.Cli/CodeMesh.Cli.csproj b/src/CodeMesh.Cli/CodeMesh.Cli.csproj index 4e95c3c..7dd2f7b 100644 --- a/src/CodeMesh.Cli/CodeMesh.Cli.csproj +++ b/src/CodeMesh.Cli/CodeMesh.Cli.csproj @@ -8,6 +8,7 @@ + diff --git a/src/CodeMesh.Cli/Program.cs b/src/CodeMesh.Cli/Program.cs index d5e96bb..9959b67 100644 --- a/src/CodeMesh.Cli/Program.cs +++ b/src/CodeMesh.Cli/Program.cs @@ -9,10 +9,12 @@ using CodeMesh.Ingestion; using CodeMesh.Ingestion.Embedding; using CodeMesh.Ingestion.Summary; +using CodeMesh.Ingestion.Summary.Qualification; using CodeMesh.Parser.CSharp; using CodeMesh.Parser.Deployment; using CodeMesh.Parser.Markdown; using CodeMesh.Parser.Python; +using CodeMesh.Parser.Rust; using CodeMesh.Storage; using CodeMesh.Storage.Mongo; using CodeMesh.Storage.Neo4j; @@ -26,6 +28,16 @@ return 0; } +if (command is "--version" or "version") +{ + var version = typeof(CodeMeshStatusService).Assembly + .GetCustomAttributes(typeof(System.Reflection.AssemblyInformationalVersionAttribute), false) + .Cast() + .Single().InformationalVersion; + Console.WriteLine($"CodeMesh {version}"); + return 0; +} + var root = ReadOption(args, "--root") ?? Directory.GetCurrentDirectory(); EnvironmentFileLoader.LoadIfExists(root); var json = args.Any(arg => string.Equals(arg, "--json", StringComparison.OrdinalIgnoreCase)); @@ -149,14 +161,19 @@ try { var subcommand = ReadPositional(args, 1)?.ToLowerInvariant(); - if (subcommand is not "coverage") + if (subcommand == "coverage") { - throw new InvalidOperationException("Usage: codemesh summaries coverage "); + var coverage = await RunSummaryCoverageAsync(args).ConfigureAwait(false); + PrintSummaryCoverage(coverage, json); + return 0; } - var coverage = await RunSummaryCoverageAsync(args).ConfigureAwait(false); - PrintSummaryCoverage(coverage, json); - return 0; + if (subcommand != "qualify") + { + throw new InvalidOperationException("Usage: codemesh summaries coverage | summaries qualify "); + } + + return await RunSummaryQualificationAsync(args, root, json).ConfigureAwait(false); } catch (Exception exception) { @@ -278,6 +295,27 @@ static bool HasFlag(string[] args, string name) return null; } +static IReadOnlyList ReadRepeatedOptions(string[] args, params string[] names) +{ + var values = new List(); + for (var index = 0; index < args.Length; index++) + { + if (!names.Any(name => string.Equals(args[index], name, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + if (index + 1 >= args.Length || args[index + 1].StartsWith("--", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Option {args[index]} requires a value."); + } + + values.Add(args[++index]); + } + + return values; +} + static string? ReadPositional(string[] args, int index) { return args.Length > index && !args[index].StartsWith("--", StringComparison.Ordinal) @@ -290,7 +328,8 @@ static async Task RunIngestAsync(string[] args) var repositoryRoot = Path.GetFullPath( ReadAnyOption(args, "--root", "--repository-root") ?? Directory.GetCurrentDirectory()); - var language = ReadOption(args, "--language") ?? "csharp"; + var languages = ReadLanguageSet(args); + var language = string.Join(',', languages); var forceEmbeddings = HasFlag(args, "--force-embeddings"); var solutionPath = CodeMeshIngestPathResolver.ResolveFile( repositoryRoot, @@ -300,13 +339,15 @@ static async Task RunIngestAsync(string[] args) repositoryRoot, ReadAnyOption(args, "--project", "--project-path"), "Project"); + var pathFilter = CreateRepositoryPathFilter(args); - if (!IsSupportedLanguage(language)) + var unsupportedLanguages = languages.Where(item => !IsSupportedLanguage(item)).ToArray(); + if (unsupportedLanguages.Length > 0) { - throw new InvalidOperationException($"Unsupported ingestion language: {language}. Supported values: csharp, deployment, markdown, python."); + throw new InvalidOperationException($"Unsupported ingestion language(s): {string.Join(", ", unsupportedLanguages)}. Supported values: csharp, deployment, markdown, python, rust."); } - var parserClient = CreateParserClient(args); + var parserClient = CreateParserClient(args, languages); await using var graphStore = CreateGraphStore(args); var contentStore = CreateContentStore(args); var registryStore = CreateRegistryStore(args); @@ -330,7 +371,8 @@ static async Task RunIngestAsync(string[] args) SummaryReasoningTokenReserve: ReadIntOption(args, "--summary-reasoning-token-reserve", 768), SummaryMaxCompletionTokens: ReadIntOption(args, "--summary-max-completion-tokens", 1536), ProjectId: ReadOption(args, "--project-id"), - SnapshotRetentionGraceSeconds: ReadIntOption(args, "--snapshot-retention-grace-seconds", 86400)); + SnapshotRetentionGraceSeconds: ReadIntOption(args, "--snapshot-retention-grace-seconds", 86400), + PathFilter: pathFilter); var orchestrator = new IngestionOrchestrator( parserClient, @@ -351,7 +393,7 @@ static async Task RunRefreshAsync(string[] args) ReadAnyOption(args, "--root", "--repository-root"), ReadAnyOption(args, "--solution", "--solution-path"), ReadAnyOption(args, "--project", "--project-path"), - ReadOption(args, "--language")); + ReadAnyOption(args, "--languages", "--language")); return await RunIngestAsync(CodeMeshRefreshWorkflow.BuildRefreshIngestArgs(args, target)).ConfigureAwait(false); } @@ -362,13 +404,14 @@ static async Task RunWatchAsync(string[] args, bool json) ReadAnyOption(args, "--root", "--repository-root"), ReadAnyOption(args, "--solution", "--solution-path"), ReadAnyOption(args, "--project", "--project-path"), - ReadOption(args, "--language")); + ReadAnyOption(args, "--languages", "--language")); var ingestArgs = CodeMeshWatchWorkflow.BuildWatchRefreshArgs(args, target); var pollSeconds = Math.Max(1, ReadIntOption(args, "--poll-seconds", 2)); var maxRuns = ReadOptionalIntOption(args, "--max-runs"); var runOnce = HasFlag(args, "--run-once"); var runCount = 0; var hasError = false; + var pathFilter = CreateRepositoryPathFilter(args); if (!json) { @@ -376,7 +419,7 @@ static async Task RunWatchAsync(string[] args, bool json) Console.WriteLine($"Polling every {pollSeconds} second(s). Press Ctrl+C to stop."); } - var snapshot = CodeMeshWatchWorkflow.CreateSnapshot(target.RootPath); + var snapshot = CodeMeshWatchWorkflow.CreateSnapshot(target.RootPath, pathFilter); while (true) { var result = await RunIngestAsync(ingestArgs).ConfigureAwait(false); @@ -392,7 +435,7 @@ static async Task RunWatchAsync(string[] args, bool json) while (true) { await Task.Delay(TimeSpan.FromSeconds(pollSeconds)).ConfigureAwait(false); - var current = CodeMeshWatchWorkflow.CreateSnapshot(target.RootPath); + var current = CodeMeshWatchWorkflow.CreateSnapshot(target.RootPath, pathFilter); if (!CodeMeshWatchWorkflow.HasChanges(snapshot, current)) { continue; @@ -572,6 +615,147 @@ static async Task RunSummaryCoverageAsync(string[] args) .ConfigureAwait(false); } +static async Task RunSummaryQualificationAsync(string[] args, string codeMeshRoot, bool json) +{ + var operation = ReadPositional(args, 2)?.ToLowerInvariant(); + switch (operation) + { + case "run": + { + var suite = await ReadJsonFileAsync(RequiredOption(args, "--suite")).ConfigureAwait(false); + var profile = await ReadJsonFileAsync(RequiredOption(args, "--profile")).ConfigureAwait(false); + var privateOutput = RequiredOption(args, "--private-output"); + var reviewOutput = RequiredOption(args, "--review-output"); + EnsureNewOutputPaths(privateOutput, reviewOutput); + if (string.Equals(profile.EndpointClass, "online", StringComparison.OrdinalIgnoreCase) && + !HasFlag(args, "--authorize-online-source")) + { + throw new InvalidOperationException("Online summary qualification requires --authorize-online-source after source-governance and provider-spend approval."); + } + + var identity = await SummaryQualificationRunner.CaptureSourceIdentityAsync(codeMeshRoot).ConfigureAwait(false); + if (identity.WorkingTreeDirty || string.IsNullOrWhiteSpace(identity.CodeMeshCommit)) + { + throw new InvalidOperationException("Summary qualification requires a clean CodeMesh Git checkout at a recorded commit."); + } + + var (provider, _) = CreateSummaryProvider(args, requireProvider: true); + using var disposableProvider = provider as IDisposable; + var result = await new SummaryQualificationRunner(provider) + .RunAsync(suite, profile, identity) + .ConfigureAwait(false); + await WriteJsonFileAsync(privateOutput, result.PrivateArchive).ConfigureAwait(false); + await WriteJsonFileAsync(reviewOutput, result.ReviewPacket).ConfigureAwait(false); + PrintQualificationRun(result, json); + return result.PrivateArchive.PreflightFailures.Count == 0 ? 0 : 1; + } + + case "bind-retrieval": + { + if (!HasFlag(args, "--confirm-isolated-indexes")) + { + throw new InvalidOperationException("Binding retrieval evidence requires --confirm-isolated-indexes after verifying no-summary and candidate stores are isolated."); + } + + var archive = await ReadJsonFileAsync(RequiredOption(args, "--archive")).ConfigureAwait(false); + var baselinePath = RequiredOption(args, "--baseline-live"); + var candidatePath = RequiredOption(args, "--candidate-live"); + var baselineJson = await File.ReadAllTextAsync(baselinePath).ConfigureAwait(false); + var candidateJson = await File.ReadAllTextAsync(candidatePath).ConfigureAwait(false); + var assessment = await ReadJsonFileAsync(RequiredOption(args, "--assessment")).ConfigureAwait(false); + if (string.Equals(CodeMeshHash.Sha256Hex(baselineJson), CodeMeshHash.Sha256Hex(candidateJson), StringComparison.Ordinal)) + { + throw new InvalidOperationException("No-summary and candidate retrieval evidence must be distinct live reports."); + } + + var baseline = ReadLiveRetrievalMetrics(baselineJson, out var baselineSuite, out var baselineRepository, out var baselineCommit, out var baselineFailures); + var candidate = ReadLiveRetrievalMetrics(candidateJson, out var candidateSuite, out var candidateRepository, out var candidateCommit, out var candidateFailures); + if (!string.Equals(baselineSuite, candidateSuite, StringComparison.Ordinal) || + !string.Equals(baselineRepository, candidateRepository, StringComparison.Ordinal) || + !string.Equals(baselineCommit, candidateCommit, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Live reports do not use the same suite and repository identity."); + } + + if (!archive.CorpusRepositories.Any(repository => + string.Equals(repository.RepositoryId, baselineRepository, StringComparison.Ordinal) && + string.Equals(repository.RepositoryCommit, baselineCommit, StringComparison.Ordinal))) + { + throw new InvalidOperationException("Live reports do not target a repository recorded in the qualification corpus."); + } + + var baselineReportHash = CodeMeshHash.Sha256Hex(baselineJson); + var candidateReportHash = CodeMeshHash.Sha256Hex(candidateJson); + if (!string.Equals(assessment.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(assessment.Kind, SummaryQualificationConstants.RetrievalAssessmentKind, StringComparison.Ordinal) || + !string.Equals(assessment.RunId, archive.RunId, StringComparison.Ordinal) || + !string.Equals(assessment.BaselineReportHash, baselineReportHash, StringComparison.Ordinal) || + !string.Equals(assessment.CandidateReportHash, candidateReportHash, StringComparison.Ordinal) || + !assessment.Reviewed) + { + throw new InvalidOperationException("Retrieval assessment is incomplete or does not match the qualification run and live reports."); + } + + var evidence = new SummaryQualificationRetrievalEvidence( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.RetrievalEvidenceKind, + archive.RunId, + archive.SuiteHash, + archive.PromptHash, + archive.CorpusRepositoryIdentityHash, + CodeMeshHash.Sha256Hex(RequiredOption(args, "--ranking-identity")), + baselineSuite, + baselineRepository, + baselineCommit, + baselineReportHash, + candidateReportHash, + baseline, + candidate, + assessment, + baselineFailures.Concat(candidateFailures).Distinct(StringComparer.Ordinal).ToArray()); + await WriteJsonFileAsync(RequiredOption(args, "--output"), evidence).ConfigureAwait(false); + PrintQualificationObject(evidence, json, $"Bound retrieval evidence for run {archive.RunId}."); + return 0; + } + + case "compile": + { + var suite = await ReadJsonFileAsync(RequiredOption(args, "--suite")).ConfigureAwait(false); + var archive = await ReadJsonFileAsync(RequiredOption(args, "--archive")).ConfigureAwait(false); + var reviewPaths = ReadRepeatedOptions(args, "--review"); + if (reviewPaths.Count < 2) + { + throw new InvalidOperationException("Summary qualification compilation requires at least two --review files."); + } + + var reviews = new List(); + foreach (var reviewPath in reviewPaths) + { + reviews.Add(await ReadJsonFileAsync(reviewPath).ConfigureAwait(false)); + } + + var retrieval = await ReadJsonFileAsync(RequiredOption(args, "--retrieval")).ConfigureAwait(false); + var deployment = await ReadJsonFileAsync(RequiredOption(args, "--deployment")).ConfigureAwait(false); + var report = SummaryQualificationCompiler.Compile(suite, archive, reviews, retrieval, deployment); + await WriteJsonFileAsync(RequiredOption(args, "--output"), report).ConfigureAwait(false); + PrintQualificationObject(report, json, $"Summary qualification outcome: {report.Outcome}"); + return report.Outcome is "qualified" or "conditionally-qualified" ? 0 : 1; + } + + case "compare": + { + var candidate = await ReadJsonFileAsync(RequiredOption(args, "--candidate")).ConfigureAwait(false); + var reference = await ReadJsonFileAsync(RequiredOption(args, "--reference")).ConfigureAwait(false); + var comparison = SummaryQualificationCompiler.Compare(candidate, reference); + PrintQualificationObject(comparison, json, string.Join(Environment.NewLine, comparison.Select(pair => $"{pair.Key}: {pair.Value:+0.###;-0.###;0}"))); + return 0; + } + + default: + throw new InvalidOperationException("Usage: codemesh summaries qualify "); + } +} + static async Task RunRepositoryDeleteAsync(string[] args) { var repositoryRef = ReadAnyOption(args, "--repository-id", "--repo") @@ -648,14 +832,22 @@ static async Task RunRepositoryDeleteAsync(string[] arg } } -static IParserClient CreateParserClient(string[] args) +static IParserClient CreateParserClient( + string[] args, + IReadOnlyList? selectedLanguages = null) { - var language = ReadOption(args, "--language") ?? "csharp"; + var languages = selectedLanguages ?? ReadLanguageSet(args); var parserUrl = ReadOption(args, "--parser-url") ?? Environment.GetEnvironmentVariable("CODEMESH_CSHARP_PARSER_URL"); if (!string.IsNullOrWhiteSpace(parserUrl)) { + if (languages.Count != 1) + { + throw new InvalidOperationException( + "--parser-url cannot be used with multi-language ingestion; configure local parser clients for one atomic generation."); + } + return new HttpParserClient(new HttpClient { BaseAddress = new Uri(EnsureTrailingSlash(parserUrl), UriKind.Absolute), @@ -663,31 +855,52 @@ static IParserClient CreateParserClient(string[] args) }); } + var clients = languages.ToDictionary( + language => language, + CreateLocalParserClient, + StringComparer.Ordinal); + return clients.Count == 1 + ? clients.Values.Single() + : new CompositeParserClient(clients); +} + +static IParserClient CreateLocalParserClient(string language) +{ return NormalizeLanguage(language) switch { "csharp" => new LocalCSharpParserClient(new CSharpParseService()), "deployment" => new LocalDeploymentParserClient(new DeploymentParseService()), "markdown" => new LocalMarkdownParserClient(new MarkdownParseService()), "python" => new LocalPythonParserClient(new PythonParseService()), - _ => throw new InvalidOperationException($"Unsupported ingestion language: {language}. Supported values: csharp, deployment, markdown, python.") + "rust" => new LocalRustParserClient(new RustParseService()), + _ => throw new InvalidOperationException($"Unsupported ingestion language: {language}. Supported values: csharp, deployment, markdown, python, rust.") }; } +static IReadOnlyList ReadLanguageSet(string[] args) +{ + return CompositeParserClient.ParseLanguages( + ReadAnyOption(args, "--languages", "--language") ?? "csharp"); +} + +static RepositoryPathFilter CreateRepositoryPathFilter(string[] args) +{ + var filter = new RepositoryPathFilter( + ReadRepeatedOptions(args, "--allow-path", "--include-path"), + ReadRepeatedOptions(args, "--deny-path", "--exclude-path"), + ExcludeKnownSecretFiles: !HasFlag(args, "--include-known-secret-files")); + _ = RepositoryPathPolicy.StableFingerprint(filter); + return filter; +} + static bool IsSupportedLanguage(string language) { - return NormalizeLanguage(language) is "csharp" or "deployment" or "markdown" or "python"; + return NormalizeLanguage(language) is "csharp" or "deployment" or "markdown" or "python" or "rust"; } static string NormalizeLanguage(string language) { - return language.Trim().ToLowerInvariant() switch - { - "cs" or "c#" => "csharp", - "deploy" or "docker" or "compose" => "deployment", - "md" => "markdown", - "py" => "python", - var value => value - }; + return CompositeParserClient.NormalizeLanguage(language); } static Neo4jGraphStore CreateGraphStore(string[] args) @@ -782,9 +995,11 @@ static QdrantVectorStore CreateVectorStore(string[] args) }; } -static (ICodeSummaryProvider Provider, bool IncludeSummaries) CreateSummaryProvider(string[] args) +static (ICodeSummaryProvider Provider, bool IncludeSummaries) CreateSummaryProvider( + string[] args, + bool requireProvider = false) { - var includeSummaries = HasFlag(args, "--include-summaries") || HasFlag(args, "--force-summaries"); + var includeSummaries = requireProvider || HasFlag(args, "--include-summaries") || HasFlag(args, "--force-summaries"); if (!includeSummaries) { return (new NoCodeSummaryProvider(), false); @@ -888,6 +1103,142 @@ static int ReadIntOption(string[] args, string name, int defaultValue) return int.TryParse(value, out var parsed) ? parsed : defaultValue; } +static string RequiredOption(string[] args, string name) +{ + return ReadOption(args, name) + ?? throw new InvalidOperationException($"Option {name} is required."); +} + +static async Task ReadJsonFileAsync(string path) +{ + await using var stream = File.OpenRead(Path.GetFullPath(path)); + return await JsonSerializer.DeserializeAsync(stream, QualificationJsonOptions()).ConfigureAwait(false) + ?? throw new InvalidOperationException($"JSON file did not contain a {typeof(T).Name}: {path}"); +} + +static async Task WriteJsonFileAsync(string path, T value) +{ + var fullPath = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(fullPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + await using var stream = new FileStream(fullPath, FileMode.CreateNew, FileAccess.Write, FileShare.None); + await JsonSerializer.SerializeAsync(stream, value, QualificationJsonOptions()).ConfigureAwait(false); + await stream.FlushAsync().ConfigureAwait(false); +} + +static void EnsureNewOutputPaths(params string[] paths) +{ + var fullPaths = paths.Select(Path.GetFullPath).ToArray(); + if (fullPaths.Distinct(StringComparer.OrdinalIgnoreCase).Count() != fullPaths.Length) + { + throw new InvalidOperationException("Qualification outputs must use distinct paths."); + } + + var existing = fullPaths.FirstOrDefault(File.Exists); + if (existing is not null) + { + throw new InvalidOperationException($"Qualification output already exists: {existing}"); + } +} + +static JsonSerializerOptions QualificationJsonOptions() +{ + return new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = true, + PropertyNameCaseInsensitive = true, + RespectNullableAnnotations = true, + RespectRequiredConstructorParameters = true, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } + }; +} + +static SummaryRetrievalMetrics ReadLiveRetrievalMetrics( + string json, + out string suite, + out string repositoryId, + out string indexedCommit, + out IReadOnlyList failures) +{ + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + if (!root.TryGetProperty("kind", out var kind) || !string.Equals(kind.GetString(), "live-report", StringComparison.Ordinal)) + { + throw new InvalidOperationException("Retrieval evidence must be an Agent Access live-report."); + } + + suite = root.GetProperty("suite").GetString() ?? string.Empty; + repositoryId = root.GetProperty("repository_id").GetString() ?? string.Empty; + indexedCommit = root.TryGetProperty("metadata", out var metadata) && + metadata.TryGetProperty("indexed_commit", out var indexedCommitElement) + ? indexedCommitElement.GetString() ?? string.Empty + : string.Empty; + var summary = root.GetProperty("summary"); + failures = root.TryGetProperty("failures", out var failureArray) + ? failureArray.EnumerateArray().Select(item => item.GetString() ?? string.Empty).Where(value => value.Length > 0).ToArray() + : []; + var cases = root.TryGetProperty("cases", out var caseArray) + ? caseArray.EnumerateArray().ToArray() + : []; + var precisionCases = cases.Where(item => item.TryGetProperty("precision_at_k", out _)).ToArray(); + var meanPrecision = precisionCases.Length == 0 + ? 0 + : precisionCases.Average(item => item.GetProperty("precision_at_k").GetDouble()); + var stabilityCases = cases.Where(item => item.TryGetProperty("ranking_stability", out _)).ToArray(); + var meanStability = stabilityCases.Length == 0 + ? 0 + : stabilityCases.Average(item => item.GetProperty("ranking_stability").GetDouble()); + + return new SummaryRetrievalMetrics( + summary.GetProperty("mean_recall_at_k").GetDouble(), + summary.GetProperty("mean_mrr").GetDouble(), + summary.GetProperty("mean_ndcg_at_k").GetDouble(), + meanPrecision, + meanStability, + summary.GetProperty("secret_leaks").GetInt32(), + root.GetProperty("passed").GetBoolean(), + root.GetProperty("comparable").GetBoolean()); +} + +static void PrintQualificationRun(SummaryQualificationRunResult result, bool json) +{ + if (json) + { + Console.WriteLine(JsonSerializer.Serialize(new + { + result.PrivateArchive.RunId, + result.PrivateArchive.SuiteName, + result.PrivateArchive.Profile.Provider, + result.PrivateArchive.Profile.Model, + WarmupCount = result.PrivateArchive.Samples.Count(sample => string.Equals(sample.Phase, "warmup", StringComparison.Ordinal)), + MeasuredSampleCount = result.PrivateArchive.Samples.Count(sample => string.Equals(sample.Phase, "measured", StringComparison.Ordinal)), + MeasuredCompletedCount = result.PrivateArchive.Samples.Count(sample => string.Equals(sample.Phase, "measured", StringComparison.Ordinal) && sample.Completed), + result.PrivateArchive.PreflightFailures + }, QualificationJsonOptions())); + return; + } + + Console.WriteLine($"Summary qualification run: {result.PrivateArchive.RunId}"); + Console.WriteLine($"Candidate: {result.ReviewPacket.CandidateLabel}"); + Console.WriteLine($"Warm-up: {(result.PrivateArchive.Samples.Single(sample => string.Equals(sample.Phase, "warmup", StringComparison.Ordinal)).Completed ? "passed" : "failed")}"); + Console.WriteLine($"Measured samples: {result.PrivateArchive.Samples.Count(sample => string.Equals(sample.Phase, "measured", StringComparison.Ordinal))}, completed: {result.PrivateArchive.Samples.Count(sample => string.Equals(sample.Phase, "measured", StringComparison.Ordinal) && sample.Completed)}"); + foreach (var failure in result.PrivateArchive.PreflightFailures) + { + Console.WriteLine($"Failure: {failure}"); + } + + Console.WriteLine("Private archive and blinded review packet were written to the requested paths."); +} + +static void PrintQualificationObject(T value, bool json, string text) +{ + Console.WriteLine(json ? JsonSerializer.Serialize(value, QualificationJsonOptions()) : text); +} + static int? ReadOptionalIntOption(string[] args, string name) { var value = ReadOption(args, name); @@ -1333,17 +1684,22 @@ static string Marker(ComponentState state) static void PrintHelp() { Console.WriteLine("CodeMesh CLI"); + Console.WriteLine(" --version Print the source version without opening stores."); Console.WriteLine(); Console.WriteLine("Usage:"); Console.WriteLine(" codemesh doctor [--root ] [--json]"); Console.WriteLine(" codemesh status [--root ] [--json]"); - Console.WriteLine(" codemesh ingest [--root ] [--project-id ] [--solution ] [--project ] [--language ] [--dry-run] [--skip-embeddings] [--force-embeddings] [--include-summaries] [--json]"); - Console.WriteLine(" codemesh refresh [--root ] [--solution ] [--project ] [--language ] [--skip-embeddings] [--force-embeddings] [--include-summaries] [--json]"); - Console.WriteLine(" codemesh watch [--root ] [--solution ] [--project ] [--language ] [--poll-seconds ] [--max-runs ] [--run-once] [--dry-run] [--skip-embeddings] [--force-embeddings] [--include-summaries] [--json]"); + Console.WriteLine(" codemesh ingest [--root ] [--project-id ] [--solution ] [--project ] [--language | --languages ] [--allow-path ] [--deny-path ] [--dry-run] [--skip-embeddings] [--force-embeddings] [--include-summaries] [--json]"); + Console.WriteLine(" codemesh refresh [--root ] [--solution ] [--project ] [--language | --languages ] [--allow-path ] [--deny-path ] [--skip-embeddings] [--force-embeddings] [--include-summaries] [--json]"); + Console.WriteLine(" codemesh watch [--root ] [--solution ] [--project ] [--language | --languages ] [--allow-path ] [--deny-path ] [--poll-seconds ] [--max-runs ] [--run-once] [--dry-run] [--skip-embeddings] [--force-embeddings] [--include-summaries] [--json]"); Console.WriteLine(" codemesh self ingest [--dry-run] [--skip-embeddings] [--include-embeddings] [--include-summaries] [--json]"); Console.WriteLine(" codemesh self smoke [--agent-access-url ] [--skip-embeddings] [--query ] [--json]"); Console.WriteLine(" codemesh embeddings verify [--embedding-provider ] [--embedding-model ] [--json]"); Console.WriteLine(" codemesh summaries coverage [--json]"); + Console.WriteLine(" codemesh summaries qualify run --suite --profile --private-output --review-output --summary-provider --summary-model [--authorize-online-source] [--json]"); + Console.WriteLine(" codemesh summaries qualify bind-retrieval --archive --baseline-live --candidate-live --assessment --ranking-identity --confirm-isolated-indexes --output [--json]"); + Console.WriteLine(" codemesh summaries qualify compile --suite --archive --review --review --retrieval --deployment --output [--json]"); + Console.WriteLine(" codemesh summaries qualify compare --candidate --reference [--json]"); Console.WriteLine(" codemesh repos list [--limit ] [--json]"); Console.WriteLine(" codemesh repo show [--json]"); Console.WriteLine(" codemesh repo delete [--json]"); @@ -1358,6 +1714,8 @@ static void PrintHelp() Console.WriteLine(" --include-embeddings --embedding-provider --ollama-url --lm-studio-url --embedding-model "); Console.WriteLine(" --embedding-batch-size --embedding-max-chars "); Console.WriteLine(" --include-summaries --force-summaries --summary-provider --summary-model "); + Console.WriteLine(" Repeat --allow-path/--deny-path for repository-relative globs; deny wins. Known secret files are excluded by default."); + Console.WriteLine(" --include-known-secret-files explicitly disables only the default secret-file exclusions; generated/cache exclusions remain."); Console.WriteLine(" --openai-url (OpenAI summaries only; authentication uses OPENAI_API_KEY)"); Console.WriteLine(" --summary-reasoning-effort (OpenAI only)"); Console.WriteLine(" --summary-max-input-chars --summary-reasoning-token-reserve --summary-max-completion-tokens "); @@ -1415,6 +1773,19 @@ public Task ParseAsync(ParseRequest request, CancellationToken canc } } +sealed class LocalRustParserClient(RustParseService parser) : IParserClient +{ + public Task GetCapabilityAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(RustParseService.Capability); + } + + public Task ParseAsync(ParseRequest request, CancellationToken cancellationToken = default) + { + return parser.ParseAsync(request, cancellationToken); + } +} + sealed record SelfSmokeResult( bool Success, string RepositoryId, diff --git a/src/CodeMesh.Cli/packages.lock.json b/src/CodeMesh.Cli/packages.lock.json index cca91eb..c35a6ae 100644 --- a/src/CodeMesh.Cli/packages.lock.json +++ b/src/CodeMesh.Cli/packages.lock.json @@ -261,6 +261,12 @@ "CodeMesh.Domain": "[1.0.0, )" } }, + "codemesh.parser.rust": { + "type": "Project", + "dependencies": { + "CodeMesh.Domain": "[1.0.0, )" + } + }, "codemesh.storage": { "type": "Project", "dependencies": { @@ -272,4 +278,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/CodeMesh.Control/Configuration/CodeMeshWatchWorkflow.cs b/src/CodeMesh.Control/Configuration/CodeMeshWatchWorkflow.cs index 9aa7840..988279b 100644 --- a/src/CodeMesh.Control/Configuration/CodeMeshWatchWorkflow.cs +++ b/src/CodeMesh.Control/Configuration/CodeMeshWatchWorkflow.cs @@ -1,3 +1,4 @@ +using CodeMesh.Domain.Contracts; using CodeMesh.Domain.Utilities; namespace CodeMesh.Control.Configuration; @@ -42,10 +43,12 @@ public static string[] BuildWatchRefreshArgs(string[] args, CodeMeshRefreshTarge return CodeMeshRefreshWorkflow.BuildRefreshIngestArgs(refreshArgs.ToArray(), target); } - public static CodeMeshWatchSnapshot CreateSnapshot(string rootPath) + public static CodeMeshWatchSnapshot CreateSnapshot( + string rootPath, + RepositoryPathFilter? pathFilter = null) { var files = new SortedDictionary(StringComparer.OrdinalIgnoreCase); - foreach (var file in EnumerateFiles(rootPath)) + foreach (var file in EnumerateFiles(rootPath, pathFilter)) { var info = new FileInfo(file); var relativePath = Path.GetRelativePath(rootPath, file); @@ -74,28 +77,11 @@ public static bool HasChanges(CodeMeshWatchSnapshot previous, CodeMeshWatchSnaps return false; } - private static IEnumerable EnumerateFiles(string rootPath) + private static IEnumerable EnumerateFiles( + string rootPath, + RepositoryPathFilter? pathFilter) { - var pending = new Stack(); - pending.Push(rootPath); - - while (pending.Count > 0) - { - var directory = pending.Pop(); - - foreach (var child in Directory.EnumerateDirectories(directory)) - { - if (!RepositoryPathPolicy.IsIgnoredPath(rootPath, child)) - { - pending.Push(child); - } - } - - foreach (var file in Directory.EnumerateFiles(directory)) - { - yield return file; - } - } + return RepositoryPathPolicy.EnumerateFiles(rootPath, "*", pathFilter); } private static bool IsWatchOptionWithValue(string arg) diff --git a/src/CodeMesh.Domain/Contracts/IngestionContracts.cs b/src/CodeMesh.Domain/Contracts/IngestionContracts.cs index 9026954..55a0c6d 100644 --- a/src/CodeMesh.Domain/Contracts/IngestionContracts.cs +++ b/src/CodeMesh.Domain/Contracts/IngestionContracts.cs @@ -27,7 +27,8 @@ public sealed record IngestionRequest( int SummaryReasoningTokenReserve = 768, int SummaryMaxCompletionTokens = 1536, string? ProjectId = null, - int SnapshotRetentionGraceSeconds = 86400); + int SnapshotRetentionGraceSeconds = 86400, + RepositoryPathFilter? PathFilter = null); public sealed record IngestionRun( string RunId, diff --git a/src/CodeMesh.Domain/Contracts/ParserContracts.cs b/src/CodeMesh.Domain/Contracts/ParserContracts.cs index 5b6e4ae..4b835eb 100644 --- a/src/CodeMesh.Domain/Contracts/ParserContracts.cs +++ b/src/CodeMesh.Domain/Contracts/ParserContracts.cs @@ -7,7 +7,13 @@ public sealed record ParseRequest( string? SolutionPath, string? ProjectPath, string Language, - IReadOnlyDictionary Options); + IReadOnlyDictionary Options, + RepositoryPathFilter? PathFilter = null); + +public sealed record RepositoryPathFilter( + IReadOnlyList? AllowPatterns = null, + IReadOnlyList? DenyPatterns = null, + bool ExcludeKnownSecretFiles = true); public sealed record ParserCapability( string WorkerName, diff --git a/src/CodeMesh.Domain/Utilities/RepositoryPathPolicy.cs b/src/CodeMesh.Domain/Utilities/RepositoryPathPolicy.cs index 5ea8bd7..aeb63d2 100644 --- a/src/CodeMesh.Domain/Utilities/RepositoryPathPolicy.cs +++ b/src/CodeMesh.Domain/Utilities/RepositoryPathPolicy.cs @@ -1,7 +1,17 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.RegularExpressions; +using CodeMesh.Domain.Contracts; + namespace CodeMesh.Domain.Utilities; public static class RepositoryPathPolicy { + public const string Version = "repository-path-policy-v2"; + + private static readonly RepositoryPathFilter DefaultFilter = new(); + private static readonly ConcurrentDictionary GlobExpressions = new(StringComparer.Ordinal); + private static readonly HashSet IgnoredDirectoryNames = new(StringComparer.OrdinalIgnoreCase) { ".codemesh-live-summary", @@ -25,19 +35,151 @@ public static class RepositoryPathPolicy "dist", "node_modules", "obj", + "target", "venv" }; - public static bool IsIgnoredPath(string repositoryRoot, string candidatePath) + private static readonly HashSet KnownSecretFileNames = new(StringComparer.OrdinalIgnoreCase) + { + ".env", + ".envrc", + ".netrc", + ".npmrc", + ".pypirc", + "application_default_credentials.json", + "accesstokens.json", + "credentials", + "credentials.json", + "credentials.toml", + "credentials.xml", + "credentials.yaml", + "credentials.yml", + "id_dsa", + "id_ecdsa", + "id_ed25519", + "id_rsa", + "kubeconfig", + "secrets.json", + "secrets.toml", + "secrets.xml", + "secrets.yaml", + "secrets.yml" + }; + + private static readonly HashSet KnownSecretExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".jks", + ".kdbx", + ".key", + ".keystore", + ".p12", + ".pem", + ".pfx", + ".snk", + ".tfstate" + }; + + private static readonly HashSet SafeEnvironmentTemplateNames = new(StringComparer.OrdinalIgnoreCase) + { + ".env.defaults", + ".env.example", + ".env.sample", + ".env.template" + }; + + public static bool IsIgnoredPath( + string repositoryRoot, + string candidatePath, + RepositoryPathFilter? filter = null) + { + var root = Path.GetFullPath(repositoryRoot); + var candidate = Path.GetFullPath(candidatePath); + var relativePath = Path.GetRelativePath(root, candidate); + return IsIgnoredRelativePath(relativePath, filter) || + ContainsReparsePoint(root, relativePath); + } + + public static bool IsIgnoredRelativePath( + string relativePath, + RepositoryPathFilter? filter = null) + { + var normalizedPath = NormalizeRelativePath(relativePath); + if (normalizedPath.Length == 0 || + Path.IsPathRooted(normalizedPath) || + IsOutsideRepository(normalizedPath)) + { + return true; + } + + var segments = normalizedPath.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Any(IsIgnoredSegment)) + { + return true; + } + + var effectiveFilter = filter ?? DefaultFilter; + if (effectiveFilter.ExcludeKnownSecretFiles && IsKnownSecretFile(normalizedPath)) + { + return true; + } + + var denyPatterns = NormalizePatterns(effectiveFilter.DenyPatterns, nameof(effectiveFilter.DenyPatterns)); + if (denyPatterns.Any(pattern => MatchesGlob(normalizedPath, pattern))) + { + return true; + } + + var allowPatterns = NormalizePatterns(effectiveFilter.AllowPatterns, nameof(effectiveFilter.AllowPatterns)); + return allowPatterns.Count > 0 && + !allowPatterns.Any(pattern => MatchesGlob(normalizedPath, pattern)); + } + + public static IEnumerable EnumerateFiles( + string repositoryRoot, + string searchPattern, + RepositoryPathFilter? filter = null) { - return IsIgnoredRelativePath(Path.GetRelativePath(repositoryRoot, candidatePath)); + ArgumentException.ThrowIfNullOrWhiteSpace(repositoryRoot); + ArgumentException.ThrowIfNullOrWhiteSpace(searchPattern); + + var root = Path.GetFullPath(repositoryRoot); + var pending = new Stack(); + pending.Push(root); + + while (pending.Count > 0) + { + var directory = pending.Pop(); + foreach (var child in Directory.EnumerateDirectories(directory)) + { + // Apply immutable generated/cache and link containment here, + // but defer allow/deny filters until a concrete file path is known. + if (!IsIgnoredPath(root, child)) + { + pending.Push(child); + } + } + + foreach (var file in Directory.EnumerateFiles(directory, searchPattern, SearchOption.TopDirectoryOnly)) + { + if (!IsIgnoredPath(root, file, filter)) + { + yield return file; + } + } + } } - public static bool IsIgnoredRelativePath(string relativePath) + public static string StableFingerprint(RepositoryPathFilter? filter) { - return relativePath - .Split(['/', '\\'], StringSplitOptions.RemoveEmptyEntries) - .Any(IsIgnoredSegment); + var effectiveFilter = filter ?? DefaultFilter; + var allowPatterns = NormalizePatterns(effectiveFilter.AllowPatterns, nameof(effectiveFilter.AllowPatterns)); + var denyPatterns = NormalizePatterns(effectiveFilter.DenyPatterns, nameof(effectiveFilter.DenyPatterns)); + return string.Join( + '\u001e', + Version, + effectiveFilter.ExcludeKnownSecretFiles ? "exclude-known-secrets" : "include-known-secrets", + $"allow={string.Join('\u001d', allowPatterns.OrderBy(value => value, StringComparer.OrdinalIgnoreCase))}", + $"deny={string.Join('\u001d', denyPatterns.OrderBy(value => value, StringComparer.OrdinalIgnoreCase))}"); } private static bool IsIgnoredSegment(string segment) @@ -45,4 +187,190 @@ private static bool IsIgnoredSegment(string segment) return IgnoredDirectoryNames.Contains(segment) || segment.EndsWith(".egg-info", StringComparison.OrdinalIgnoreCase); } + + private static bool ContainsReparsePoint(string repositoryRoot, string relativePath) + { + var current = repositoryRoot; + foreach (var segment in relativePath.Split( + ['/', '\\'], + StringSplitOptions.RemoveEmptyEntries)) + { + current = Path.Combine(current, segment); + try + { + if ((File.GetAttributes(current) & FileAttributes.ReparsePoint) != 0) + { + return true; + } + } + catch (Exception exception) when ( + exception is FileNotFoundException or + DirectoryNotFoundException or + UnauthorizedAccessException or + IOException) + { + return true; + } + } + + return false; + } + + private static bool IsKnownSecretFile(string normalizedPath) + { + var fileName = normalizedPath[(normalizedPath.LastIndexOf('/') + 1)..]; + if (IsSafeEnvironmentTemplate(fileName)) + { + return false; + } + + if (fileName.StartsWith(".env.", StringComparison.OrdinalIgnoreCase) || + IsServiceAccountFile(fileName) || + KnownSecretFileNames.Contains(fileName) || + KnownSecretExtensions.Contains(Path.GetExtension(fileName)) || + fileName.EndsWith(".tfstate.backup", StringComparison.OrdinalIgnoreCase) || + IsPrivateKeyName(fileName)) + { + return true; + } + + return normalizedPath.Equals(".aws/credentials", StringComparison.OrdinalIgnoreCase) || + normalizedPath.Equals(".docker/config.json", StringComparison.OrdinalIgnoreCase) || + normalizedPath.Equals( + ".config/gcloud/application_default_credentials.json", + StringComparison.OrdinalIgnoreCase); + } + + private static bool IsPrivateKeyName(string fileName) + { + return fileName.StartsWith("id_rsa_", StringComparison.OrdinalIgnoreCase) || + fileName.StartsWith("id_dsa_", StringComparison.OrdinalIgnoreCase) || + fileName.StartsWith("id_ecdsa_", StringComparison.OrdinalIgnoreCase) || + fileName.StartsWith("id_ed25519_", StringComparison.OrdinalIgnoreCase); + } + + private static bool IsServiceAccountFile(string fileName) + { + return (fileName.StartsWith("service-account", StringComparison.OrdinalIgnoreCase) || + fileName.StartsWith("service_account", StringComparison.OrdinalIgnoreCase)) && + string.Equals(Path.GetExtension(fileName), ".json", StringComparison.OrdinalIgnoreCase); + } + + private static bool IsSafeEnvironmentTemplate(string fileName) + { + return SafeEnvironmentTemplateNames.Contains(fileName) || + fileName.StartsWith(".env.", StringComparison.OrdinalIgnoreCase) && + (fileName.EndsWith(".defaults", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".example", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".sample", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".template", StringComparison.OrdinalIgnoreCase)); + } + + private static IReadOnlyList NormalizePatterns( + IReadOnlyList? patterns, + string parameterName) + { + if (patterns is null || patterns.Count == 0) + { + return Array.Empty(); + } + + return patterns + .Select(pattern => NormalizePattern(pattern, parameterName)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string NormalizePattern(string pattern, string parameterName) + { + if (string.IsNullOrWhiteSpace(pattern)) + { + throw new ArgumentException("Repository path patterns must not be empty.", parameterName); + } + + var normalized = pattern.Trim().Replace('\\', '/'); + while (normalized.StartsWith("./", StringComparison.Ordinal)) + { + normalized = normalized[2..]; + } + + var segments = normalized.Split('/'); + if (normalized.Any(char.IsControl) || + normalized.Contains("//", StringComparison.Ordinal) || + normalized.StartsWith("/", StringComparison.Ordinal) || + Regex.IsMatch(normalized, "^[a-z]:/", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant) || + Path.IsPathRooted(normalized) || + segments.Contains(".", StringComparer.Ordinal) || + segments.Contains("..", StringComparer.Ordinal)) + { + throw new ArgumentException( + $"Repository path pattern must be repository-relative and must not traverse parents: {pattern}", + parameterName); + } + + if (normalized.EndsWith("/", StringComparison.Ordinal)) + { + normalized += "**"; + } + + return normalized.ToLowerInvariant(); + } + + private static bool MatchesGlob(string normalizedPath, string normalizedPattern) + { + var regex = GlobExpressions.GetOrAdd(normalizedPattern, CreateGlobExpression); + return regex.IsMatch(normalizedPath); + } + + private static Regex CreateGlobExpression(string normalizedPattern) + { + var expression = new StringBuilder("^"); + for (var index = 0; index < normalizedPattern.Length; index++) + { + var character = normalizedPattern[index]; + if (character == '*' && index + 1 < normalizedPattern.Length && normalizedPattern[index + 1] == '*') + { + if (index + 2 < normalizedPattern.Length && normalizedPattern[index + 2] == '/') + { + expression.Append("(?:.*/)?"); + index += 2; + } + else + { + expression.Append(".*"); + index++; + } + continue; + } + + expression.Append(character switch + { + '*' => "[^/]*", + '?' => "[^/]", + _ => Regex.Escape(character.ToString()) + }); + } + + expression.Append('$'); + return new Regex( + expression.ToString(), + RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, + TimeSpan.FromSeconds(1)); + } + + private static string NormalizeRelativePath(string relativePath) + { + var normalized = relativePath.Trim().Replace('\\', '/'); + while (normalized.StartsWith("./", StringComparison.Ordinal)) + { + normalized = normalized[2..]; + } + return normalized; + } + + private static bool IsOutsideRepository(string normalizedPath) + { + return normalizedPath.Equals("..", StringComparison.Ordinal) || + normalizedPath.StartsWith("../", StringComparison.Ordinal); + } } diff --git a/src/CodeMesh.Ingestion/CompositeParserClient.cs b/src/CodeMesh.Ingestion/CompositeParserClient.cs new file mode 100644 index 0000000..92f5e22 --- /dev/null +++ b/src/CodeMesh.Ingestion/CompositeParserClient.cs @@ -0,0 +1,266 @@ +using CodeMesh.Domain.Contracts; +using CodeMesh.Domain.Graph; +using CodeMesh.Domain.Utilities; + +namespace CodeMesh.Ingestion; + +public sealed class CompositeParserClient : IParserClient +{ + private readonly IReadOnlyDictionary _clients; + private readonly IReadOnlyList _languages; + + public CompositeParserClient(IReadOnlyDictionary clients) + { + ArgumentNullException.ThrowIfNull(clients); + _clients = clients.ToDictionary( + item => NormalizeLanguage(item.Key), + item => item.Value, + StringComparer.Ordinal); + _languages = _clients.Keys.OrderBy(value => value, StringComparer.Ordinal).ToArray(); + if (_languages.Count < 2) + { + throw new ArgumentException( + "Composite parsing requires at least two distinct languages.", + nameof(clients)); + } + } + + public async Task GetCapabilityAsync( + CancellationToken cancellationToken = default) + { + var capabilities = await Task.WhenAll(_languages.Select(language => + _clients[language].GetCapabilityAsync(cancellationToken))).ConfigureAwait(false); + return new ParserCapability( + "codemesh-parser-composite", + string.Join(',', _languages), + "1.0.0", + capabilities + .SelectMany(capability => capability.SupportedFileExtensions) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(value => value, StringComparer.OrdinalIgnoreCase) + .ToArray(), + capabilities + .SelectMany(capability => capability.SupportedInputs) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(value => value, StringComparer.OrdinalIgnoreCase) + .ToArray()); + } + + public async Task ParseAsync( + ParseRequest request, + CancellationToken cancellationToken = default) + { + var requestedLanguages = ParseLanguages(request.Language); + if (!_languages.SequenceEqual(requestedLanguages, StringComparer.Ordinal)) + { + throw new InvalidOperationException( + $"Composite parser is configured for '{string.Join(',', _languages)}' " + + $"but the request selected '{string.Join(',', requestedLanguages)}'."); + } + + var results = await Task.WhenAll(requestedLanguages.Select(language => + _clients[language].ParseAsync( + request with { Language = language }, + cancellationToken))).ConfigureAwait(false); + + var nodes = results.SelectMany(result => result.Nodes).ToArray(); + var relationships = results.SelectMany(result => result.Relationships).ToList(); + AddPyO3Relationships(nodes, relationships); + relationships = CoalesceRelationships(relationships); + + EnsureUnique( + nodes, + node => node.Id, + "node"); + EnsureUnique( + relationships, + relationship => relationship.Id, + "relationship"); + + return new ParseResult( + $"codemesh-parser-composite-v1[{string.Join(';', results.Select(result => $"{result.Language}={result.ParserName}"))}]", + string.Join(',', requestedLanguages), + nodes, + relationships, + results + .SelectMany(result => result.Contents) + .GroupBy(content => content.Hash, StringComparer.Ordinal) + .Select(group => group.First()) + .ToArray(), + results.SelectMany(result => result.Diagnostics).ToArray(), + results.Max(result => result.ParsedAt)); + } + + public static IReadOnlyList ParseLanguages(string languageSet) + { + var languages = languageSet + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(NormalizeLanguage) + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToArray(); + if (languages.Length == 0) + { + throw new InvalidOperationException("At least one ingestion language is required."); + } + + return languages; + } + + public static string NormalizeLanguage(string language) + { + return language.Trim().ToLowerInvariant() switch + { + "cs" or "c#" => "csharp", + "deploy" or "docker" or "compose" => "deployment", + "md" => "markdown", + "py" => "python", + "rs" => "rust", + var value => value + }; + } + + private static void EnsureUnique( + IEnumerable values, + Func identity, + string kind) + { + var duplicate = values + .GroupBy(identity, StringComparer.Ordinal) + .FirstOrDefault(group => group.Count() > 1); + if (duplicate is not null) + { + throw new InvalidOperationException( + $"Composite parser results contain duplicate {kind} id '{duplicate.Key}'."); + } + } + + private static List CoalesceRelationships( + IEnumerable relationships) + { + var coalesced = new List(); + foreach (var group in relationships.GroupBy( + relationship => relationship.Id, + StringComparer.Ordinal)) + { + var first = group.First(); + if (group.Any(relationship => + !string.Equals( + relationship.SourceNodeId, + first.SourceNodeId, + StringComparison.Ordinal) || + !string.Equals( + relationship.TargetNodeId, + first.TargetNodeId, + StringComparison.Ordinal) || + relationship.Kind != first.Kind || + !string.Equals( + relationship.Label, + first.Label, + StringComparison.Ordinal))) + { + throw new InvalidOperationException( + $"Composite parser relationship id collision '{group.Key}'."); + } + coalesced.Add(first); + } + return coalesced; + } + + private static void AddPyO3Relationships( + IReadOnlyList nodes, + List relationships) + { + var rustExports = nodes + .Where(node => + string.Equals(node.Language, "rust", StringComparison.Ordinal) && + node.Metadata.ContainsKey("pythonExportName")) + .ToArray(); + if (rustExports.Length == 0) + { + return; + } + + var exportsByName = rustExports + .GroupBy(node => node.Metadata["pythonExportName"], StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.ToArray(), StringComparer.Ordinal); + var pythonImports = nodes + .Where(node => + string.Equals(node.Language, "python", StringComparison.Ordinal) && + node.Metadata.TryGetValue("pythonKind", out var kind) && + string.Equals(kind, "import", StringComparison.Ordinal)) + .ToDictionary(node => node.Id, StringComparer.Ordinal); + + foreach (var import in pythonImports.Values) + { + import.Metadata.TryGetValue("importModule", out var module); + import.Metadata.TryGetValue("importedName", out var importedName); + var moduleName = module?.Split('.').LastOrDefault(); + var targetName = string.IsNullOrWhiteSpace(importedName) ? moduleName : importedName; + AddCrossLanguageRelationship( + import.Id, + targetName, + exportsByName, + CodeRelationshipKind.References, + targetName, + relationships); + } + + foreach (var invocation in relationships + .Where(relationship => + relationship.Kind == CodeRelationshipKind.Invokes && + pythonImports.ContainsKey(relationship.TargetNodeId)) + .ToArray()) + { + AddCrossLanguageRelationship( + invocation.SourceNodeId, + invocation.Label, + exportsByName, + CodeRelationshipKind.Invokes, + invocation.Label, + relationships); + } + } + + private static void AddCrossLanguageRelationship( + string sourceNodeId, + string? exportName, + IReadOnlyDictionary exportsByName, + CodeRelationshipKind kind, + string? label, + List relationships) + { + if (string.IsNullOrWhiteSpace(exportName) || + !exportsByName.TryGetValue(exportName, out var targets) || + targets.Length != 1) + { + return; + } + + var target = targets[0]; + var id = CodeMeshHash.StableId( + "cross-language", + "pyo3", + kind.ToString(), + sourceNodeId, + target.Id, + label ?? string.Empty); + if (relationships.Any(relationship => string.Equals(relationship.Id, id, StringComparison.Ordinal))) + { + return; + } + relationships.Add(new CodeRelationship( + id, + sourceNodeId, + target.Id, + kind, + label, + new Dictionary + { + ["boundary"] = "pyo3", + ["sourceLanguage"] = "python", + ["targetLanguage"] = "rust" + })); + } +} diff --git a/src/CodeMesh.Ingestion/IngestionOrchestrator.cs b/src/CodeMesh.Ingestion/IngestionOrchestrator.cs index fbb3fa8..2f0b1ac 100644 --- a/src/CodeMesh.Ingestion/IngestionOrchestrator.cs +++ b/src/CodeMesh.Ingestion/IngestionOrchestrator.cs @@ -21,6 +21,11 @@ public async Task IngestAsync( IngestionRequest request, CancellationToken cancellationToken = default) { + request = request with + { + Language = string.Join(',', CompositeParserClient.ParseLanguages(request.Language)) + }; + _ = RepositoryPathPolicy.StableFingerprint(request.PathFilter); var startedAt = DateTimeOffset.UtcNow; var repositoryRoot = Path.GetFullPath(request.RepositoryRoot); var git = await RepositoryGitSnapshot @@ -57,10 +62,22 @@ await snapshotRegistry.UpsertCheckoutAsync( request.SolutionPath, request.ProjectPath, request.Language, - request.Options ?? new Dictionary()); + request.Options ?? new Dictionary(), + request.PathFilter); var parseResult = await parserClient.ParseAsync(parseRequest, cancellationToken) .ConfigureAwait(false); + var parseErrors = parseResult.Diagnostics + .Where(diagnostic => string.Equals( + diagnostic.Severity, + "error", + StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (parseErrors.Length > 0) + { + throw new InvalidOperationException( + $"Parser failed before snapshot publication: {string.Join("; ", parseErrors.Select(error => $"{error.Code}: {error.Message}"))}"); + } var redaction = ContentRedactor.Redact(parseResult.Contents); var diagnostics = parseResult.Diagnostics.ToList(); diff --git a/src/CodeMesh.Ingestion/SnapshotIdentity.cs b/src/CodeMesh.Ingestion/SnapshotIdentity.cs index 5233aeb..6ee03cf 100644 --- a/src/CodeMesh.Ingestion/SnapshotIdentity.cs +++ b/src/CodeMesh.Ingestion/SnapshotIdentity.cs @@ -35,11 +35,13 @@ public static ComputedSnapshotIdentity Compute( request.Language, NormalizeScopePath(request.RepositoryRoot, request.SolutionPath), NormalizeScopePath(request.RepositoryRoot, request.ProjectPath), - StableOptions(request.Options)); + StableOptions(request.Options), + RepositoryPathPolicy.StableFingerprint(request.PathFilter)); var parserProfileFingerprint = CodeMeshHash.StableId( parseResult.ParserName, parseResult.Language, - RedactionProfileVersion); + RedactionProfileVersion, + RepositoryPathPolicy.Version); var snapshotId = $"snp_{CodeMeshHash.StableId( IdentityFormatVersion, projectId, diff --git a/src/CodeMesh.Ingestion/Summary/LmStudioCodeSummaryProvider.cs b/src/CodeMesh.Ingestion/Summary/LmStudioCodeSummaryProvider.cs index 3ccf99e..ad5ac9d 100644 --- a/src/CodeMesh.Ingestion/Summary/LmStudioCodeSummaryProvider.cs +++ b/src/CodeMesh.Ingestion/Summary/LmStudioCodeSummaryProvider.cs @@ -37,6 +37,8 @@ public LmStudioCodeSummaryProvider( public string Model { get; } + public string GenerationConfigurationFingerprint => "temperature:0.1;response-format:text"; + public async Task SummarizeAsync( CodeSummaryRequest request, CancellationToken cancellationToken = default) @@ -67,7 +69,22 @@ public async Task SummarizeAsync( .ConfigureAwait(false); var content = result?.Choices.FirstOrDefault()?.Message.Content; - return SummaryPrompt.ParseResult(content); + var summary = SummaryPrompt.ParseResult(content); + if (result?.Usage is { } usage) + { + summary = summary with + { + Usage = new CodeSummaryUsage( + usage.PromptTokens, + 0, + 0, + usage.CompletionTokens, + 0, + usage.TotalTokens) + }; + } + + return summary; } public void Dispose() @@ -104,7 +121,13 @@ private sealed record ResponseFormat( [property: JsonPropertyName("type")] string Type); private sealed record ChatCompletionResponse( - [property: JsonPropertyName("choices")] IReadOnlyList Choices); + [property: JsonPropertyName("choices")] IReadOnlyList Choices, + [property: JsonPropertyName("usage")] ChatCompletionUsage? Usage); + + private sealed record ChatCompletionUsage( + [property: JsonPropertyName("prompt_tokens")] int PromptTokens, + [property: JsonPropertyName("completion_tokens")] int CompletionTokens, + [property: JsonPropertyName("total_tokens")] int TotalTokens); private sealed record ChatChoice( [property: JsonPropertyName("message")] ChatChoiceMessage Message); diff --git a/src/CodeMesh.Ingestion/Summary/OllamaCodeSummaryProvider.cs b/src/CodeMesh.Ingestion/Summary/OllamaCodeSummaryProvider.cs index f0b5d2d..4c70d55 100644 --- a/src/CodeMesh.Ingestion/Summary/OllamaCodeSummaryProvider.cs +++ b/src/CodeMesh.Ingestion/Summary/OllamaCodeSummaryProvider.cs @@ -37,6 +37,8 @@ public OllamaCodeSummaryProvider( public string Model { get; } + public string GenerationConfigurationFingerprint => "temperature:0.1;format:json"; + public async Task SummarizeAsync( CodeSummaryRequest request, CancellationToken cancellationToken = default) @@ -68,7 +70,22 @@ public async Task SummarizeAsync( .ReadFromJsonAsync(JsonOptions, cancellationToken) .ConfigureAwait(false); - return SummaryPrompt.ParseResult(result?.Message.Content); + var summary = SummaryPrompt.ParseResult(result?.Message.Content); + if (result?.PromptEvalCount is { } inputTokens && result.EvalCount is { } outputTokens) + { + summary = summary with + { + Usage = new CodeSummaryUsage( + inputTokens, + 0, + 0, + outputTokens, + 0, + inputTokens + outputTokens) + }; + } + + return summary; } public void Dispose() @@ -106,7 +123,9 @@ private sealed record ChatOptions( [property: JsonPropertyName("num_predict")] int NumPredict); private sealed record ChatResponse( - [property: JsonPropertyName("message")] ChatResponseMessage Message); + [property: JsonPropertyName("message")] ChatResponseMessage Message, + [property: JsonPropertyName("prompt_eval_count")] int? PromptEvalCount, + [property: JsonPropertyName("eval_count")] int? EvalCount); private sealed record ChatResponseMessage( [property: JsonPropertyName("content")] string Content); diff --git a/src/CodeMesh.Ingestion/Summary/Qualification/SummaryQualification.cs b/src/CodeMesh.Ingestion/Summary/Qualification/SummaryQualification.cs new file mode 100644 index 0000000..0508f74 --- /dev/null +++ b/src/CodeMesh.Ingestion/Summary/Qualification/SummaryQualification.cs @@ -0,0 +1,1618 @@ +using System.Diagnostics; +using CodeMesh.Domain.Graph; +using CodeMesh.Domain.Utilities; + +namespace CodeMesh.Ingestion.Summary.Qualification; + +public static class SummaryQualificationConstants +{ + public const string SchemaVersion = "1.0"; + public const string SuiteKind = "summary-qualification-suite"; + public const string ProfileKind = "summary-qualification-profile"; + public const string ArchiveKind = "summary-qualification-private-archive"; + public const string ReviewPacketKind = "summary-qualification-review-packet"; + public const string ReviewKind = "summary-qualification-review"; + public const string RetrievalAssessmentKind = "summary-qualification-retrieval-assessment"; + public const string RetrievalEvidenceKind = "summary-qualification-retrieval-evidence"; + public const string DeploymentEvidenceKind = "summary-qualification-deployment-evidence"; + public const string ReportKind = "summary-qualification-report"; +} + +public sealed record SummaryQualificationGates( + double MinimumFirstAttemptSchemaRate = 0.995, + double MinimumEventualCompletionRate = 0.999, + int MaximumSecretCanaryLeaks = 0, + int MaximumCriticalHallucinations = 0, + double MinimumAtomicFactualPrecision = 0.95, + double MinimumRequiredFactRecall = 0.80, + double MinimumStratumRequiredFactRecall = 0.70, + double MaximumAbsoluteRetrievalRegression = 0.01, + int MaximumSafetyFailures = 0, + double? MaximumP95LatencyMs = null, + double? MinimumNodesPerMinute = null, + long? MaximumTotalTokens = null, + double? MaximumTokensPerSuccessfulNode = null, + double? MaximumTotalDurationMs = null, + double? MaximumRetryTokenOverheadRatio = null, + double? MaximumPeakHostMemoryMb = null, + double? MaximumPeakAcceleratorMemoryMb = null, + decimal? MaximumOnlineCost = null); + +public sealed record SummaryQualificationCorpusItem( + string Id, + string RepositoryId, + string RepositoryCommit, + string NodeId, + string ContentHash, + string Language, + CodeNodeKind NodeKind, + string Name, + string Project, + string FilePath, + int StartLine, + int EndLine, + string Source, + string InputSizeBand, + string Criticality, + IReadOnlyList Strata, + IReadOnlyList RequiredFacts, + IReadOnlyList OptionalFacts, + IReadOnlyList ForbiddenClaims, + IReadOnlyList RequiredTags, + IReadOnlyList AcceptableTags, + string Partition, + string ExpectedResponsibility, + IReadOnlyList ImportantInputs, + IReadOnlyList ImportantOutputs, + IReadOnlyList SideEffects, + IReadOnlyList FailureBehavior, + string ReferenceSummary, + IReadOnlyList? SecretCanaries = null); + +public sealed record SummaryQualificationSuite( + string SchemaVersion, + string Kind, + string Name, + string Version, + IReadOnlyList Items, + SummaryQualificationGates Gates); + +public sealed record SummaryQualificationProfile( + string SchemaVersion, + string Kind, + string Name, + string IntendedUse, + string EndpointClass, + string Provider, + string Model, + string ModelRevision, + string Quantization, + string NumericPrecision, + string ChatTemplateHash, + int ContextLength, + int MaxInputCharacters, + int ReasoningTokenReserve, + int MaxCompletionTokens, + int Repetitions, + IReadOnlyDictionary GenerationSettings, + IReadOnlyDictionary Host, + IReadOnlyDictionary? OnlinePolicy = null, + IReadOnlyList? Conditions = null); + +public sealed record SummaryQualificationSourceIdentity( + string CodeMeshCommit, + bool WorkingTreeDirty, + string? Branch, + string? RemoteUrl, + string OperatingSystem, + string RuntimeVersion); + +public sealed record SummaryQualificationRepositoryIdentity( + string RepositoryId, + string RepositoryCommit); + +public sealed record SummaryQualificationSample( + string SampleId, + string ItemId, + int Repetition, + string Phase, + bool Completed, + long LatencyMs, + string? FailureCategory, + string? Summary, + IReadOnlyList Tags, + string? Confidence, + CodeSummaryUsage? Usage, + IReadOnlyDictionary? ProviderMetadata, + string RedactedContentHash, + bool InputTruncated, + int InputCharacters); + +public sealed record SummaryQualificationPrivateArchive( + string SchemaVersion, + string Kind, + string RunId, + string SuiteName, + string SuiteVersion, + string SuiteHash, + string CorpusRepositoryIdentityHash, + IReadOnlyList CorpusRepositories, + string ProfileHash, + string ProviderGenerationConfigurationFingerprint, + SummaryQualificationSourceIdentity SourceIdentity, + SummaryQualificationProfile Profile, + string PromptVersion, + string PromptHash, + DateTimeOffset StartedAt, + DateTimeOffset CompletedAt, + IReadOnlyList Samples, + IReadOnlyList PreflightFailures); + +public sealed record SummaryQualificationReviewItem( + string SampleId, + string ItemId, + string Source, + IReadOnlyList RequiredFacts, + IReadOnlyList OptionalFacts, + IReadOnlyList ForbiddenClaims, + IReadOnlyList RequiredTags, + IReadOnlyList AcceptableTags, + string ExpectedResponsibility, + IReadOnlyList ImportantInputs, + IReadOnlyList ImportantOutputs, + IReadOnlyList SideEffects, + IReadOnlyList FailureBehavior, + string ReferenceSummary, + string? CandidateSummary, + IReadOnlyList CandidateTags, + string? CandidateConfidence); + +public sealed record SummaryQualificationReviewPacket( + string SchemaVersion, + string Kind, + string RunId, + string CandidateLabel, + string SuiteHash, + IReadOnlyList Items); + +public sealed record SummaryQualificationSampleReview( + string SampleId, + string ItemId, + int SupportedFactCount, + int GeneratedFactCount, + int RequiredFactCount, + int RequiredFactPresentCount, + bool CriticalHallucination, + bool Contradiction, + bool PurposeAccurate, + int BehaviorFactCount, + int BehaviorFactPresentCount, + int UsefulSupportedFactCount, + bool OverallFactuallyCorrect, + bool ConfidenceAppropriate, + int TagTruePositiveCount, + int TagFalsePositiveCount, + int TagFalseNegativeCount, + int SafetyFailureCount = 0, + string? Limitation = null); + +public sealed record SummaryQualificationReview( + string SchemaVersion, + string Kind, + string RunId, + string ReviewerId, + bool Blinded, + string ResolutionRecordHash, + IReadOnlyList Samples); + +public sealed record SummaryRetrievalMetrics( + double RecallAtK, + double MeanReciprocalRank, + double NdcgAtK, + double PrecisionAtK, + double RankingStability, + int SecretLeaks, + bool Passed, + bool Comparable); + +public sealed record SummaryQualificationRetrievalAssessment( + string SchemaVersion, + string Kind, + string RunId, + string BaselineReportHash, + string CandidateReportHash, + int CorrectSummaryHitContributions, + int FalsePositiveSummaryHits, + int QueriesImproved, + int QueriesUnchanged, + int QueriesDegraded, + double BaselineContextRelevance, + double CandidateContextRelevance, + double BaselineContextTokenDensity, + double CandidateContextTokenDensity, + int UnresolvedMaterialRegressions, + bool Reviewed); + +public sealed record SummaryQualificationRetrievalEvidence( + string SchemaVersion, + string Kind, + string RunId, + string SuiteHash, + string PromptHash, + string CorpusRepositoryIdentityHash, + string RankingIdentityHash, + string LiveSuite, + string RepositoryId, + string RepositoryCommit, + string BaselineReportHash, + string CandidateReportHash, + SummaryRetrievalMetrics Baseline, + SummaryRetrievalMetrics Candidate, + SummaryQualificationRetrievalAssessment Assessment, + IReadOnlyList SafetyFailures); + +public sealed record SummaryQualificationDeploymentEvidence( + string SchemaVersion, + string Kind, + string RunId, + string ProfileHash, + string MeasurementMethod, + double PeakHostMemoryMb, + double? PeakAcceleratorMemoryMb, + decimal? ProviderReportedCost, + string? PricingSnapshotHash, + bool Complete); + +public sealed record SummaryQualificationStratumMetrics( + int SampleCount, + double AtomicFactualPrecision, + double RequiredFactRecall, + double PurposeAccuracy, + double BehaviorCoverage); + +public sealed record SummaryQualificationMetrics( + int Attempted, + int Completed, + double FirstAttemptSchemaRate, + double EventualCompletionRate, + int SecretCanaryLeaks, + double AtomicFactualPrecision, + double RequiredFactRecall, + IReadOnlyDictionary RequiredFactRecallByStratum, + int CriticalHallucinations, + int Contradictions, + double PurposeAccuracy, + double BehaviorCoverage, + double ConfidenceCalibrationAccuracy, + double InformationDensityPer1000OutputTokens, + double IdenticalInputStability, + double TagPrecision, + double TagRecall, + double TagF1, + double P50LatencyMs, + double P95LatencyMs, + double MaximumLatencyMs, + double WarmupLatencyMs, + double NodesPerMinute, + double SuccessfulNodesPerHour, + long? InputTokens, + long? CachedInputTokens, + long? CacheWriteInputTokens, + long? OutputTokens, + long? ReasoningTokens, + long? TotalTokens, + double? TokensPerAttemptedNode, + double? TokensPerSuccessfulNode, + double RetryTokenOverheadRatio, + double SupportedFactsPer1000OutputTokens, + double RequiredFactsPer1000OutputTokens, + double TotalDurationMs, + double? PeakHostMemoryMb, + double? PeakAcceleratorMemoryMb, + decimal? ProviderReportedCost, + IReadOnlyDictionary ByStratum, + SummaryRetrievalMetrics? BaselineRetrieval, + SummaryRetrievalMetrics? CandidateRetrieval, + SummaryQualificationRetrievalAssessment? RetrievalAssessment); + +public sealed record SummaryQualificationReport( + string SchemaVersion, + string Kind, + string RunId, + string Outcome, + string IntendedUse, + string SuiteName, + string SuiteVersion, + string SuiteHash, + string CorpusRepositoryIdentityHash, + string ProfileHash, + string Provider, + string Model, + string ModelRevision, + string PromptVersion, + string PromptHash, + string ProviderGenerationConfigurationFingerprint, + string RankingIdentityHash, + string RetrievalSuite, + string RetrievalRepositoryId, + string RetrievalRepositoryCommit, + string DeploymentEvidenceHash, + string CodeMeshCommit, + DateTimeOffset GeneratedAt, + SummaryQualificationMetrics Metrics, + IReadOnlyList FailedSampleIds, + IReadOnlyList FailedGates, + IReadOnlyList InvalidReasons, + IReadOnlyList Conditions, + IReadOnlyList Limitations); + +public sealed record SummaryQualificationRunResult( + SummaryQualificationPrivateArchive PrivateArchive, + SummaryQualificationReviewPacket ReviewPacket); + +public sealed class SummaryQualificationRunner(ICodeSummaryProvider provider) +{ + public async Task RunAsync( + SummaryQualificationSuite suite, + SummaryQualificationProfile profile, + SummaryQualificationSourceIdentity sourceIdentity, + CancellationToken cancellationToken = default) + { + var preflightFailures = Validate(suite, profile, sourceIdentity, provider); + if (preflightFailures.Count > 0) + { + throw new InvalidOperationException( + $"Summary qualification preflight failed: {string.Join(" ", preflightFailures)}"); + } + + var suiteHash = Fingerprint.Suite(suite); + var profileHash = Fingerprint.Profile(profile); + var repositoryIdentityHash = Fingerprint.RepositoryIdentity(suite.Items); + var promptVersion = SummaryPrompt.DefaultPromptVersion; + var promptHash = SummaryPrompt.PromptHash(promptVersion); + var qualificationItems = suite.Items + .Where(item => string.Equals(item.Partition, "qualification", StringComparison.Ordinal)) + .ToArray(); + var warmupItem = suite.Items.First(item => string.Equals(item.Partition, "calibration", StringComparison.Ordinal)); + var runId = CodeMeshHash.StableId( + "summary-qualification-run-v1", + sourceIdentity.CodeMeshCommit, + suiteHash, + profileHash, + promptHash, + provider.GenerationConfigurationFingerprint, + DateTimeOffset.UtcNow.ToString("O", System.Globalization.CultureInfo.InvariantCulture)); + var startedAt = DateTimeOffset.UtcNow; + var samples = new List((qualificationItems.Length * profile.Repetitions) + 1); + + async Task ExecuteSampleAsync( + SummaryQualificationCorpusItem item, + int repetition, + string phase) + { + var redactedSource = ContentRedactor.RedactText(item.Source); + var redactedHash = CodeMeshHash.Sha256Hex(redactedSource); + var node = CreateNode(item, redactedHash); + var content = new CodeContent( + redactedHash, + redactedSource, + "text/plain", + startedAt, + startedAt); + cancellationToken.ThrowIfCancellationRequested(); + var request = SummaryBudgetPolicy.CreateRequest( + node, + content, + promptVersion, + new SummaryBudgetPolicyOptions( + profile.MaxInputCharacters, + profile.ReasoningTokenReserve, + profile.MaxCompletionTokens)); + var sampleId = CodeMeshHash.StableId( + runId, + item.Id, + phase, + repetition.ToString(System.Globalization.CultureInfo.InvariantCulture)); + var stopwatch = Stopwatch.StartNew(); + try + { + var result = await provider.SummarizeAsync(request, cancellationToken).ConfigureAwait(false); + ValidateProviderResult(result); + stopwatch.Stop(); + return new SummaryQualificationSample( + sampleId, + item.Id, + repetition, + phase, + true, + stopwatch.ElapsedMilliseconds, + null, + result.Summary, + result.Tags, + result.Confidence, + result.Usage, + result.ProviderMetadata, + redactedHash, + request.Budget?.InputTruncated ?? false, + request.Budget?.InputCharacters ?? 0); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + stopwatch.Stop(); + return new SummaryQualificationSample( + sampleId, + item.Id, + repetition, + phase, + false, + stopwatch.ElapsedMilliseconds, + FailureCategory(exception), + null, + [], + null, + null, + null, + redactedHash, + request.Budget?.InputTruncated ?? false, + request.Budget?.InputCharacters ?? 0); + } + } + + var runtimeFailures = new List(); + var warmup = await ExecuteSampleAsync(warmupItem, 0, "warmup").ConfigureAwait(false); + samples.Add(warmup); + if (!warmup.Completed) + { + runtimeFailures.Add($"Warm-up failed with category '{warmup.FailureCategory ?? "unknown"}'; measured repetitions were not started."); + } + else + { + foreach (var item in qualificationItems) + { + for (var repetition = 1; repetition <= profile.Repetitions; repetition++) + { + samples.Add(await ExecuteSampleAsync(item, repetition, "measured").ConfigureAwait(false)); + } + } + } + + var archive = new SummaryQualificationPrivateArchive( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.ArchiveKind, + runId, + suite.Name, + suite.Version, + suiteHash, + repositoryIdentityHash, + suite.Items + .Select(item => new SummaryQualificationRepositoryIdentity(item.RepositoryId, item.RepositoryCommit)) + .Distinct() + .OrderBy(item => item.RepositoryId, StringComparer.Ordinal) + .ThenBy(item => item.RepositoryCommit, StringComparer.Ordinal) + .ToArray(), + profileHash, + provider.GenerationConfigurationFingerprint, + sourceIdentity, + profile, + promptVersion, + promptHash, + startedAt, + DateTimeOffset.UtcNow, + samples, + runtimeFailures); + var itemsById = suite.Items.ToDictionary(item => item.Id, StringComparer.Ordinal); + var packet = new SummaryQualificationReviewPacket( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.ReviewPacketKind, + runId, + $"candidate-{runId[..12]}", + suiteHash, + samples + .Where(sample => string.Equals(sample.Phase, "measured", StringComparison.Ordinal)) + .Select(sample => CreateReviewItem(sample, itemsById[sample.ItemId])) + .ToArray()); + + return new SummaryQualificationRunResult(archive, packet); + } + + public static async Task CaptureSourceIdentityAsync( + string codeMeshRoot, + CancellationToken cancellationToken = default) + { + var snapshot = await RepositoryGitSnapshot.CaptureAsync(codeMeshRoot, cancellationToken).ConfigureAwait(false); + return new SummaryQualificationSourceIdentity( + snapshot.Commit ?? string.Empty, + snapshot.WorkingTreeDirty, + snapshot.Branch, + snapshot.RemoteUrl, + System.Runtime.InteropServices.RuntimeInformation.OSDescription, + System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription); + } + + private static List Validate( + SummaryQualificationSuite suite, + SummaryQualificationProfile profile, + SummaryQualificationSourceIdentity sourceIdentity, + ICodeSummaryProvider candidate) + { + var failures = new List(); + if (!string.Equals(suite.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(suite.Kind, SummaryQualificationConstants.SuiteKind, StringComparison.Ordinal)) + { + failures.Add("Unsupported suite schema or kind."); + } + + if (!string.Equals(profile.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(profile.Kind, SummaryQualificationConstants.ProfileKind, StringComparison.Ordinal)) + { + failures.Add("Unsupported deployment-profile schema or kind."); + } + + if (suite.Items.Count == 0 || suite.Items.Select(item => item.Id).Distinct(StringComparer.Ordinal).Count() != suite.Items.Count) + { + failures.Add("The suite must contain uniquely identified corpus items."); + } + + if (!suite.Items.Any(item => string.Equals(item.Partition, "calibration", StringComparison.Ordinal)) || + !suite.Items.Any(item => string.Equals(item.Partition, "qualification", StringComparison.Ordinal))) + { + failures.Add("The suite must contain separate calibration and held-out qualification partitions."); + } + + foreach (var item in suite.Items) + { + if (string.IsNullOrWhiteSpace(item.RepositoryCommit) || + string.IsNullOrWhiteSpace(item.ContentHash) || + !string.Equals(CodeMeshHash.Sha256Hex(item.Source), item.ContentHash, StringComparison.Ordinal)) + { + failures.Add($"Corpus item '{item.Id}' has incomplete or mismatched immutable source identity."); + } + + if (string.IsNullOrWhiteSpace(item.Id) || + string.IsNullOrWhiteSpace(item.NodeId) || + string.IsNullOrWhiteSpace(item.Language) || + string.IsNullOrWhiteSpace(item.FilePath) || + string.IsNullOrWhiteSpace(item.ExpectedResponsibility) || + string.IsNullOrWhiteSpace(item.ReferenceSummary) || + item.RequiredFacts.Count == 0 || + item.Strata.Count == 0 || + item.NodeKind == CodeNodeKind.Unknown || + item.Partition is not ("calibration" or "qualification")) + { + failures.Add($"Corpus item '{item.Id}' is missing required labels, strata, or node identity."); + } + + if (RepositoryPathPolicy.IsIgnoredRelativePath(item.FilePath)) + { + failures.Add($"Corpus item '{item.Id}' uses a generated, cache, or known-secret file path."); + } + + if ((item.SecretCanaries ?? []).Any(canary => + string.IsNullOrWhiteSpace(canary) || + !item.Source.Contains(canary, StringComparison.Ordinal))) + { + failures.Add($"Corpus item '{item.Id}' declares a missing or empty secret canary."); + } + } + + if (string.IsNullOrWhiteSpace(sourceIdentity.CodeMeshCommit) || sourceIdentity.WorkingTreeDirty) + { + failures.Add("CodeMesh must be at a recorded clean commit."); + } + + if (!string.Equals(profile.Provider, candidate.Provider, StringComparison.OrdinalIgnoreCase) || + !string.Equals(profile.Model, candidate.Model, StringComparison.Ordinal)) + { + failures.Add("The deployment profile does not match the configured provider and model."); + } + + if (string.IsNullOrWhiteSpace(candidate.GenerationConfigurationFingerprint) || + !profile.GenerationSettings.TryGetValue("providerGenerationConfigurationFingerprint", out var expectedConfiguration) || + !string.Equals(expectedConfiguration, candidate.GenerationConfigurationFingerprint, StringComparison.Ordinal)) + { + failures.Add("The deployment profile does not match the provider's effective generation configuration fingerprint."); + } + + if (!profile.GenerationSettings.TryGetValue("retryPolicy", out var retryPolicy) || + !string.Equals(retryPolicy, "none", StringComparison.Ordinal)) + { + failures.Add("Qualification runner v1 supports only an explicit retryPolicy of 'none'."); + } + + RequireKeys( + failures, + "generation settings", + profile.GenerationSettings, + [ + "providerGenerationConfigurationFingerprint", + "temperature", + "topP", + "topK", + "seed", + "stopSequences", + "requestedReasoningEffort", + "effectiveReasoningEffort", + "retryPolicy", + "timeout", + "concurrency", + "batching" + ]); + RequireKeys( + failures, + "host identity", + profile.Host, + [ + "operatingSystem", + "cpu", + "memory", + "accelerator", + "acceleratorMemory", + "servingRuntime" + ]); + + if (string.IsNullOrWhiteSpace(profile.Name) || + string.IsNullOrWhiteSpace(profile.ModelRevision) || + string.IsNullOrWhiteSpace(profile.Quantization) || + string.IsNullOrWhiteSpace(profile.NumericPrecision) || + string.IsNullOrWhiteSpace(profile.ChatTemplateHash) || + profile.Host.Count == 0) + { + failures.Add("The deployment profile is missing model-build, template, precision, or host identity."); + } + + if (profile.Repetitions < 3) + { + failures.Add("At least three measured repetitions are required."); + } + + if (profile.MaxInputCharacters < 512 || profile.MaxCompletionTokens < 448 || profile.ContextLength <= 0) + { + failures.Add("The deployment profile has invalid context or token budgets."); + } + + var intendedUses = new[] { "local-offline", "private-network", "online-approved-data" }; + if (!intendedUses.Contains(profile.IntendedUse, StringComparer.Ordinal)) + { + failures.Add("The intended use is not recognized."); + } + + var endpointClasses = new[] { "local", "private-network", "online" }; + if (!endpointClasses.Contains(profile.EndpointClass, StringComparer.Ordinal)) + { + failures.Add("The endpoint class is not recognized."); + } + + if (string.Equals(profile.EndpointClass, "online", StringComparison.Ordinal) && + !string.Equals(profile.IntendedUse, "online-approved-data", StringComparison.Ordinal)) + { + failures.Add("An online endpoint can only be evaluated for online-approved-data use."); + } + + if (string.Equals(profile.EndpointClass, "local", StringComparison.Ordinal) && + (!profile.Host.TryGetValue("operatingSystem", out var profileOperatingSystem) || + !string.Equals(profileOperatingSystem, sourceIdentity.OperatingSystem, StringComparison.Ordinal))) + { + failures.Add("A local deployment profile must match the captured host operating-system identity."); + } + + if (string.Equals(profile.IntendedUse, "online-approved-data", StringComparison.Ordinal) && + (profile.OnlinePolicy is null || profile.OnlinePolicy.Count == 0)) + { + failures.Add("Online qualification requires a recorded governance and pricing policy snapshot."); + } + else if (string.Equals(profile.IntendedUse, "online-approved-data", StringComparison.Ordinal)) + { + RequireKeys( + failures, + "online policy", + profile.OnlinePolicy!, + ["apiVersion", "serviceRegion", "retentionPolicy", "pricingSnapshotHash", "maximumCost"]); + } + + return failures; + } + + private static CodeNode CreateNode(SummaryQualificationCorpusItem item, string redactedHash) + { + return new CodeNode( + item.NodeId, + $"qualification:{item.RepositoryId}:{item.NodeId}", + item.NodeKind, + item.Name, + item.Language, + item.Project, + new SourceSpan(item.FilePath, item.StartLine, 1, item.EndLine, 1), + redactedHash, + new Dictionary(StringComparer.Ordinal) + { + ["summaryQualificationItemId"] = item.Id, + ["summaryQualificationCriticality"] = item.Criticality + }); + } + + private static void RequireKeys( + List failures, + string label, + IReadOnlyDictionary values, + IReadOnlyList requiredKeys) + { + var missing = requiredKeys + .Where(key => !values.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + .ToArray(); + if (missing.Length > 0) + { + failures.Add($"The deployment profile {label} are missing: {string.Join(", ", missing)}."); + } + } + + private static SummaryQualificationReviewItem CreateReviewItem( + SummaryQualificationSample sample, + SummaryQualificationCorpusItem item) + { + return new SummaryQualificationReviewItem( + sample.SampleId, + item.Id, + ContentRedactor.RedactText(item.Source), + item.RequiredFacts, + item.OptionalFacts, + item.ForbiddenClaims, + item.RequiredTags, + item.AcceptableTags, + item.ExpectedResponsibility, + item.ImportantInputs, + item.ImportantOutputs, + item.SideEffects, + item.FailureBehavior, + item.ReferenceSummary, + sample.Summary, + sample.Tags, + sample.Confidence); + } + + private static string FailureCategory(Exception exception) + { + return exception switch + { + TimeoutException or TaskCanceledException => "timeout", + HttpRequestException => "provider-transport", + InvalidOperationException when exception.Message.Contains("JSON", StringComparison.OrdinalIgnoreCase) => "schema-invalid", + InvalidOperationException when exception.Message.Contains("empty", StringComparison.OrdinalIgnoreCase) => "empty-response", + _ => "provider-error" + }; + } + + private static void ValidateProviderResult(CodeSummaryResult result) + { + if (string.IsNullOrWhiteSpace(result.Summary) || + result.Tags is null || + result.Tags.Count > 8 || + result.Tags.Any(string.IsNullOrWhiteSpace) || + result.Confidence is not ("high" or "medium" or "low")) + { + throw new InvalidOperationException("Summary provider result violated the production JSON contract."); + } + + if (result.Usage is { } usage && + (usage.InputTokens < 0 || + usage.CachedInputTokens < 0 || + usage.CacheWriteInputTokens < 0 || + usage.OutputTokens < 0 || + usage.ReasoningTokens < 0 || + usage.TotalTokens < 0)) + { + throw new InvalidOperationException("Summary provider returned invalid usage counts."); + } + } +} + +public static class SummaryQualificationCompiler +{ + public static SummaryQualificationReport Compile( + SummaryQualificationSuite suite, + SummaryQualificationPrivateArchive archive, + IReadOnlyList reviews, + SummaryQualificationRetrievalEvidence? retrievalEvidence, + SummaryQualificationDeploymentEvidence? deploymentEvidence) + { + var invalid = ValidateInputs(suite, archive, reviews, retrievalEvidence, deploymentEvidence); + var measured = archive.Samples + .Where(sample => string.Equals(sample.Phase, "measured", StringComparison.Ordinal)) + .ToArray(); + var completed = measured.Where(sample => sample.Completed).ToArray(); + var sampleReviews = MergeReviews(suite, completed, reviews, invalid); + var canariesByItem = suite.Items.ToDictionary( + item => item.Id, + item => item.SecretCanaries ?? [], + StringComparer.Ordinal); + var canaryLeaks = completed.Sum(sample => CountCanaryLeaks(sample, canariesByItem[sample.ItemId])); + var generatedFacts = sampleReviews.Sum(review => review.GeneratedFactCount); + var supportedFacts = sampleReviews.Sum(review => review.SupportedFactCount); + var requiredFacts = sampleReviews.Sum(review => review.RequiredFactCount); + var presentFacts = sampleReviews.Sum(review => review.RequiredFactPresentCount); + var behaviorFacts = sampleReviews.Sum(review => review.BehaviorFactCount); + var presentBehaviorFacts = sampleReviews.Sum(review => review.BehaviorFactPresentCount); + var usefulSupportedFacts = sampleReviews.Sum(review => review.UsefulSupportedFactCount); + var strata = RequiredRecallByStratum(suite, sampleReviews); + var elapsedMs = Math.Max(1, measured.Sum(sample => sample.LatencyMs)); + var totalDurationMs = Math.Max(1, (archive.CompletedAt - archive.StartedAt).TotalMilliseconds); + var usageComplete = archive.Samples.Count > 0 && archive.Samples.All(sample => sample.Usage is not null); + long? inputTokens = usageComplete ? archive.Samples.Sum(sample => (long)sample.Usage!.InputTokens) : null; + long? cachedInputTokens = usageComplete ? archive.Samples.Sum(sample => (long)sample.Usage!.CachedInputTokens) : null; + long? cacheWriteInputTokens = usageComplete ? archive.Samples.Sum(sample => (long)sample.Usage!.CacheWriteInputTokens) : null; + long? outputTokens = usageComplete ? archive.Samples.Sum(sample => (long)sample.Usage!.OutputTokens) : null; + long? reasoningTokens = usageComplete ? archive.Samples.Sum(sample => (long)sample.Usage!.ReasoningTokens) : null; + long? totalTokens = usageComplete ? archive.Samples.Sum(sample => (long)sample.Usage!.TotalTokens) : null; + var measuredOutputTokens = measured.Length > 0 && measured.All(sample => sample.Usage is not null) + ? measured.Sum(sample => (long)sample.Usage!.OutputTokens) + : (long?)null; + var successfulAttempts = archive.Samples.Count(sample => sample.Completed); + var metrics = new SummaryQualificationMetrics( + measured.Length, + completed.Length, + Rate(completed.Length, measured.Length), + Rate(completed.Length, measured.Length), + canaryLeaks, + Rate(supportedFacts, generatedFacts), + Rate(presentFacts, requiredFacts), + strata, + sampleReviews.Count(review => review.CriticalHallucination), + sampleReviews.Count(review => review.Contradiction), + Rate(sampleReviews.Count(review => review.PurposeAccurate), sampleReviews.Count), + Rate(presentBehaviorFacts, behaviorFacts), + Rate(sampleReviews.Count(review => review.ConfidenceAppropriate), sampleReviews.Count), + PerThousand(usefulSupportedFacts, measuredOutputTokens), + IdenticalInputStability(measured, archive.Profile.Repetitions), + Rate(sampleReviews.Sum(review => review.TagTruePositiveCount), sampleReviews.Sum(review => review.TagTruePositiveCount + review.TagFalsePositiveCount)), + Rate(sampleReviews.Sum(review => review.TagTruePositiveCount), sampleReviews.Sum(review => review.TagTruePositiveCount + review.TagFalseNegativeCount)), + 0, + Percentile(measured.Select(sample => (double)sample.LatencyMs), 0.50), + Percentile(measured.Select(sample => (double)sample.LatencyMs), 0.95), + measured.Length == 0 ? 0 : measured.Max(sample => (double)sample.LatencyMs), + archive.Samples.FirstOrDefault(sample => string.Equals(sample.Phase, "warmup", StringComparison.Ordinal))?.LatencyMs ?? 0, + completed.Length / (elapsedMs / 60000d), + completed.Length / (elapsedMs / 3600000d), + inputTokens, + cachedInputTokens, + cacheWriteInputTokens, + outputTokens, + reasoningTokens, + totalTokens, + totalTokens is null || archive.Samples.Count == 0 ? null : totalTokens.Value / (double)archive.Samples.Count, + totalTokens is null || successfulAttempts == 0 ? null : totalTokens.Value / (double)successfulAttempts, + 0, + PerThousand(supportedFacts, measuredOutputTokens), + PerThousand(presentFacts, measuredOutputTokens), + totalDurationMs, + deploymentEvidence?.PeakHostMemoryMb, + deploymentEvidence?.PeakAcceleratorMemoryMb, + deploymentEvidence?.ProviderReportedCost, + MetricsByStratum(suite, sampleReviews), + retrievalEvidence?.Baseline, + retrievalEvidence?.Candidate, + retrievalEvidence?.Assessment); + var tagF1 = HarmonicMean(metrics.TagPrecision, metrics.TagRecall); + metrics = metrics with { TagF1 = tagF1 }; + + var failed = invalid.Count == 0 + ? FailedGates(suite.Gates, metrics, sampleReviews, retrievalEvidence, deploymentEvidence) + : []; + var outcome = invalid.Count > 0 + ? "invalid-run" + : failed.Count > 0 + ? "not-qualified" + : archive.Profile.Conditions is { Count: > 0 } + ? "conditionally-qualified" + : "qualified"; + var limitations = sampleReviews + .Where(review => !string.IsNullOrWhiteSpace(review.Limitation)) + .Select(review => $"Reviewer limitation recorded for sample {review.SampleId}.") + .Distinct(StringComparer.Ordinal) + .ToArray(); + var failedSampleIds = measured + .Where(sample => !sample.Completed) + .Select(sample => sample.SampleId) + .Concat(sampleReviews + .Where(review => + review.CriticalHallucination || + review.Contradiction || + !review.PurposeAccurate || + !review.OverallFactuallyCorrect || + !review.ConfidenceAppropriate || + review.SafetyFailureCount > 0 || + review.SupportedFactCount < review.GeneratedFactCount || + review.RequiredFactPresentCount < review.RequiredFactCount || + review.BehaviorFactPresentCount < review.BehaviorFactCount) + .Select(review => review.SampleId)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + return new SummaryQualificationReport( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.ReportKind, + archive.RunId, + outcome, + archive.Profile.IntendedUse, + archive.SuiteName, + archive.SuiteVersion, + archive.SuiteHash, + archive.CorpusRepositoryIdentityHash, + archive.ProfileHash, + archive.Profile.Provider, + archive.Profile.Model, + archive.Profile.ModelRevision, + archive.PromptVersion, + archive.PromptHash, + archive.ProviderGenerationConfigurationFingerprint, + retrievalEvidence?.RankingIdentityHash ?? string.Empty, + retrievalEvidence?.LiveSuite ?? string.Empty, + retrievalEvidence?.RepositoryId ?? string.Empty, + retrievalEvidence?.RepositoryCommit ?? string.Empty, + deploymentEvidence is null + ? string.Empty + : CodeMeshHash.Sha256Hex(System.Text.Json.JsonSerializer.Serialize(deploymentEvidence)), + archive.SourceIdentity.CodeMeshCommit, + DateTimeOffset.UtcNow, + metrics, + failedSampleIds, + failed, + invalid, + archive.Profile.Conditions ?? [], + limitations); + } + + public static IReadOnlyDictionary Compare( + SummaryQualificationReport candidate, + SummaryQualificationReport reference) + { + if (!string.Equals(candidate.SuiteHash, reference.SuiteHash, StringComparison.Ordinal) || + !string.Equals(candidate.PromptHash, reference.PromptHash, StringComparison.Ordinal) || + !string.Equals(candidate.CorpusRepositoryIdentityHash, reference.CorpusRepositoryIdentityHash, StringComparison.Ordinal) || + !string.Equals(candidate.RankingIdentityHash, reference.RankingIdentityHash, StringComparison.Ordinal) || + !string.Equals(candidate.RetrievalSuite, reference.RetrievalSuite, StringComparison.Ordinal) || + !string.Equals(candidate.RetrievalRepositoryId, reference.RetrievalRepositoryId, StringComparison.Ordinal) || + !string.Equals(candidate.RetrievalRepositoryCommit, reference.RetrievalRepositoryCommit, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "Qualification reports have incompatible corpus, prompt, repository, or ranking identities."); + } + + return new Dictionary(StringComparer.Ordinal) + { + ["firstAttemptSchemaRate"] = candidate.Metrics.FirstAttemptSchemaRate - reference.Metrics.FirstAttemptSchemaRate, + ["atomicFactualPrecision"] = candidate.Metrics.AtomicFactualPrecision - reference.Metrics.AtomicFactualPrecision, + ["requiredFactRecall"] = candidate.Metrics.RequiredFactRecall - reference.Metrics.RequiredFactRecall, + ["purposeAccuracy"] = candidate.Metrics.PurposeAccuracy - reference.Metrics.PurposeAccuracy, + ["tagF1"] = candidate.Metrics.TagF1 - reference.Metrics.TagF1, + ["p95LatencyMs"] = candidate.Metrics.P95LatencyMs - reference.Metrics.P95LatencyMs, + ["nodesPerMinute"] = candidate.Metrics.NodesPerMinute - reference.Metrics.NodesPerMinute + }; + } + + private static List ValidateInputs( + SummaryQualificationSuite suite, + SummaryQualificationPrivateArchive archive, + IReadOnlyList reviews, + SummaryQualificationRetrievalEvidence? retrieval, + SummaryQualificationDeploymentEvidence? deployment) + { + var invalid = new List(); + if (!string.Equals(archive.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(archive.Kind, SummaryQualificationConstants.ArchiveKind, StringComparison.Ordinal)) + { + invalid.Add("Unsupported private-archive schema or kind."); + } + + if (!string.Equals(archive.SuiteHash, Fingerprint.Suite(suite), StringComparison.Ordinal)) + { + invalid.Add("The private archive does not match the supplied suite."); + } + + var expectedRepositoryIdentity = Fingerprint.RepositoryIdentity(suite.Items); + var expectedRepositories = suite.Items + .Select(item => $"{item.RepositoryId}\u001e{item.RepositoryCommit}") + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToArray(); + var archivedRepositories = archive.CorpusRepositories + .Select(item => $"{item.RepositoryId}\u001e{item.RepositoryCommit}") + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToArray(); + if (!string.Equals(archive.CorpusRepositoryIdentityHash, expectedRepositoryIdentity, StringComparison.Ordinal) || + !expectedRepositories.SequenceEqual(archivedRepositories, StringComparer.Ordinal)) + { + invalid.Add("The private archive corpus-repository identity is incomplete or mismatched."); + } + + if (archive.SourceIdentity.WorkingTreeDirty || string.IsNullOrWhiteSpace(archive.SourceIdentity.CodeMeshCommit)) + { + invalid.Add("The CodeMesh source identity is not a clean recorded commit."); + } + + if (archive.PreflightFailures.Count > 0) + { + invalid.AddRange(archive.PreflightFailures); + } + + if (suite.Gates.MaximumP95LatencyMs is null || + suite.Gates.MinimumNodesPerMinute is null || + suite.Gates.MaximumTotalTokens is null || + suite.Gates.MaximumTokensPerSuccessfulNode is null || + suite.Gates.MaximumTotalDurationMs is null || + suite.Gates.MaximumRetryTokenOverheadRatio is null) + { + invalid.Add("A qualification suite must predeclare duration, latency, throughput, token, and retry-overhead deployment budgets."); + } + + if (string.Equals(archive.Profile.EndpointClass, "online", StringComparison.Ordinal)) + { + if (suite.Gates.MaximumOnlineCost is null) + { + invalid.Add("An online qualification suite must predeclare a maximum provider cost."); + } + } + else + { + if (suite.Gates.MaximumPeakHostMemoryMb is null) + { + invalid.Add("A local or private-network qualification suite must predeclare a peak host-memory budget."); + } + + var accelerator = archive.Profile.Host.GetValueOrDefault("accelerator"); + if (!string.Equals(accelerator, "none", StringComparison.OrdinalIgnoreCase) && + suite.Gates.MaximumPeakAcceleratorMemoryMb is null) + { + invalid.Add("An accelerated qualification suite must predeclare a peak accelerator-memory budget."); + } + } + + if (reviews.Count < 2) + { + invalid.Add("At least two blinded reviewer files are required."); + } + + foreach (var review in reviews) + { + if (!string.Equals(review.RunId, archive.RunId, StringComparison.Ordinal) || + !string.Equals(review.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(review.Kind, SummaryQualificationConstants.ReviewKind, StringComparison.Ordinal) || + !review.Blinded) + { + invalid.Add($"Reviewer '{review.ReviewerId}' is not a compatible blinded review."); + } + } + + var resolutionHashes = reviews + .Select(review => review.ResolutionRecordHash) + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + if (resolutionHashes.Length != 1 || reviews.Any(review => string.IsNullOrWhiteSpace(review.ResolutionRecordHash))) + { + invalid.Add("Blinded reviews must bind the same non-empty disagreement-resolution record hash."); + } + + if (retrieval is null) + { + invalid.Add("Bound no-summary and candidate live-retrieval evidence is required."); + } + else if (!string.Equals(retrieval.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(retrieval.Kind, SummaryQualificationConstants.RetrievalEvidenceKind, StringComparison.Ordinal) || + !string.Equals(retrieval.RunId, archive.RunId, StringComparison.Ordinal) || + !string.Equals(retrieval.SuiteHash, archive.SuiteHash, StringComparison.Ordinal) || + !string.Equals(retrieval.PromptHash, archive.PromptHash, StringComparison.Ordinal) || + !string.Equals(retrieval.CorpusRepositoryIdentityHash, archive.CorpusRepositoryIdentityHash, StringComparison.Ordinal) || + string.IsNullOrWhiteSpace(retrieval.RankingIdentityHash) || + string.IsNullOrWhiteSpace(retrieval.LiveSuite) || + !archive.CorpusRepositories.Any(repository => + string.Equals(repository.RepositoryId, retrieval.RepositoryId, StringComparison.Ordinal) && + string.Equals(repository.RepositoryCommit, retrieval.RepositoryCommit, StringComparison.Ordinal)) || + string.IsNullOrWhiteSpace(retrieval.BaselineReportHash) || + string.IsNullOrWhiteSpace(retrieval.CandidateReportHash) || + string.Equals(retrieval.BaselineReportHash, retrieval.CandidateReportHash, StringComparison.Ordinal) || + !string.Equals(retrieval.Assessment.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(retrieval.Assessment.Kind, SummaryQualificationConstants.RetrievalAssessmentKind, StringComparison.Ordinal) || + !string.Equals(retrieval.Assessment.RunId, archive.RunId, StringComparison.Ordinal) || + !string.Equals(retrieval.Assessment.BaselineReportHash, retrieval.BaselineReportHash, StringComparison.Ordinal) || + !string.Equals(retrieval.Assessment.CandidateReportHash, retrieval.CandidateReportHash, StringComparison.Ordinal) || + !retrieval.Assessment.Reviewed || + HasInvalidRetrievalAssessmentCounts(retrieval.Assessment)) + { + invalid.Add("Live-retrieval evidence is incomplete or bound to a different qualification identity."); + } + + if (deployment is null) + { + invalid.Add("Bound deployment resource or cost evidence is required."); + } + else if (!string.Equals(deployment.SchemaVersion, SummaryQualificationConstants.SchemaVersion, StringComparison.Ordinal) || + !string.Equals(deployment.Kind, SummaryQualificationConstants.DeploymentEvidenceKind, StringComparison.Ordinal) || + !string.Equals(deployment.RunId, archive.RunId, StringComparison.Ordinal) || + !string.Equals(deployment.ProfileHash, archive.ProfileHash, StringComparison.Ordinal) || + string.IsNullOrWhiteSpace(deployment.MeasurementMethod) || + deployment.PeakHostMemoryMb < 0 || + deployment.PeakAcceleratorMemoryMb < 0 || + deployment.ProviderReportedCost < 0 || + !deployment.Complete) + { + invalid.Add("Deployment resource or cost evidence is incomplete or bound to a different qualification identity."); + } + else if (string.Equals(archive.Profile.EndpointClass, "online", StringComparison.Ordinal) && + (deployment.ProviderReportedCost is null || + string.IsNullOrWhiteSpace(deployment.PricingSnapshotHash) || + !string.Equals( + deployment.PricingSnapshotHash, + archive.Profile.OnlinePolicy?.GetValueOrDefault("pricingSnapshotHash"), + StringComparison.Ordinal))) + { + invalid.Add("Online deployment evidence must include provider cost bound to the profile pricing snapshot."); + } + else if (!string.Equals(archive.Profile.EndpointClass, "online", StringComparison.Ordinal) && + (deployment.PeakHostMemoryMb <= 0 || + (!string.Equals(archive.Profile.Host.GetValueOrDefault("accelerator"), "none", StringComparison.OrdinalIgnoreCase) && + deployment.PeakAcceleratorMemoryMb is null))) + { + invalid.Add("Local or private-network deployment evidence must include host and applicable accelerator memory measurements."); + } + + return invalid; + } + + private static bool HasInvalidRetrievalAssessmentCounts(SummaryQualificationRetrievalAssessment assessment) + { + return assessment.CorrectSummaryHitContributions < 0 || + assessment.FalsePositiveSummaryHits < 0 || + assessment.QueriesImproved < 0 || + assessment.QueriesUnchanged < 0 || + assessment.QueriesDegraded < 0 || + assessment.UnresolvedMaterialRegressions < 0 || + assessment.BaselineContextRelevance is < 0 or > 1 || + assessment.CandidateContextRelevance is < 0 or > 1 || + assessment.BaselineContextTokenDensity < 0 || + assessment.CandidateContextTokenDensity < 0 || + assessment.QueriesImproved + assessment.QueriesUnchanged + assessment.QueriesDegraded == 0; + } + + private static IReadOnlyList MergeReviews( + SummaryQualificationSuite suite, + IReadOnlyList completed, + IReadOnlyList reviews, + List invalid) + { + if (reviews.Count == 0) + { + return []; + } + + var completedIds = completed.Select(sample => sample.SampleId).ToHashSet(StringComparer.Ordinal); + var reviewerIds = reviews.Select(review => review.ReviewerId).ToArray(); + if (reviewerIds.Distinct(StringComparer.Ordinal).Count() != reviewerIds.Length) + { + invalid.Add("Reviewer identifiers must be unique."); + } + + foreach (var review in reviews) + { + var ids = review.Samples.Select(sample => sample.SampleId).ToArray(); + if (ids.Distinct(StringComparer.Ordinal).Count() != ids.Length || + ids.Any(id => !completedIds.Contains(id)) || + completedIds.Any(id => !ids.Contains(id, StringComparer.Ordinal))) + { + invalid.Add($"Reviewer '{review.ReviewerId}' did not score every completed sample exactly once."); + } + } + + if (invalid.Count > 0) + { + return []; + } + + var merged = new List(completed.Count); + var itemsById = suite.Items.ToDictionary(item => item.Id, StringComparer.Ordinal); + foreach (var sample in completed) + { + var scores = reviews + .Select(review => review.Samples.Single(score => string.Equals(score.SampleId, sample.SampleId, StringComparison.Ordinal))) + .ToArray(); + if (scores.Any(score => + !string.Equals(score.ItemId, sample.ItemId, StringComparison.Ordinal) || + !itemsById.TryGetValue(sample.ItemId, out var corpusItem) || + score.RequiredFactCount != corpusItem.RequiredFacts.Count || + score.BehaviorFactCount != BehaviorFactCount(corpusItem) || + score.SupportedFactCount < 0 || + score.GeneratedFactCount < score.SupportedFactCount || + score.RequiredFactPresentCount < 0 || + score.RequiredFactCount < score.RequiredFactPresentCount || + score.BehaviorFactPresentCount < 0 || + score.BehaviorFactCount < score.BehaviorFactPresentCount || + score.UsefulSupportedFactCount < 0 || + score.UsefulSupportedFactCount > score.SupportedFactCount || + score.TagTruePositiveCount < 0 || + score.TagFalsePositiveCount < 0 || + score.TagFalseNegativeCount < 0 || + score.SafetyFailureCount < 0)) + { + invalid.Add($"Sample '{sample.SampleId}' contains invalid reviewer counts."); + continue; + } + + merged.Add(new SummaryQualificationSampleReview( + sample.SampleId, + sample.ItemId, + scores.Min(score => score.SupportedFactCount), + scores.Max(score => score.GeneratedFactCount), + scores.Max(score => score.RequiredFactCount), + scores.Min(score => score.RequiredFactPresentCount), + scores.Any(score => score.CriticalHallucination), + scores.Any(score => score.Contradiction), + scores.All(score => score.PurposeAccurate), + scores.Max(score => score.BehaviorFactCount), + scores.Min(score => score.BehaviorFactPresentCount), + scores.Min(score => score.UsefulSupportedFactCount), + scores.All(score => score.OverallFactuallyCorrect), + scores.All(score => score.ConfidenceAppropriate), + scores.Min(score => score.TagTruePositiveCount), + scores.Max(score => score.TagFalsePositiveCount), + scores.Max(score => score.TagFalseNegativeCount), + scores.Max(score => score.SafetyFailureCount), + string.Join("; ", scores.Select(score => score.Limitation).Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal)))); + } + + return merged; + } + + private static int BehaviorFactCount(SummaryQualificationCorpusItem item) + { + return item.ImportantInputs.Count + + item.ImportantOutputs.Count + + item.SideEffects.Count + + item.FailureBehavior.Count; + } + + private static List FailedGates( + SummaryQualificationGates gates, + SummaryQualificationMetrics metrics, + IReadOnlyList reviews, + SummaryQualificationRetrievalEvidence? retrieval, + SummaryQualificationDeploymentEvidence? deployment) + { + var failed = new List(); + AddMinimum(failed, "First-attempt schema-valid rate", metrics.FirstAttemptSchemaRate, gates.MinimumFirstAttemptSchemaRate); + AddMinimum(failed, "Eventual completion rate", metrics.EventualCompletionRate, gates.MinimumEventualCompletionRate); + AddMaximum(failed, "Secret-canary leaks", metrics.SecretCanaryLeaks, gates.MaximumSecretCanaryLeaks); + AddMaximum(failed, "Critical hallucinations", metrics.CriticalHallucinations, gates.MaximumCriticalHallucinations); + AddMinimum(failed, "Atomic factual precision", metrics.AtomicFactualPrecision, gates.MinimumAtomicFactualPrecision); + AddMinimum(failed, "Required-fact recall", metrics.RequiredFactRecall, gates.MinimumRequiredFactRecall); + foreach (var (stratum, recall) in metrics.RequiredFactRecallByStratum) + { + AddMinimum(failed, $"Required-fact recall for stratum '{stratum}'", recall, gates.MinimumStratumRequiredFactRecall); + } + + AddMaximum(failed, "Reviewer safety failures", reviews.Sum(review => review.SafetyFailureCount), gates.MaximumSafetyFailures); + if (retrieval is not null) + { + AddMaximum(failed, "Retrieval Recall@k regression", retrieval.Baseline.RecallAtK - retrieval.Candidate.RecallAtK, gates.MaximumAbsoluteRetrievalRegression); + AddMaximum(failed, "Retrieval MRR regression", retrieval.Baseline.MeanReciprocalRank - retrieval.Candidate.MeanReciprocalRank, gates.MaximumAbsoluteRetrievalRegression); + AddMaximum(failed, "Retrieval nDCG regression", retrieval.Baseline.NdcgAtK - retrieval.Candidate.NdcgAtK, gates.MaximumAbsoluteRetrievalRegression); + AddMaximum(failed, "Candidate retrieval secret leaks", retrieval.Candidate.SecretLeaks, gates.MaximumSecretCanaryLeaks); + AddMaximum(failed, "Retrieval safety failures", retrieval.SafetyFailures.Count, gates.MaximumSafetyFailures); + AddMaximum(failed, "Unresolved material retrieval regressions", retrieval.Assessment.UnresolvedMaterialRegressions, 0); + if (!retrieval.Baseline.Comparable || !retrieval.Candidate.Comparable || !retrieval.Baseline.Passed || !retrieval.Candidate.Passed) + { + failed.Add("No-summary and candidate live-retrieval reports must be comparable and pass their declared suites."); + } + } + + if (gates.MaximumP95LatencyMs is { } maximumP95) + { + AddMaximum(failed, "P95 latency (ms)", metrics.P95LatencyMs, maximumP95); + } + + if (gates.MinimumNodesPerMinute is { } minimumRate) + { + AddMinimum(failed, "Nodes per minute", metrics.NodesPerMinute, minimumRate); + } + + if (gates.MaximumTotalTokens is { } maximumTokens) + { + if (metrics.TotalTokens is null) + { + failed.Add("Provider token usage is required by the deployment budget but was not reported."); + } + else + { + AddMaximum(failed, "Total tokens", metrics.TotalTokens.Value, maximumTokens); + } + } + + if (gates.MaximumTokensPerSuccessfulNode is { } maximumPerNode) + { + if (metrics.TokensPerSuccessfulNode is null) + { + failed.Add("Per-node token usage is required by the deployment budget but was not reported."); + } + else + { + AddMaximum(failed, "Tokens per successful node", metrics.TokensPerSuccessfulNode.Value, maximumPerNode); + } + } + + if (gates.MaximumTotalDurationMs is { } maximumDuration) + { + AddMaximum(failed, "Total measured duration (ms)", metrics.TotalDurationMs, maximumDuration); + } + + if (gates.MaximumRetryTokenOverheadRatio is { } maximumRetryOverhead) + { + AddMaximum(failed, "Retry token overhead ratio", metrics.RetryTokenOverheadRatio, maximumRetryOverhead); + } + + if (gates.MaximumPeakHostMemoryMb is { } maximumHostMemory) + { + if (deployment is null) + { + failed.Add("Peak host-memory evidence is required by the deployment budget."); + } + else + { + AddMaximum(failed, "Peak host memory (MB)", deployment.PeakHostMemoryMb, maximumHostMemory); + } + } + + if (gates.MaximumPeakAcceleratorMemoryMb is { } maximumAcceleratorMemory) + { + if (deployment?.PeakAcceleratorMemoryMb is not { } peakAcceleratorMemory) + { + failed.Add("Peak accelerator-memory evidence is required by the deployment budget."); + } + else + { + AddMaximum(failed, "Peak accelerator memory (MB)", peakAcceleratorMemory, maximumAcceleratorMemory); + } + } + + if (gates.MaximumOnlineCost is { } maximumOnlineCost) + { + if (deployment?.ProviderReportedCost is not { } providerCost) + { + failed.Add("Provider-reported cost is required by the online deployment budget."); + } + else if (providerCost > maximumOnlineCost) + { + failed.Add($"Provider-reported cost {providerCost:0.####} exceeds {maximumOnlineCost:0.####}."); + } + } + + return failed; + } + + private static IReadOnlyDictionary RequiredRecallByStratum( + SummaryQualificationSuite suite, + IReadOnlyList reviews) + { + var itemsById = suite.Items.ToDictionary(item => item.Id, StringComparer.Ordinal); + var result = new Dictionary(StringComparer.Ordinal); + foreach (var review in reviews) + { + if (!itemsById.TryGetValue(review.ItemId, out var item)) + { + continue; + } + + foreach (var stratum in ItemStrata(item)) + { + var current = result.GetValueOrDefault(stratum); + result[stratum] = (current.Present + review.RequiredFactPresentCount, current.Required + review.RequiredFactCount); + } + } + + return result.ToDictionary(pair => pair.Key, pair => Rate(pair.Value.Present, pair.Value.Required), StringComparer.Ordinal); + } + + private static IReadOnlyDictionary MetricsByStratum( + SummaryQualificationSuite suite, + IReadOnlyList reviews) + { + var itemsById = suite.Items.ToDictionary(item => item.Id, StringComparer.Ordinal); + var groups = new Dictionary>(StringComparer.Ordinal); + foreach (var review in reviews) + { + if (!itemsById.TryGetValue(review.ItemId, out var item)) + { + continue; + } + + foreach (var stratum in ItemStrata(item)) + { + if (!groups.TryGetValue(stratum, out var values)) + { + values = []; + groups[stratum] = values; + } + values.Add(review); + } + } + + return groups.ToDictionary( + pair => pair.Key, + pair => + { + var values = pair.Value; + return new SummaryQualificationStratumMetrics( + values.Count, + Rate(values.Sum(value => value.SupportedFactCount), values.Sum(value => value.GeneratedFactCount)), + Rate(values.Sum(value => value.RequiredFactPresentCount), values.Sum(value => value.RequiredFactCount)), + Rate(values.Count(value => value.PurposeAccurate), values.Count), + Rate(values.Sum(value => value.BehaviorFactPresentCount), values.Sum(value => value.BehaviorFactCount))); + }, + StringComparer.Ordinal); + } + + private static IEnumerable ItemStrata(SummaryQualificationCorpusItem item) + { + return item.Strata + .Append(item.Language) + .Append(item.NodeKind.ToString()) + .Append(item.InputSizeBand) + .Append(item.Criticality) + .Distinct(StringComparer.Ordinal); + } + + private static double IdenticalInputStability( + IReadOnlyList measured, + int repetitions) + { + var groups = measured.GroupBy(sample => sample.ItemId, StringComparer.Ordinal).ToArray(); + if (groups.Length == 0) + { + return 0; + } + + var stable = groups.Count(group => + group.Count() == repetitions && + group.All(sample => sample.Completed) && + group.Select(sample => CodeMeshHash.StableId( + sample.Summary, + string.Join("\u001e", sample.Tags), + sample.Confidence)) + .Distinct(StringComparer.Ordinal) + .Count() == 1); + return Rate(stable, groups.Length); + } + + private static double PerThousand(int facts, long? outputTokens) + { + return outputTokens is > 0 + ? (facts * 1000d) / outputTokens.Value + : 0; + } + + private static int CountCanaryLeaks(SummaryQualificationSample sample, IReadOnlyList canaries) + { + var output = $"{sample.Summary}\n{string.Join("\n", sample.Tags)}"; + return canaries.Count(canary => !string.IsNullOrEmpty(canary) && output.Contains(canary, StringComparison.Ordinal)); + } + + private static double Percentile(IEnumerable values, double percentile) + { + var ordered = values.Order().ToArray(); + if (ordered.Length == 0) + { + return 0; + } + + var index = (int)Math.Ceiling(percentile * ordered.Length) - 1; + return ordered[Math.Clamp(index, 0, ordered.Length - 1)]; + } + + private static double Rate(int numerator, int denominator) => denominator == 0 ? 0 : numerator / (double)denominator; + + private static double HarmonicMean(double left, double right) => left + right == 0 ? 0 : 2 * left * right / (left + right); + + private static void AddMinimum(List failed, string name, double actual, double minimum) + { + if (actual < minimum) + { + failed.Add($"{name} {actual:0.###} is below {minimum:0.###}."); + } + } + + private static void AddMaximum(List failed, string name, double actual, double maximum) + { + if (actual > maximum) + { + failed.Add($"{name} {actual:0.###} exceeds {maximum:0.###}."); + } + } +} + +internal static class Fingerprint +{ + public static string Suite(SummaryQualificationSuite suite) + { + var parts = new List + { + suite.SchemaVersion, + suite.Kind, + suite.Name, + suite.Version, + System.Text.Json.JsonSerializer.Serialize(suite.Gates) + }; + foreach (var item in suite.Items.OrderBy(item => item.Id, StringComparer.Ordinal)) + { + parts.AddRange([ + item.Id, + item.RepositoryId, + item.RepositoryCommit, + item.NodeId, + item.ContentHash, + item.Language, + item.NodeKind.ToString(), + item.Name, + item.Project, + item.FilePath, + item.StartLine.ToString(System.Globalization.CultureInfo.InvariantCulture), + item.EndLine.ToString(System.Globalization.CultureInfo.InvariantCulture), + item.InputSizeBand, + item.Criticality, + string.Join("\u001e", item.Strata.Order(StringComparer.Ordinal)), + string.Join("\u001e", item.RequiredFacts), + string.Join("\u001e", item.OptionalFacts), + string.Join("\u001e", item.ForbiddenClaims), + string.Join("\u001e", item.RequiredTags), + string.Join("\u001e", item.AcceptableTags), + item.Partition, + item.ExpectedResponsibility, + string.Join("\u001e", item.ImportantInputs), + string.Join("\u001e", item.ImportantOutputs), + string.Join("\u001e", item.SideEffects), + string.Join("\u001e", item.FailureBehavior), + item.ReferenceSummary, + string.Join("\u001e", (item.SecretCanaries ?? []).Order(StringComparer.Ordinal)) + ]); + } + + return CodeMeshHash.StableId(parts.ToArray()); + } + + public static string Profile(SummaryQualificationProfile profile) + { + return CodeMeshHash.StableId( + profile.SchemaVersion, + profile.Kind, + profile.Name, + profile.IntendedUse, + profile.EndpointClass, + profile.Provider, + profile.Model, + profile.ModelRevision, + profile.Quantization, + profile.NumericPrecision, + profile.ChatTemplateHash, + profile.ContextLength.ToString(System.Globalization.CultureInfo.InvariantCulture), + profile.MaxInputCharacters.ToString(System.Globalization.CultureInfo.InvariantCulture), + profile.ReasoningTokenReserve.ToString(System.Globalization.CultureInfo.InvariantCulture), + profile.MaxCompletionTokens.ToString(System.Globalization.CultureInfo.InvariantCulture), + profile.Repetitions.ToString(System.Globalization.CultureInfo.InvariantCulture), + Map(profile.GenerationSettings), + Map(profile.Host), + Map(profile.OnlinePolicy), + string.Join("\u001e", (profile.Conditions ?? []).Order(StringComparer.Ordinal))); + } + + public static string RepositoryIdentity(IReadOnlyList items) + { + return CodeMeshHash.StableId(items + .Select(item => $"{item.RepositoryId}\u001e{item.RepositoryCommit}") + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToArray()); + } + + private static string Map(IReadOnlyDictionary? values) + { + return values is null + ? string.Empty + : string.Join("\u001e", values.OrderBy(pair => pair.Key, StringComparer.Ordinal).Select(pair => $"{pair.Key}={pair.Value}")); + } +} diff --git a/src/CodeMesh.Ingestion/Summary/SummaryPrompt.cs b/src/CodeMesh.Ingestion/Summary/SummaryPrompt.cs index 4a6d548..a30d474 100644 --- a/src/CodeMesh.Ingestion/Summary/SummaryPrompt.cs +++ b/src/CodeMesh.Ingestion/Summary/SummaryPrompt.cs @@ -95,8 +95,13 @@ public static CodeSummaryResult ParseResult(string? content) throw new InvalidOperationException("Summary provider returned JSON without a summary."); } + if (parsed.Tags is null) + { + throw new InvalidOperationException("Summary provider returned JSON without tags."); + } + var confidence = NormalizeConfidence(parsed.Confidence); - var tags = (parsed.Tags ?? []) + var tags = parsed.Tags .Select(tag => tag.Trim()) .Where(tag => !string.IsNullOrWhiteSpace(tag)) .Distinct(StringComparer.OrdinalIgnoreCase) @@ -130,12 +135,17 @@ private static string NormalizeConfidence(JsonElement? confidence) { if (confidence is not { } value) { - return "medium"; + throw new InvalidOperationException("Summary provider returned JSON without confidence."); } if (value.ValueKind == JsonValueKind.Number && value.TryGetDouble(out var score)) { + if (score is < 0 or > 1) + { + throw new InvalidOperationException("Summary provider returned confidence outside the range 0 to 1."); + } + return score switch { >= 0.8 => "high", @@ -146,12 +156,17 @@ private static string NormalizeConfidence(JsonElement? confidence) if (value.ValueKind != JsonValueKind.String) { - return "medium"; + throw new InvalidOperationException("Summary provider returned an invalid confidence value."); } var raw = value.GetString()?.Trim(); if (double.TryParse(raw, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out score)) { + if (score is < 0 or > 1) + { + throw new InvalidOperationException("Summary provider returned confidence outside the range 0 to 1."); + } + return score switch { >= 0.8 => "high", @@ -164,7 +179,8 @@ private static string NormalizeConfidence(JsonElement? confidence) { "high" => "high", "low" => "low", - _ => "medium" + "medium" => "medium", + _ => throw new InvalidOperationException("Summary provider returned an unsupported confidence label.") }; } diff --git a/src/CodeMesh.Parser.CSharp/CSharpParseService.cs b/src/CodeMesh.Parser.CSharp/CSharpParseService.cs index a9a1421..a6c1dbb 100644 --- a/src/CodeMesh.Parser.CSharp/CSharpParseService.cs +++ b/src/CodeMesh.Parser.CSharp/CSharpParseService.cs @@ -38,9 +38,9 @@ public sealed class CSharpParseService public static ParserCapability Capability { get; } = new( "codemesh-parser-csharp", "csharp", - "0.2.0", + "0.2.3", [".cs", ".csproj", ".sln", ".slnx"], - ["repositoryRoot", "solutionPath", "projectPath"]); + ["repositoryRoot", "solutionPath", "projectPath", "pathFilter"]); public async Task ParseAsync( ParseRequest request, @@ -168,7 +168,18 @@ private static async Task> LoadProjectContext var projectPath = ResolveInputPath(request.ProjectPath, repositoryRoot); if (projectPath is not null && File.Exists(projectPath)) { - var loadedProject = await TryLoadProjectAsync(projectPath, repositoryRoot, diagnostics, cancellationToken) + if (IsGeneratedOutputPath(repositoryRoot, projectPath, ManifestPathFilter(request.PathFilter))) + { + throw new InvalidOperationException( + $"Explicit project path is outside the repository path policy: {projectPath}"); + } + + var loadedProject = await TryLoadProjectAsync( + projectPath, + repositoryRoot, + request.PathFilter, + diagnostics, + cancellationToken) .ConfigureAwait(false); if (loadedProject.Count > 0) { @@ -179,7 +190,18 @@ private static async Task> LoadProjectContext var solutionPath = ResolveInputPath(request.SolutionPath, repositoryRoot); if (solutionPath is not null && File.Exists(solutionPath)) { - var loadedSolution = await TryLoadSolutionAsync(solutionPath, repositoryRoot, diagnostics, cancellationToken) + if (IsGeneratedOutputPath(repositoryRoot, solutionPath, ManifestPathFilter(request.PathFilter))) + { + throw new InvalidOperationException( + $"Explicit solution path is outside the repository path policy: {solutionPath}"); + } + + var loadedSolution = await TryLoadSolutionAsync( + solutionPath, + repositoryRoot, + request.PathFilter, + diagnostics, + cancellationToken) .ConfigureAwait(false); if (loadedSolution.Count > 0) { @@ -190,11 +212,17 @@ private static async Task> LoadProjectContext var discoveredSolution = Directory .EnumerateFiles(repositoryRoot, "*.sln", SearchOption.TopDirectoryOnly) .Concat(Directory.EnumerateFiles(repositoryRoot, "*.slnx", SearchOption.TopDirectoryOnly)) + .Where(path => !IsGeneratedOutputPath(repositoryRoot, path, ManifestPathFilter(request.PathFilter))) .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) .FirstOrDefault(); if (discoveredSolution is not null) { - var loadedSolution = await TryLoadSolutionAsync(discoveredSolution, repositoryRoot, diagnostics, cancellationToken) + var loadedSolution = await TryLoadSolutionAsync( + discoveredSolution, + repositoryRoot, + request.PathFilter, + diagnostics, + cancellationToken) .ConfigureAwait(false); if (loadedSolution.Count > 0) { @@ -202,15 +230,19 @@ private static async Task> LoadProjectContext } } - var discoveredProjects = Directory - .EnumerateFiles(repositoryRoot, "*.csproj", SearchOption.AllDirectories) - .Where(file => !IsGeneratedOutputPath(repositoryRoot, file)) + var discoveredProjects = RepositoryPathPolicy + .EnumerateFiles(repositoryRoot, "*.csproj", ManifestPathFilter(request.PathFilter)) .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) .Take(2) .ToArray(); if (discoveredProjects.Length == 1) { - var loadedProject = await TryLoadProjectAsync(discoveredProjects[0], repositoryRoot, diagnostics, cancellationToken) + var loadedProject = await TryLoadProjectAsync( + discoveredProjects[0], + repositoryRoot, + request.PathFilter, + diagnostics, + cancellationToken) .ConfigureAwait(false); if (loadedProject.Count > 0) { @@ -230,6 +262,7 @@ private static async Task> LoadProjectContext private static async Task> TryLoadProjectAsync( string projectPath, string repositoryRoot, + RepositoryPathFilter? pathFilter, List diagnostics, CancellationToken cancellationToken) { @@ -238,12 +271,18 @@ private static async Task> TryLoadProjectAsyn EnsureMSBuildRegistered(); var workspaceDiagnostics = new List(); - using var workspace = CreateWorkspace(workspaceDiagnostics); + using var outputs = new DesignTimeOutputs(); + using var workspace = CreateWorkspace(workspaceDiagnostics, outputs.Root); var project = await workspace.OpenProjectAsync(projectPath, cancellationToken: cancellationToken) .ConfigureAwait(false); AddWorkspaceDiagnostics(workspaceDiagnostics, diagnostics); - var contexts = await CreateProjectParseContextsAsync([project], repositoryRoot, diagnostics, cancellationToken) + var contexts = await CreateProjectParseContextsAsync( + [project], + repositoryRoot, + pathFilter, + diagnostics, + cancellationToken) .ConfigureAwait(false); if (contexts.Count > 0) { @@ -268,6 +307,7 @@ private static async Task> TryLoadProjectAsyn private static async Task> TryLoadSolutionAsync( string solutionPath, string repositoryRoot, + RepositoryPathFilter? pathFilter, List diagnostics, CancellationToken cancellationToken) { @@ -276,12 +316,18 @@ private static async Task> TryLoadSolutionAsy EnsureMSBuildRegistered(); var workspaceDiagnostics = new List(); - using var workspace = CreateWorkspace(workspaceDiagnostics); + using var outputs = new DesignTimeOutputs(); + using var workspace = CreateWorkspace(workspaceDiagnostics, outputs.Root); var solution = await workspace.OpenSolutionAsync(solutionPath, cancellationToken: cancellationToken) .ConfigureAwait(false); AddWorkspaceDiagnostics(workspaceDiagnostics, diagnostics); - var contexts = await CreateProjectParseContextsAsync(solution.Projects, repositoryRoot, diagnostics, cancellationToken) + var contexts = await CreateProjectParseContextsAsync( + solution.Projects, + repositoryRoot, + pathFilter, + diagnostics, + cancellationToken) .ConfigureAwait(false); if (contexts.Count > 0) { @@ -303,16 +349,26 @@ private static async Task> TryLoadSolutionAsy } } - private static MSBuildWorkspace CreateWorkspace(List diagnostics) + private static MSBuildWorkspace CreateWorkspace(List diagnostics, string outputRoot) { var workspace = MSBuildWorkspace.Create(new Dictionary { - ["Configuration"] = "Debug" + ["Configuration"] = "Debug", + ["CodeMeshDesignTimeOutputRoot"] = outputRoot }); workspace.RegisterWorkspaceFailedHandler(args => diagnostics.Add(args.Diagnostic)); return workspace; } + private sealed class DesignTimeOutputs : IDisposable + { + private readonly DirectoryInfo directory = Directory.CreateTempSubdirectory("codemesh-msbuild-"); + + public string Root => directory.FullName + Path.DirectorySeparatorChar; + + public void Dispose() => directory.Delete(recursive: true); + } + private static void EnsureMSBuildRegistered() { if (MSBuildLocator.IsRegistered) @@ -345,6 +401,7 @@ private static void AddWorkspaceDiagnostics( private static async Task> CreateProjectParseContextsAsync( IEnumerable projects, string repositoryRoot, + RepositoryPathFilter? pathFilter, List diagnostics, CancellationToken cancellationToken) { @@ -369,7 +426,11 @@ private static async Task> CreateProjectParse { cancellationToken.ThrowIfCancellationRequested(); - var context = await TryCreateWorkspaceFileContextAsync(document, repositoryRoot, cancellationToken) + var context = await TryCreateWorkspaceFileContextAsync( + document, + repositoryRoot, + pathFilter, + cancellationToken) .ConfigureAwait(false); if (context is not null) { @@ -377,8 +438,15 @@ private static async Task> CreateProjectParse } } + files.Sort((left, right) => CompareSourcePaths(left.RelativePath, right.RelativePath)); + if (files.Count > 0) { + var includedTrees = files + .Select(file => file.SyntaxTree) + .ToHashSet(); + compilation = compilation.RemoveSyntaxTrees( + compilation.SyntaxTrees.Where(tree => !includedTrees.Contains(tree))); contexts.Add(new ProjectParseContext(project.Name, compilation, files)); } } @@ -389,6 +457,7 @@ private static async Task> CreateProjectParse private static async Task TryCreateWorkspaceFileContextAsync( Document document, string repositoryRoot, + RepositoryPathFilter? pathFilter, CancellationToken cancellationToken) { if (document.SourceCodeKind != SourceCodeKind.Regular || @@ -399,7 +468,8 @@ private static async Task> CreateProjectParse } var fullPath = Path.GetFullPath(document.FilePath); - if (!IsUnderDirectory(repositoryRoot, fullPath) || IsGeneratedOutputPath(repositoryRoot, fullPath)) + if (!IsUnderDirectory(repositoryRoot, fullPath) || + IsGeneratedOutputPath(repositoryRoot, fullPath, pathFilter)) { return null; } @@ -422,7 +492,7 @@ private static async Task> LoadFallbackProjec { var projectName = ResolveProjectName(request, repositoryRoot); var fileContexts = new List(); - var files = EnumerateCSharpFiles(repositoryRoot).ToArray(); + var files = EnumerateCSharpFiles(repositoryRoot, request.PathFilter).ToArray(); if (files.Length == 0) { @@ -531,10 +601,35 @@ private static void EmitFileNode( now)); } - private static IEnumerable EnumerateCSharpFiles(string repositoryRoot) + private static IEnumerable EnumerateCSharpFiles( + string repositoryRoot, + RepositoryPathFilter? pathFilter) + { + return RepositoryPathPolicy.EnumerateFiles(repositoryRoot, "*.cs", pathFilter) + .OrderBy(file => file, Comparer.Create(CompareSourcePaths)); + } + + private static int CompareSourcePaths(string left, string right) { - return Directory.EnumerateFiles(repositoryRoot, "*.cs", SearchOption.AllDirectories) - .Where(file => !IsGeneratedOutputPath(repositoryRoot, file)); + var leftStem = Path.GetFileNameWithoutExtension(left); + var rightStem = Path.GetFileNameWithoutExtension(right); + var segmentComparison = leftStem.Count(character => character == '.') + .CompareTo(rightStem.Count(character => character == '.')); + if (segmentComparison != 0) + { + return segmentComparison; + } + + var lengthComparison = leftStem.Length.CompareTo(rightStem.Length); + if (lengthComparison != 0) + { + return lengthComparison; + } + + var insensitiveComparison = StringComparer.OrdinalIgnoreCase.Compare(left, right); + return insensitiveComparison != 0 + ? insensitiveComparison + : StringComparer.Ordinal.Compare(left, right); } private static string FileNodeId(string relativePath) @@ -1455,9 +1550,19 @@ private static IEnumerable CreateMetadataReferences() } } - private static bool IsGeneratedOutputPath(string repositoryRoot, string file) + private static bool IsGeneratedOutputPath( + string repositoryRoot, + string file, + RepositoryPathFilter? pathFilter = null) + { + return RepositoryPathPolicy.IsIgnoredPath(repositoryRoot, file, pathFilter); + } + + private static RepositoryPathFilter ManifestPathFilter(RepositoryPathFilter? pathFilter) { - return RepositoryPathPolicy.IsIgnoredPath(repositoryRoot, file); + return new RepositoryPathFilter( + DenyPatterns: pathFilter?.DenyPatterns, + ExcludeKnownSecretFiles: pathFilter?.ExcludeKnownSecretFiles ?? true); } private static bool IsUnderDirectory(string rootPath, string candidatePath) diff --git a/src/CodeMesh.Parser.CSharp/CodeMesh.DesignTime.targets b/src/CodeMesh.Parser.CSharp/CodeMesh.DesignTime.targets new file mode 100644 index 0000000..0125d8b --- /dev/null +++ b/src/CodeMesh.Parser.CSharp/CodeMesh.DesignTime.targets @@ -0,0 +1,11 @@ + + + + <_CodeMeshProjectOutputKey>$([MSBuild]::StableStringHash('$(MSBuildProjectFullPath)|$(Configuration)|$(Platform)|$(TargetFramework)|$(RuntimeIdentifier)', 'Sha256')) + <_CodeMeshProjectOutputRoot>$(CodeMeshDesignTimeOutputRoot)$(_CodeMeshProjectOutputKey)/ + $(_CodeMeshProjectOutputRoot)obj/ + $(_CodeMeshProjectOutputRoot)bin/ + $(OutputPath) + + diff --git a/src/CodeMesh.Parser.CSharp/CodeMesh.Parser.CSharp.csproj b/src/CodeMesh.Parser.CSharp/CodeMesh.Parser.CSharp.csproj index e4f0390..564484d 100644 --- a/src/CodeMesh.Parser.CSharp/CodeMesh.Parser.CSharp.csproj +++ b/src/CodeMesh.Parser.CSharp/CodeMesh.Parser.CSharp.csproj @@ -12,6 +12,10 @@ + + + + net10.0 enable diff --git a/src/CodeMesh.Parser.CSharp/Dockerfile b/src/CodeMesh.Parser.CSharp/Dockerfile index 6e517c0..932435f 100644 --- a/src/CodeMesh.Parser.CSharp/Dockerfile +++ b/src/CodeMesh.Parser.CSharp/Dockerfile @@ -3,16 +3,22 @@ WORKDIR /src COPY CodeMesh.sln ./ COPY NuGet.config ./ +COPY Directory.Build.props VERSION ./ COPY src/CodeMesh.Domain/ ./src/CodeMesh.Domain/ COPY src/CodeMesh.Parser.CSharp/ ./src/CodeMesh.Parser.CSharp/ RUN dotnet restore src/CodeMesh.Parser.CSharp/CodeMesh.Parser.CSharp.csproj --locked-mode -RUN dotnet publish src/CodeMesh.Parser.CSharp/CodeMesh.Parser.CSharp.csproj -c Release -o /app/publish --no-restore +RUN dotnet publish src/CodeMesh.Parser.CSharp/CodeMesh.Parser.CSharp.csproj -c Release -p:PublishDir=/app/publish/ --no-restore -FROM mcr.microsoft.com/dotnet/aspnet:10.0 +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS runtime WORKDIR /app COPY --from=build /app/publish ./ +RUN codemesh_sdk_version="$(dotnet --version)" \ + && codemesh_import_dir="/usr/share/dotnet/sdk/$codemesh_sdk_version/Current/Microsoft.Common.targets/ImportBefore" \ + && mkdir -p "$codemesh_import_dir" \ + && cp /app/CodeMesh.DesignTime.targets "$codemesh_import_dir/CodeMesh.DesignTime.targets" + ENV ASPNETCORE_URLS=http://+:8080 EXPOSE 8080 diff --git a/src/CodeMesh.Parser.Deployment/DeploymentParseService.cs b/src/CodeMesh.Parser.Deployment/DeploymentParseService.cs index 1097651..b9f2470 100644 --- a/src/CodeMesh.Parser.Deployment/DeploymentParseService.cs +++ b/src/CodeMesh.Parser.Deployment/DeploymentParseService.cs @@ -10,9 +10,9 @@ public sealed partial class DeploymentParseService public static ParserCapability Capability { get; } = new( "codemesh-parser-deployment", "deployment", - "0.1.0", + "0.1.1", ["Dockerfile", ".dockerfile", ".yml", ".yaml"], - ["repositoryRoot"]); + ["repositoryRoot", "pathFilter"]); public async Task ParseAsync( ParseRequest request, @@ -30,7 +30,7 @@ public async Task ParseAsync( var diagnostics = new List(); var parsedAt = DateTimeOffset.UtcNow; - foreach (var file in EnumerateDeploymentFiles(repositoryRoot)) + foreach (var file in EnumerateDeploymentFiles(repositoryRoot, request.PathFilter)) { cancellationToken.ThrowIfCancellationRequested(); var relativePath = NormalizePath(Path.GetRelativePath(repositoryRoot, file)); @@ -511,10 +511,12 @@ [new ParserDiagnostic("CMDEP000", message, "error")], DateTimeOffset.UtcNow); } - private static IEnumerable EnumerateDeploymentFiles(string repositoryRoot) + private static IEnumerable EnumerateDeploymentFiles( + string repositoryRoot, + RepositoryPathFilter? pathFilter) { - return Directory.EnumerateFiles(repositoryRoot, "*", SearchOption.AllDirectories) - .Where(file => !IsExcluded(repositoryRoot, file) && (IsDockerfile(file) || IsComposeFile(file))) + return RepositoryPathPolicy.EnumerateFiles(repositoryRoot, "*", pathFilter) + .Where(file => IsDockerfile(file) || IsComposeFile(file)) .OrderBy(file => file, StringComparer.OrdinalIgnoreCase); } @@ -535,11 +537,6 @@ private static bool IsComposeFile(string file) string.Equals(name, "compose.yaml", StringComparison.OrdinalIgnoreCase); } - private static bool IsExcluded(string repositoryRoot, string file) - { - return RepositoryPathPolicy.IsIgnoredPath(repositoryRoot, file); - } - private static void AddJoined(Dictionary metadata, string key, IEnumerable values) { var joined = string.Join(",", values.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.OrdinalIgnoreCase)); diff --git a/src/CodeMesh.Parser.Markdown/MarkdownParseService.cs b/src/CodeMesh.Parser.Markdown/MarkdownParseService.cs index 945619d..893ff18 100644 --- a/src/CodeMesh.Parser.Markdown/MarkdownParseService.cs +++ b/src/CodeMesh.Parser.Markdown/MarkdownParseService.cs @@ -10,9 +10,9 @@ public sealed partial class MarkdownParseService public static ParserCapability Capability { get; } = new( "codemesh-parser-markdown", "markdown", - "0.1.0", + "0.1.1", [".md", ".markdown"], - ["repositoryRoot"]); + ["repositoryRoot", "pathFilter"]); public async Task ParseAsync( ParseRequest request, @@ -30,7 +30,7 @@ public async Task ParseAsync( var diagnostics = new List(); var parsedAt = DateTimeOffset.UtcNow; - foreach (var file in EnumerateMarkdownFiles(repositoryRoot)) + foreach (var file in EnumerateMarkdownFiles(repositoryRoot, request.PathFilter)) { cancellationToken.ThrowIfCancellationRequested(); var relativePath = NormalizePath(Path.GetRelativePath(repositoryRoot, file)); @@ -169,10 +169,12 @@ [new ParserDiagnostic("CMMD000", message, "error")], DateTimeOffset.UtcNow); } - private static IEnumerable EnumerateMarkdownFiles(string repositoryRoot) + private static IEnumerable EnumerateMarkdownFiles( + string repositoryRoot, + RepositoryPathFilter? pathFilter) { - return Directory.EnumerateFiles(repositoryRoot, "*.*", SearchOption.AllDirectories) - .Where(file => IsMarkdownFile(file) && !IsExcluded(repositoryRoot, file)) + return RepositoryPathPolicy.EnumerateFiles(repositoryRoot, "*.*", pathFilter) + .Where(IsMarkdownFile) .OrderBy(file => file, StringComparer.OrdinalIgnoreCase); } @@ -183,11 +185,6 @@ private static bool IsMarkdownFile(string file) string.Equals(extension, ".markdown", StringComparison.OrdinalIgnoreCase); } - private static bool IsExcluded(string repositoryRoot, string file) - { - return RepositoryPathPolicy.IsIgnoredPath(repositoryRoot, file); - } - private static string AnchorFor(string title) { var lower = title.Trim().ToLowerInvariant(); diff --git a/src/CodeMesh.Parser.Python/PythonParseService.cs b/src/CodeMesh.Parser.Python/PythonParseService.cs index f9df04e..e9cc90a 100644 --- a/src/CodeMesh.Parser.Python/PythonParseService.cs +++ b/src/CodeMesh.Parser.Python/PythonParseService.cs @@ -10,9 +10,9 @@ public sealed partial class PythonParseService public static ParserCapability Capability { get; } = new( "codemesh-parser-python", "python", - "0.1.0", + "0.1.1", [".py"], - ["repositoryRoot"]); + ["repositoryRoot", "pathFilter"]); public async Task ParseAsync( ParseRequest request, @@ -30,7 +30,7 @@ public async Task ParseAsync( var diagnostics = new List(); var parsedAt = DateTimeOffset.UtcNow; - foreach (var file in EnumeratePythonFiles(repositoryRoot)) + foreach (var file in EnumeratePythonFiles(repositoryRoot, request.PathFilter)) { cancellationToken.ThrowIfCancellationRequested(); var relativePath = NormalizePath(Path.GetRelativePath(repositoryRoot, file)); @@ -88,11 +88,24 @@ private static void ParseFile( contents.Add(new CodeContent(fileHash, text, "text/x-python", parsedAt, parsedAt)); var lines = text.Replace("\r\n", "\n").Replace('\r', '\n').Split('\n'); - var stack = new List { new(fileNodeId, string.Empty, -1, CodeNodeKind.File) }; + var imports = ParseImports( + repositoryRoot, + relativePath, + lines, + fileNodeId, + parsedAt, + nodes, + relationships, + contents); + var stack = new List + { + new(fileNodeId, string.Empty, -1, CodeNodeKind.File, 1, 1, Math.Max(1, lines.Length), Path.GetFileName(relativePath)) + }; + var scopes = new List(); for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) { var line = lines[lineIndex]; - var match = DefinitionRegex().Match(line); + var match = MatchDefinition(lines, lineIndex, out var headerEndIndex); if (!match.Success) { continue; @@ -108,16 +121,32 @@ private static void ParseFile( var kindText = match.Groups["kind"].Value; var nodeKind = string.Equals(kindText, "class", StringComparison.Ordinal) ? CodeNodeKind.Class - : CodeNodeKind.Method; + : IsTestFunction(name, Decorators(lines, lineIndex)) + ? CodeNodeKind.TestCase + : CodeNodeKind.Method; var parent = stack[^1]; var qualifiedName = string.IsNullOrWhiteSpace(parent.QualifiedName) ? name : $"{parent.QualifiedName}.{name}"; var nodeId = $"python:{nodeKind.ToString().ToLowerInvariant()}:{ModuleName(relativePath)}.{qualifiedName}"; - var content = ExtractBlock(lines, lineIndex, indent); + var content = ExtractBlock(lines, lineIndex, headerEndIndex, indent); var contentHash = CodeMeshHash.Sha256Hex(content); var startLine = lineIndex + 1; var endLine = startLine + Math.Max(0, CountLines(content) - 1); + var metadata = new Dictionary + { + ["filePath"] = relativePath, + ["module"] = ModuleName(relativePath), + ["qualifiedName"] = qualifiedName, + ["parentNodeId"] = parent.NodeId, + ["async"] = match.Groups["async"].Success.ToString(), + ["decorators"] = string.Join(' ', Decorators(lines, lineIndex)) + }; + if (nodeKind == CodeNodeKind.TestCase) + { + metadata["testRole"] = "case"; + metadata["testFramework"] = "python"; + } var node = new CodeNode( nodeId, nodeId, @@ -127,14 +156,7 @@ private static void ParseFile( ProjectName(repositoryRoot), new SourceSpan(relativePath, startLine, indent + 1, endLine, 1), contentHash, - new Dictionary - { - ["filePath"] = relativePath, - ["module"] = ModuleName(relativePath), - ["qualifiedName"] = qualifiedName, - ["parentNodeId"] = parent.NodeId, - ["async"] = match.Groups["async"].Success.ToString() - }); + metadata); nodes.Add(node); contents.Add(new CodeContent(contentHash, content, "text/x-python-fragment", parsedAt, parsedAt)); relationships.Add(new CodeRelationship( @@ -144,8 +166,216 @@ private static void ParseFile( CodeRelationshipKind.Contains, null, new Dictionary { ["filePath"] = relativePath })); - stack.Add(new PythonScope(nodeId, qualifiedName, indent, nodeKind)); + var scope = new PythonScope( + nodeId, + qualifiedName, + indent, + nodeKind, + startLine, + headerEndIndex + 1, + endLine, + name); + stack.Add(scope); + scopes.Add(scope); + lineIndex = headerEndIndex; + } + + AddSemanticRelationships( + relativePath, + lines, + fileNodeId, + scopes, + imports, + relationships); + } + + private static IReadOnlyList ParseImports( + string repositoryRoot, + string relativePath, + string[] lines, + string fileNodeId, + DateTimeOffset parsedAt, + List nodes, + List relationships, + List contents) + { + var imports = new List(); + var seenImports = new HashSet(StringComparer.Ordinal); + for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + { + var line = lines[lineIndex]; + var fromMatch = FromImportRegex().Match(line); + var importMatch = ImportRegex().Match(line); + if (!fromMatch.Success && !importMatch.Success) + { + continue; + } + + var module = fromMatch.Success ? fromMatch.Groups["module"].Value : string.Empty; + var items = (fromMatch.Success ? fromMatch.Groups["items"] : importMatch.Groups["items"]) + .Value + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var item in items) + { + var parts = AliasRegex().Match(item); + if (!parts.Success) + { + continue; + } + var importedName = parts.Groups["name"].Value; + var importedModule = fromMatch.Success ? module : importedName; + var alias = parts.Groups["alias"].Success + ? parts.Groups["alias"].Value + : fromMatch.Success + ? importedName + : importedName.Split('.').First(); + var importKey = $"{importedModule}\0{importedName}\0{alias}"; + if (!seenImports.Add(importKey)) + { + continue; + } + var nodeId = $"python:import:{CodeMeshHash.StableId(relativePath, importedModule, importedName, alias)}"; + var content = line.Trim(); + var contentHash = CodeMeshHash.Sha256Hex(content); + var node = new CodeNode( + nodeId, + nodeId, + CodeNodeKind.Declaration, + fromMatch.Success ? importedName : importedModule, + "python", + ProjectName(repositoryRoot), + new SourceSpan(relativePath, lineIndex + 1, LeadingSpaces(line) + 1, lineIndex + 1, line.Length + 1), + contentHash, + new Dictionary + { + ["filePath"] = relativePath, + ["module"] = ModuleName(relativePath), + ["pythonKind"] = "import", + ["importModule"] = importedModule, + ["importedName"] = fromMatch.Success ? importedName : string.Empty, + ["alias"] = alias, + ["parentNodeId"] = fileNodeId + }); + nodes.Add(node); + contents.Add(new CodeContent(contentHash, content, "text/x-python-fragment", parsedAt, parsedAt)); + relationships.Add(new CodeRelationship( + CodeMeshHash.StableId("python", "contains", fileNodeId, nodeId), + fileNodeId, + nodeId, + CodeRelationshipKind.Contains, + null, + new Dictionary { ["filePath"] = relativePath })); + imports.Add(new PythonImport(node, importedModule, fromMatch.Success ? importedName : null, alias)); + } } + return imports; + } + + private static void AddSemanticRelationships( + string relativePath, + string[] lines, + string fileNodeId, + IReadOnlyList scopes, + IReadOnlyList imports, + List relationships) + { + var definitions = scopes + .GroupBy(scope => scope.Name, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.ToArray(), StringComparer.Ordinal); + var importsByAlias = imports + .GroupBy(item => item.Alias, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.ToArray(), StringComparer.Ordinal); + + foreach (var item in imports) + { + var targetName = item.ImportedName ?? item.Module.Split('.').Last(); + if (definitions.TryGetValue(targetName, out var targets) && targets.Length == 1) + { + relationships.Add(PythonRelationship( + item.Node.Id, + targets[0].NodeId, + CodeRelationshipKind.References, + targetName, + relativePath)); + } + } + + foreach (var scope in scopes.Where(scope => scope.Kind is CodeNodeKind.Method or CodeNodeKind.TestCase)) + { + var startIndex = Math.Min(scope.HeaderEndLine, lines.Length); + var count = Math.Max(0, Math.Min(scope.EndLine, lines.Length) - startIndex); + var body = string.Join('\n', lines.Skip(startIndex).Take(count)); + foreach (Match call in CallRegex().Matches(body)) + { + var owner = call.Groups["owner"].Success ? call.Groups["owner"].Value : null; + var name = call.Groups["name"].Value; + if (IgnoredCallNames.Contains(name)) + { + continue; + } + + if (owner is null && definitions.TryGetValue(name, out var targets) && targets.Length == 1) + { + relationships.Add(PythonRelationship( + scope.NodeId, + targets[0].NodeId, + CodeRelationshipKind.Invokes, + name, + relativePath)); + continue; + } + + var importAlias = owner ?? name; + if (importsByAlias.TryGetValue(importAlias, out var imported) && imported.Length == 1) + { + relationships.Add(PythonRelationship( + scope.NodeId, + imported[0].Node.Id, + CodeRelationshipKind.Invokes, + owner is null ? imported[0].ImportedName ?? name : name, + relativePath)); + } + } + } + } + + private static CodeRelationship PythonRelationship( + string sourceNodeId, + string targetNodeId, + CodeRelationshipKind kind, + string label, + string relativePath) + { + return new CodeRelationship( + CodeMeshHash.StableId("python", kind.ToString(), sourceNodeId, targetNodeId, label), + sourceNodeId, + targetNodeId, + kind, + label, + new Dictionary { ["filePath"] = relativePath }); + } + + private static IReadOnlyList Decorators(string[] lines, int definitionIndex) + { + var decorators = new List(); + for (var index = definitionIndex - 1; index >= 0; index--) + { + var trimmed = lines[index].Trim(); + if (!trimmed.StartsWith('@')) + { + break; + } + decorators.Insert(0, trimmed); + } + return decorators; + } + + private static bool IsTestFunction(string name, IReadOnlyList decorators) + { + return name.StartsWith("test_", StringComparison.Ordinal) || + decorators.Any(decorator => + decorator.Contains("pytest", StringComparison.OrdinalIgnoreCase) || + decorator.Contains("unittest", StringComparison.OrdinalIgnoreCase)); } private static ParseResult Failure(ParseRequest request, string message) @@ -160,21 +390,21 @@ [new ParserDiagnostic("CMPY000", message, "error")], DateTimeOffset.UtcNow); } - private static IEnumerable EnumeratePythonFiles(string repositoryRoot) + private static IEnumerable EnumeratePythonFiles( + string repositoryRoot, + RepositoryPathFilter? pathFilter) { - return Directory.EnumerateFiles(repositoryRoot, "*.py", SearchOption.AllDirectories) - .Where(file => !IsExcluded(repositoryRoot, file)) + return RepositoryPathPolicy.EnumerateFiles(repositoryRoot, "*.py", pathFilter) .OrderBy(file => file, StringComparer.OrdinalIgnoreCase); } - private static bool IsExcluded(string repositoryRoot, string file) + private static string ExtractBlock( + string[] lines, + int startIndex, + int headerEndIndex, + int indent) { - return RepositoryPathPolicy.IsIgnoredPath(repositoryRoot, file); - } - - private static string ExtractBlock(string[] lines, int startIndex, int indent) - { - var endExclusive = startIndex + 1; + var endExclusive = headerEndIndex + 1; for (; endExclusive < lines.Length; endExclusive++) { var line = lines[endExclusive]; @@ -225,6 +455,126 @@ private static int LeadingSpaces(string line) return count; } + private static Match MatchDefinition( + string[] lines, + int startIndex, + out int headerEndIndex) + { + headerEndIndex = startIndex; + var first = lines[startIndex]; + if (!DefinitionStartRegex().IsMatch(first)) + { + return DefinitionRegex().Match(string.Empty); + } + + var header = first.TrimEnd(); + var balance = DelimiterBalance(first); + while ((balance > 0 || !HasHeaderTerminator(header)) && + headerEndIndex + 1 < lines.Length) + { + headerEndIndex++; + var continuation = lines[headerEndIndex].Trim(); + header = $"{header} {continuation}"; + balance += DelimiterBalance(continuation); + if (headerEndIndex - startIndex >= 100) + { + break; + } + } + + return DefinitionRegex().Match(header); + } + + private static bool HasHeaderTerminator(string header) + { + var balance = 0; + var quote = '\0'; + var escaped = false; + foreach (var character in header) + { + if (escaped) + { + escaped = false; + continue; + } + if (character == '\\' && quote != '\0') + { + escaped = true; + continue; + } + if (character is '\'' or '"') + { + if (quote == character) + { + quote = '\0'; + } + else if (quote == '\0') + { + quote = character; + } + continue; + } + if (quote != '\0') + { + continue; + } + balance += character switch + { + '(' or '[' or '{' => 1, + ')' or ']' or '}' => -1, + _ => 0 + }; + if (character == ':' && balance == 0) + { + return true; + } + } + return false; + } + + private static int DelimiterBalance(string line) + { + var balance = 0; + var quote = '\0'; + var escaped = false; + foreach (var character in line) + { + if (escaped) + { + escaped = false; + continue; + } + if (character == '\\' && quote != '\0') + { + escaped = true; + continue; + } + if (character is '\'' or '"') + { + if (quote == character) + { + quote = '\0'; + } + else if (quote == '\0') + { + quote = character; + } + continue; + } + if (quote != '\0') + { + continue; + } + balance += character switch + { + '(' or '[' or '{' => 1, + ')' or ']' or '}' => -1, + _ => 0 + }; + } + return balance; + } + private static string ProjectName(string repositoryRoot) { return new DirectoryInfo(repositoryRoot).Name; @@ -242,12 +592,42 @@ private static string NormalizePath(string path) return path.Replace('\\', '/'); } - [GeneratedRegex(@"^(?\s*)(?:(?async)\s+)?(?class|def)\s+(?[A-Za-z_][A-Za-z0-9_]*)\b.*:\s*(?:#.*)?$")] + [GeneratedRegex(@"^(?\s*)(?:(?async)\s+)?(?class|def)\s+(?[A-Za-z_][A-Za-z0-9_]*)\b.*:.*$")] private static partial Regex DefinitionRegex(); + [GeneratedRegex(@"^\s*(?:(?:async)\s+)?(?:class|def)\s+[A-Za-z_][A-Za-z0-9_]*\b")] + private static partial Regex DefinitionStartRegex(); + + [GeneratedRegex(@"^\s*import\s+(?[^#]+?)(?:\s*#.*)?$")] + private static partial Regex ImportRegex(); + + [GeneratedRegex(@"^\s*from\s+(?[A-Za-z_][A-Za-z0-9_.]*)\s+import\s+(?[^#]+?)(?:\s*#.*)?$")] + private static partial Regex FromImportRegex(); + + [GeneratedRegex(@"^(?[A-Za-z_][A-Za-z0-9_.]*)(?:\s+as\s+(?[A-Za-z_][A-Za-z0-9_]*))?$")] + private static partial Regex AliasRegex(); + + [GeneratedRegex(@"(?[A-Za-z_][A-Za-z0-9_]*)\s*\.\s*)?(?[A-Za-z_][A-Za-z0-9_]*)\s*\(")] + private static partial Regex CallRegex(); + + private static readonly HashSet IgnoredCallNames = new(StringComparer.Ordinal) + { + "and", "assert", "async", "await", "class", "def", "del", "elif", "else", "except", "False", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "None", "nonlocal", "not", "or", "pass", "raise", "return", "True", "try", "while", "with", "yield" + }; + private sealed record PythonScope( string NodeId, string QualifiedName, int Indent, - CodeNodeKind Kind); + CodeNodeKind Kind, + int StartLine, + int HeaderEndLine, + int EndLine, + string Name); + + private sealed record PythonImport( + CodeNode Node, + string Module, + string? ImportedName, + string Alias); } diff --git a/src/CodeMesh.Parser.Rust/CodeMesh.Parser.Rust.csproj b/src/CodeMesh.Parser.Rust/CodeMesh.Parser.Rust.csproj new file mode 100644 index 0000000..d15db8a --- /dev/null +++ b/src/CodeMesh.Parser.Rust/CodeMesh.Parser.Rust.csproj @@ -0,0 +1,13 @@ + + + + + + + + net10.0 + enable + enable + + + diff --git a/src/CodeMesh.Parser.Rust/RustParseService.cs b/src/CodeMesh.Parser.Rust/RustParseService.cs new file mode 100644 index 0000000..dd73cb2 --- /dev/null +++ b/src/CodeMesh.Parser.Rust/RustParseService.cs @@ -0,0 +1,659 @@ +using System.Text.RegularExpressions; +using CodeMesh.Domain.Contracts; +using CodeMesh.Domain.Graph; +using CodeMesh.Domain.Utilities; + +namespace CodeMesh.Parser.Rust; + +public sealed partial class RustParseService +{ + public static ParserCapability Capability { get; } = new( + "codemesh-parser-rust", + "rust", + "0.1.1", + [".rs"], + ["repositoryRoot", "pathFilter"]); + + public async Task ParseAsync( + ParseRequest request, + CancellationToken cancellationToken = default) + { + var repositoryRoot = Path.GetFullPath(request.RepositoryRoot); + if (!Directory.Exists(repositoryRoot)) + { + return Failure($"Repository root not found: {repositoryRoot}"); + } + + var nodes = new List(); + var relationships = new List(); + var contents = new List(); + var diagnostics = new List(); + var parsedAt = DateTimeOffset.UtcNow; + + foreach (var file in EnumerateRustFiles(repositoryRoot, request.PathFilter)) + { + cancellationToken.ThrowIfCancellationRequested(); + var relativePath = NormalizePath(Path.GetRelativePath(repositoryRoot, file)); + try + { + var text = await File.ReadAllTextAsync(file, cancellationToken).ConfigureAwait(false); + ParseFile( + repositoryRoot, + relativePath, + text, + parsedAt, + nodes, + relationships, + contents, + diagnostics); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + diagnostics.Add(new ParserDiagnostic( + "CMRS001", + $"Could not read Rust file '{relativePath}': {exception.Message}", + "warning", + SourceSpan.Unknown(relativePath))); + } + } + + return new ParseResult( + Capability.WorkerName, + Capability.Language, + nodes, + relationships, + contents.GroupBy(content => content.Hash, StringComparer.Ordinal).Select(group => group.First()).ToArray(), + diagnostics, + parsedAt); + } + + private static void ParseFile( + string repositoryRoot, + string relativePath, + string text, + DateTimeOffset parsedAt, + List nodes, + List relationships, + List contents, + List diagnostics) + { + var project = ProjectName(repositoryRoot); + var module = ModuleName(relativePath); + var lines = NormalizeLines(text); + var fileHash = CodeMeshHash.Sha256Hex(text); + var fileNodeId = $"rust:file:{relativePath}"; + var fileKind = IsTestFile(relativePath) ? CodeNodeKind.TestFile : CodeNodeKind.File; + nodes.Add(new CodeNode( + fileNodeId, + fileNodeId, + fileKind, + Path.GetFileName(relativePath), + "rust", + project, + new SourceSpan(relativePath, 1, 1, Math.Max(1, lines.Length), 1), + fileHash, + new Dictionary + { + ["filePath"] = relativePath, + ["module"] = module, + ["rustKind"] = "file" + })); + contents.Add(new CodeContent(fileHash, text, "text/x-rust", parsedAt, parsedAt)); + + var declarations = CollectDeclarations(lines, relativePath, diagnostics); + var built = new List(); + foreach (var declaration in declarations) + { + var parent = built + .Where(candidate => + candidate.Declaration.StartLine < declaration.StartLine && + candidate.Declaration.EndLine >= declaration.EndLine && + candidate.Declaration.HasBody) + .OrderByDescending(candidate => candidate.Declaration.StartLine) + .FirstOrDefault(); + var qualifiedName = parent is null + ? declaration.Name + : $"{parent.QualifiedName}::{declaration.Name}"; + var nodeId = NodeId(module, declaration, qualifiedName); + var content = string.Join( + Environment.NewLine, + lines[(declaration.StartLine - 1)..declaration.EndLine]); + var contentHash = CodeMeshHash.Sha256Hex(content); + var metadata = DeclarationMetadata(relativePath, module, declaration, qualifiedName, parent?.Node.Id ?? fileNodeId); + var node = new CodeNode( + nodeId, + nodeId, + declaration.NodeKind, + declaration.Name, + "rust", + project, + new SourceSpan( + relativePath, + declaration.StartLine, + declaration.StartColumn, + declaration.EndLine, + 1), + contentHash, + metadata); + nodes.Add(node); + contents.Add(new CodeContent( + contentHash, + content, + "text/x-rust-fragment", + parsedAt, + parsedAt)); + relationships.Add(Relationship( + parent?.Node.Id ?? fileNodeId, + nodeId, + CodeRelationshipKind.Contains, + null, + relativePath)); + built.Add(new BuiltDeclaration(declaration, node, qualifiedName, content)); + } + + AddSemanticRelationships(built, relationships, relativePath); + } + + private static IReadOnlyList CollectDeclarations( + string[] lines, + string relativePath, + List diagnostics) + { + var declarations = new List(); + var attributes = new List(); + for (var index = 0; index < lines.Length; index++) + { + var trimmed = StripLineComment(lines[index]).Trim(); + if (trimmed.StartsWith("#[", StringComparison.Ordinal)) + { + attributes.Add(trimmed); + continue; + } + + if (string.IsNullOrWhiteSpace(trimmed)) + { + continue; + } + + var statement = DeclarationStatement(lines, index); + var useMatch = UseRegex().Match(statement); + var implMatch = ImplRegex().Match(statement); + var declarationMatch = DeclarationRegex().Match(statement); + RustDeclaration? declaration = null; + if (useMatch.Success) + { + var path = NormalizeWhitespace(useMatch.Groups["path"].Value); + declaration = new RustDeclaration( + "use", + path, + CodeNodeKind.Declaration, + index + 1, + LeadingColumn(lines[index]), + FindDeclarationEnd(lines, index), + false, + statement, + attributes.ToArray(), + null, + null); + } + else if (implMatch.Success) + { + var traitName = NullIfWhiteSpace(implMatch.Groups["trait"].Value); + var typeName = implMatch.Groups["type"].Value; + declaration = new RustDeclaration( + "impl", + traitName is null ? $"impl {typeName}" : $"impl {traitName} for {typeName}", + CodeNodeKind.Declaration, + index + 1, + LeadingColumn(lines[index]), + FindDeclarationEnd(lines, index), + statement.Contains('{'), + statement, + attributes.ToArray(), + traitName, + typeName); + } + else if (declarationMatch.Success) + { + var rustKind = declarationMatch.Groups["kind"].Value; + var nodeKind = NodeKind(rustKind, attributes, declarationMatch.Groups["name"].Value); + declaration = new RustDeclaration( + rustKind, + declarationMatch.Groups["name"].Value, + nodeKind, + index + 1, + LeadingColumn(lines[index]), + FindDeclarationEnd(lines, index), + statement.Contains('{'), + statement, + attributes.ToArray(), + null, + null); + } + + if (declaration is not null) + { + declarations.Add(declaration); + attributes.Clear(); + continue; + } + + if (attributes.Count > 0 && !trimmed.StartsWith("//", StringComparison.Ordinal)) + { + attributes.Clear(); + } + } + + if (declarations.Count == 0 && lines.Any(line => !string.IsNullOrWhiteSpace(line))) + { + diagnostics.Add(new ParserDiagnostic( + "CMRS002", + $"Rust file '{relativePath}' contained no supported declarations.", + "info", + SourceSpan.Unknown(relativePath))); + } + + return declarations; + } + + private static void AddSemanticRelationships( + IReadOnlyList declarations, + List relationships, + string relativePath) + { + var byName = declarations + .GroupBy(item => item.Declaration.Name, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.ToArray(), StringComparer.Ordinal); + var callableByName = declarations + .Where(item => item.Declaration.RustKind == "fn") + .GroupBy(item => item.Declaration.Name, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.ToArray(), StringComparer.Ordinal); + var typeByName = declarations + .Where(item => item.Declaration.RustKind is "struct" or "enum" or "trait") + .GroupBy(item => item.Declaration.Name, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.ToArray(), StringComparer.Ordinal); + + foreach (var declaration in declarations) + { + if (declaration.Declaration.RustKind == "impl") + { + AddResolvedRelationship( + declaration.Node.Id, + declaration.Declaration.TraitName, + typeByName, + CodeRelationshipKind.Implements, + relationships, + relativePath); + AddResolvedRelationship( + declaration.Node.Id, + declaration.Declaration.TypeName, + typeByName, + CodeRelationshipKind.References, + relationships, + relativePath); + } + + if (declaration.Declaration.RustKind == "use") + { + var target = declaration.Declaration.Name + .TrimEnd(':', '*') + .Split("::", StringSplitOptions.RemoveEmptyEntries) + .LastOrDefault(); + AddResolvedRelationship( + declaration.Node.Id, + target, + byName, + CodeRelationshipKind.References, + relationships, + relativePath); + } + + if (declaration.Declaration.RustKind != "fn") + { + continue; + } + + var body = BodyOnly(declaration.Content); + foreach (Match match in CallRegex().Matches(body).Cast().Concat(MethodCallRegex().Matches(body).Cast())) + { + var name = match.Groups["name"].Value; + if (IgnoredCallNames.Contains(name) || + !callableByName.TryGetValue(name, out var targets) || + targets.Length != 1) + { + continue; + } + + relationships.Add(Relationship( + declaration.Node.Id, + targets[0].Node.Id, + CodeRelationshipKind.Invokes, + name, + relativePath)); + } + + foreach (var (name, targets) in typeByName) + { + if (targets.Length == 1 && SignatureContainsType(declaration.Declaration.Signature, name)) + { + relationships.Add(Relationship( + declaration.Node.Id, + targets[0].Node.Id, + CodeRelationshipKind.UsesType, + name, + relativePath)); + } + } + } + } + + private static IReadOnlyDictionary DeclarationMetadata( + string relativePath, + string module, + RustDeclaration declaration, + string qualifiedName, + string parentNodeId) + { + var metadata = new Dictionary + { + ["filePath"] = relativePath, + ["module"] = module, + ["qualifiedName"] = qualifiedName, + ["parentNodeId"] = parentNodeId, + ["rustKind"] = declaration.RustKind, + ["signature"] = declaration.Signature, + ["attributes"] = string.Join(' ', declaration.Attributes) + }; + if (declaration.TraitName is not null) + { + metadata["traitName"] = declaration.TraitName; + } + if (declaration.TypeName is not null) + { + metadata["typeName"] = declaration.TypeName; + } + var exportKind = PythonExportKind(declaration.Attributes); + if (exportKind is not null) + { + metadata["pythonExportKind"] = exportKind; + metadata["pythonExportName"] = PythonExportName(declaration.Attributes) ?? declaration.Name; + } + if (declaration.NodeKind == CodeNodeKind.TestCase) + { + metadata["testRole"] = "case"; + metadata["testFramework"] = "rust"; + } + return metadata; + } + + private static CodeNodeKind NodeKind( + string rustKind, + IReadOnlyList attributes, + string name) + { + return rustKind switch + { + "mod" => CodeNodeKind.Namespace, + "struct" => CodeNodeKind.Struct, + "enum" => CodeNodeKind.Enum, + "trait" => CodeNodeKind.Interface, + "fn" when IsTest(attributes, name) => CodeNodeKind.TestCase, + "fn" => CodeNodeKind.Method, + _ => CodeNodeKind.Declaration + }; + } + + private static string NodeId( + string module, + RustDeclaration declaration, + string qualifiedName) + { + var kind = declaration.RustKind; + return kind == "impl" + ? $"rust:impl:{module}:{CodeMeshHash.StableId(declaration.Signature)}" + : $"rust:{kind}:{module}:{qualifiedName}"; + } + + private static CodeRelationship Relationship( + string sourceNodeId, + string targetNodeId, + CodeRelationshipKind kind, + string? label, + string relativePath) + { + return new CodeRelationship( + CodeMeshHash.StableId("rust", kind.ToString(), sourceNodeId, targetNodeId, label ?? string.Empty), + sourceNodeId, + targetNodeId, + kind, + label, + new Dictionary { ["filePath"] = relativePath }); + } + + private static void AddResolvedRelationship( + string sourceNodeId, + string? name, + IReadOnlyDictionary targetsByName, + CodeRelationshipKind kind, + List relationships, + string relativePath) + { + if (name is not null && + targetsByName.TryGetValue(name.Split("::").Last(), out var targets) && + targets.Length == 1) + { + relationships.Add(Relationship( + sourceNodeId, + targets[0].Node.Id, + kind, + name, + relativePath)); + } + } + + private static int FindDeclarationEnd(string[] lines, int startIndex) + { + var depth = 0; + var opened = false; + for (var index = startIndex; index < lines.Length; index++) + { + var line = StripLineComment(lines[index]); + foreach (var character in line) + { + if (character == '{') + { + opened = true; + depth++; + } + else if (character == '}' && opened) + { + depth--; + } + } + if (opened && depth <= 0) + { + return index + 1; + } + if (!opened && line.Contains(';')) + { + return index + 1; + } + } + return Math.Max(1, lines.Length); + } + + private static string DeclarationStatement(string[] lines, int startIndex) + { + var parts = new List(); + for (var index = startIndex; index < Math.Min(lines.Length, startIndex + 12); index++) + { + var line = StripLineComment(lines[index]).Trim(); + if (!string.IsNullOrWhiteSpace(line)) + { + parts.Add(line); + } + if (line.Contains('{') || line.Contains(';')) + { + break; + } + } + return NormalizeWhitespace(string.Join(' ', parts)); + } + + private static string? PythonExportKind(IReadOnlyList attributes) + { + var joined = string.Join(' ', attributes); + if (joined.Contains("pymodule", StringComparison.Ordinal)) return "module"; + if (joined.Contains("pyclass", StringComparison.Ordinal)) return "class"; + if (joined.Contains("pyfunction", StringComparison.Ordinal)) return "function"; + if (joined.Contains("pymethods", StringComparison.Ordinal)) return "methods"; + return null; + } + + private static string? PythonExportName(IReadOnlyList attributes) + { + var match = PyO3NameRegex().Match(string.Join(' ', attributes)); + return match.Success ? match.Groups["name"].Value : null; + } + + private static bool IsTest(IReadOnlyList attributes, string name) + { + return name.StartsWith("test_", StringComparison.Ordinal) || + attributes.Any(attribute => + attribute.Contains("#[test]", StringComparison.Ordinal) || + attribute.Contains("::test", StringComparison.Ordinal)); + } + + private static bool SignatureContainsType(string signature, string name) + { + return Regex.IsMatch(signature, $@"\b{Regex.Escape(name)}\b", RegexOptions.CultureInvariant); + } + + private static string BodyOnly(string content) + { + var opening = content.IndexOf('{'); + return opening >= 0 ? content[(opening + 1)..] : string.Empty; + } + + private static string StripLineComment(string line) + { + var index = line.IndexOf("//", StringComparison.Ordinal); + return index >= 0 ? line[..index] : line; + } + + private static string NormalizeWhitespace(string value) + { + return Regex.Replace(value, @"\s+", " ").Trim(); + } + + private static string? NullIfWhiteSpace(string value) + { + return string.IsNullOrWhiteSpace(value) ? null : value; + } + + private static int LeadingColumn(string line) + { + var index = 0; + while (index < line.Length && char.IsWhiteSpace(line[index])) index++; + return index + 1; + } + + private static string[] NormalizeLines(string text) + { + return text.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n'); + } + + private static IEnumerable EnumerateRustFiles( + string repositoryRoot, + RepositoryPathFilter? pathFilter) + { + return RepositoryPathPolicy.EnumerateFiles(repositoryRoot, "*.rs", pathFilter) + .OrderBy(file => file, StringComparer.OrdinalIgnoreCase); + } + + private static bool IsTestFile(string relativePath) + { + var segments = relativePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + return segments.Any(segment => string.Equals(segment, "tests", StringComparison.OrdinalIgnoreCase)) || + Path.GetFileNameWithoutExtension(relativePath).EndsWith("_test", StringComparison.OrdinalIgnoreCase); + } + + private static string ProjectName(string repositoryRoot) + { + return new DirectoryInfo(repositoryRoot).Name; + } + + private static string ModuleName(string relativePath) + { + var path = Path.ChangeExtension(relativePath, null).Replace('\\', '/'); + var parts = path.Split('/', StringSplitOptions.RemoveEmptyEntries).ToList(); + if (parts.Count > 0 && string.Equals(parts[0], "src", StringComparison.OrdinalIgnoreCase)) + { + parts.RemoveAt(0); + } + if (parts.Count > 0 && parts[^1] is "lib" or "main" or "mod") + { + parts.RemoveAt(parts.Count - 1); + } + return parts.Count == 0 ? "crate" : $"crate::{string.Join("::", parts)}"; + } + + private static string NormalizePath(string path) + { + return path.Replace('\\', '/'); + } + + private static ParseResult Failure(string message) + { + return new ParseResult( + Capability.WorkerName, + Capability.Language, + Array.Empty(), + Array.Empty(), + Array.Empty(), + [new ParserDiagnostic("CMRS000", message, "error")], + DateTimeOffset.UtcNow); + } + + private static readonly HashSet IgnoredCallNames = new(StringComparer.Ordinal) + { + "as", "async", "await", "else", "for", "if", "loop", "match", "move", "return", "while" + }; + + [GeneratedRegex(@"^(?:(?:pub(?:\([^)]*\))?)\s+)?(?:(?:default|unsafe|const|async)\s+)*(?mod|struct|enum|trait|fn)\s+(?[A-Za-z_][A-Za-z0-9_]*)\b", RegexOptions.CultureInvariant)] + private static partial Regex DeclarationRegex(); + + [GeneratedRegex(@"^(?:(?:pub(?:\([^)]*\))?)\s+)?impl(?:\s*<[^>{}]*>)?\s+(?:(?[A-Za-z_][A-Za-z0-9_:]*)\s+for\s+)?(?[A-Za-z_][A-Za-z0-9_:]*)\b", RegexOptions.CultureInvariant)] + private static partial Regex ImplRegex(); + + [GeneratedRegex(@"^(?:(?:pub(?:\([^)]*\))?)\s+)?use\s+(?[^;]+);", RegexOptions.CultureInvariant)] + private static partial Regex UseRegex(); + + [GeneratedRegex(@"(?[A-Za-z_][A-Za-z0-9_]*)\s*!?\s*\(", RegexOptions.CultureInvariant)] + private static partial Regex CallRegex(); + + [GeneratedRegex(@"\.\s*(?[A-Za-z_][A-Za-z0-9_]*)\s*\(", RegexOptions.CultureInvariant)] + private static partial Regex MethodCallRegex(); + + [GeneratedRegex("\\bname\\s*=\\s*\"(?[A-Za-z_][A-Za-z0-9_]*)\"", RegexOptions.CultureInvariant)] + private static partial Regex PyO3NameRegex(); + + private sealed record RustDeclaration( + string RustKind, + string Name, + CodeNodeKind NodeKind, + int StartLine, + int StartColumn, + int EndLine, + bool HasBody, + string Signature, + IReadOnlyList Attributes, + string? TraitName, + string? TypeName); + + private sealed record BuiltDeclaration( + RustDeclaration Declaration, + CodeNode Node, + string QualifiedName, + string Content); +} diff --git a/src/CodeMesh.Parser.Rust/packages.lock.json b/src/CodeMesh.Parser.Rust/packages.lock.json new file mode 100644 index 0000000..f3eed40 --- /dev/null +++ b/src/CodeMesh.Parser.Rust/packages.lock.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "codemesh.domain": { + "type": "Project" + } + } + } +} diff --git a/src/CodeMesh.Storage/Neo4j/Neo4jGraphStore.cs b/src/CodeMesh.Storage/Neo4j/Neo4jGraphStore.cs index 0801a4b..e77d24f 100644 --- a/src/CodeMesh.Storage/Neo4j/Neo4jGraphStore.cs +++ b/src/CodeMesh.Storage/Neo4j/Neo4jGraphStore.cs @@ -19,6 +19,7 @@ public sealed class Neo4jGraphStore : IStoreHealthCheck, IAsyncDisposable { + private const int WriteBatchSize = 1000; private readonly IDriver _driver; private readonly string _uri; private readonly string? _database; @@ -51,30 +52,40 @@ public async Task UpsertGraphAsync(CodeGraphBatch batch, CancellationToken cance if (batch.Nodes.Count > 0) { - await RunAndConsumeAsync( - session, - """ - UNWIND $nodes AS node - MERGE (n:CodeMeshNode {storageKey: node.storageKey}) - SET n += node - """, - new { nodes = batch.Nodes.Select(node => ToNodeProperties(node, batch)).ToArray() }, - cancellationToken).ConfigureAwait(false); + foreach (var nodes in batch.Nodes + .Select(node => ToNodeProperties(node, batch)) + .Chunk(WriteBatchSize)) + { + await RunAndConsumeAsync( + session, + """ + UNWIND $nodes AS node + MERGE (n:CodeMeshNode {storageKey: node.storageKey}) + SET n += node + """, + new { nodes }, + cancellationToken).ConfigureAwait(false); + } } if (batch.Relationships.Count > 0) { - await RunAndConsumeAsync( - session, - """ - UNWIND $relationships AS relationship - MATCH (source:CodeMeshNode {storageKey: relationship.sourceStorageKey}) - MATCH (target:CodeMeshNode {storageKey: relationship.targetStorageKey}) - MERGE (source)-[r:CODEMESH_REL {storageKey: relationship.storageKey}]->(target) - SET r += relationship - """, - new { relationships = batch.Relationships.Select(relationship => ToRelationshipProperties(relationship, batch)).ToArray() }, - cancellationToken).ConfigureAwait(false); + foreach (var relationships in batch.Relationships + .Select(relationship => ToRelationshipProperties(relationship, batch)) + .Chunk(WriteBatchSize)) + { + await RunAndConsumeAsync( + session, + """ + UNWIND $relationships AS relationship + MATCH (source:CodeMeshNode {storageKey: relationship.sourceStorageKey}) + MATCH (target:CodeMeshNode {storageKey: relationship.targetStorageKey}) + MERGE (source)-[r:CODEMESH_REL {storageKey: relationship.storageKey}]->(target) + SET r += relationship + """, + new { relationships }, + cancellationToken).ConfigureAwait(false); + } } } diff --git a/tests/CodeMesh.Tests/CodeMesh.Tests.csproj b/tests/CodeMesh.Tests/CodeMesh.Tests.csproj index 6e84ddf..6795946 100644 --- a/tests/CodeMesh.Tests/CodeMesh.Tests.csproj +++ b/tests/CodeMesh.Tests/CodeMesh.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/tests/CodeMesh.Tests/Program.cs b/tests/CodeMesh.Tests/Program.cs index 2823b56..cf31882 100644 --- a/tests/CodeMesh.Tests/Program.cs +++ b/tests/CodeMesh.Tests/Program.cs @@ -13,10 +13,12 @@ using CodeMesh.Ingestion; using CodeMesh.Ingestion.Embedding; using CodeMesh.Ingestion.Summary; +using CodeMesh.Ingestion.Summary.Qualification; using CodeMesh.Parser.CSharp; using CodeMesh.Parser.Deployment; using CodeMesh.Parser.Markdown; using CodeMesh.Parser.Python; +using CodeMesh.Parser.Rust; using CodeMesh.Storage; using CodeMesh.Storage.Memory; using CodeMesh.Storage.Mongo; @@ -29,6 +31,9 @@ { ("sha256 helper is deterministic", TestHash), ("repository path policy ignores generated directories", TestRepositoryPathPolicyIgnoresGeneratedDirectories), + ("repository path policy rejects linked paths", TestRepositoryPathPolicyRejectsLinkedPaths), + ("repository path policy excludes known secrets and applies filters", TestRepositoryPathPolicyExcludesKnownSecretsAndAppliesFilters), + ("repository path policy rejects unsafe patterns", TestRepositoryPathPolicyRejectsUnsafePatterns), ("environment file loader fills missing variables", TestEnvironmentFileLoaderFillsMissingVariables), ("ingest path resolver accepts repository and caller relative files", TestIngestPathResolverAcceptsRelativeFiles), ("ingest path resolver rejects missing explicit files", TestIngestPathResolverRejectsMissingFiles), @@ -42,6 +47,7 @@ ("agent access contracts map REST JSON", TestAgentAccessContractsMapRestJson), ("agent access client calls REST endpoints", TestAgentAccessClientCallsRestEndpoints), ("ingestion writes parser output", TestIngestionWritesParserOutput), + ("multi-language ingestion publishes one deterministic snapshot", TestMultiLanguageIngestionPublishesOneDeterministicSnapshot), ("ingestion redacts sensitive content before stores and embeddings", TestIngestionRedactsSensitiveContentBeforeStoresAndEmbeddings), ("dry-run ingestion skips stores", TestDryRunSkipsStores), ("repository refresh removes stale records", TestRepositoryRefreshRemovesStaleRecords), @@ -50,6 +56,7 @@ ("summary budgets scale with node complexity", TestSummaryBudgetsScaleWithNodeComplexity), ("summary generator persists budget metadata", TestSummaryGeneratorPersistsBudgetMetadata), ("summary generator regenerates when configuration changes", TestSummaryGeneratorRegeneratesWhenConfigurationChanges), + ("summary qualification runner redacts inputs and enforces evidence gates", TestSummaryQualificationRunnerAndCompiler), ("forced embedding ingestion rebuilds unchanged embeddings", TestForcedEmbeddingIngestionRebuildsUnchangedEmbeddings), ("embedding verifier reports missing and stale vectors", TestEmbeddingVerifierReportsMissingAndStaleVectors), ("repository cleanup removes indexed data", TestRepositoryCleanupRemovesIndexedData), @@ -81,12 +88,17 @@ ("csharp parser emits cross-project invocation relationships", TestCSharpParserEmitsCrossProjectInvocationRelationships), ("csharp parser separates partial declarations from symbols", TestCSharpParserSeparatesPartialDeclarations), ("csharp parser honors project compile items", TestCSharpParserHonorsProjectCompileItems), + ("csharp parser preserves isolated design-time editor configuration", TestCSharpParserDesignTimeEditorConfiguration), + ("csharp parser applies repository path filters", TestCSharpParserAppliesRepositoryPathFilters), ("deployment parser emits Dockerfile and Compose nodes", TestDeploymentParserEmitsDockerfileAndComposeNodes), ("deployment parser parses repository runtime artifacts", TestDeploymentParserParsesRepositoryRuntimeArtifacts), ("markdown parser emits documents sections and containment", TestMarkdownParserEmitsDocumentsSectionsAndContainment), ("markdown parser parses repository docs", TestMarkdownParserParsesRepositoryDocs), ("python parser emits classes functions and containment", TestPythonParserEmitsClassesFunctionsAndContainment), + ("python parser emits imports calls and test cases", TestPythonParserEmitsImportsCallsAndTestCases), ("python parser parses Agent Access project", TestPythonParserParsesAgentAccessProject), + ("rust parser emits declarations relationships tests and PyO3 exports", TestRustParserEmitsDeclarationsRelationshipsTestsAndPyO3Exports), + ("composite parser links Python calls to Rust PyO3 exports", TestCompositeParserLinksPythonCallsToRustPyO3Exports), ("csharp parser parses YoutubeDownloader sample", TestYoutubeDownloaderSampleParser) }; @@ -142,6 +154,113 @@ static Task TestRepositoryPathPolicyIgnoresGeneratedDirectories() return Task.CompletedTask; } +static Task TestRepositoryPathPolicyRejectsLinkedPaths() +{ + var root = Path.Combine(Path.GetTempPath(), $"codemesh-path-link-root-{Guid.NewGuid():N}"); + var outside = Path.Combine(Path.GetTempPath(), $"codemesh-path-link-outside-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + Directory.CreateDirectory(outside); + + try + { + var outsideFile = Path.Combine(outside, "External.cs"); + var normalFile = Path.Combine(root, "Normal.cs"); + File.WriteAllText(outsideFile, "public sealed class External;"); + File.WriteAllText(normalFile, "public sealed class Normal;"); + var linkedDirectory = Path.Combine(root, "linked-directory"); + var linkedFile = Path.Combine(root, "Linked.cs"); + try + { + Directory.CreateSymbolicLink(linkedDirectory, outside); + File.CreateSymbolicLink(linkedFile, outsideFile); + } + catch (Exception exception) when ( + exception is PlatformNotSupportedException or + UnauthorizedAccessException or + IOException) + { + return Task.CompletedTask; + } + + Equal(true, RepositoryPathPolicy.IsIgnoredPath(root, linkedDirectory), "Repository path policy did not reject a linked directory."); + Equal(true, RepositoryPathPolicy.IsIgnoredPath(root, Path.Combine(linkedDirectory, "External.cs")), "Repository path policy did not reject a file below a linked directory."); + Equal(true, RepositoryPathPolicy.IsIgnoredPath(root, linkedFile), "Repository path policy did not reject a linked file."); + Equal(false, RepositoryPathPolicy.IsIgnoredPath(root, normalFile), "Repository path policy rejected a normal repository file."); + var enumerated = RepositoryPathPolicy.EnumerateFiles(root, "*.cs") + .Select(Path.GetFileName) + .Order(StringComparer.Ordinal) + .ToArray(); + Equal(1, enumerated.Length, "Repository traversal included a linked file or traversed a linked directory."); + Equal("Normal.cs", enumerated[0], "Repository traversal did not retain the normal repository file."); + } + finally + { + Directory.Delete(root, recursive: true); + Directory.Delete(outside, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestRepositoryPathPolicyExcludesKnownSecretsAndAppliesFilters() +{ + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath(".env"), "Repository path policy did not exclude .env."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("config/.env.local"), "Repository path policy did not exclude environment-specific secrets."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("infra/secrets.yaml"), "Repository path policy did not exclude a known secret manifest."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("cloud/service-account-prod.json"), "Repository path policy did not exclude service-account material."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("keys/deploy.pem"), "Repository path policy did not exclude a private-key extension."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("state/app.tfstate.backup"), "Repository path policy did not exclude Terraform state backup."); + Equal(false, RepositoryPathPolicy.IsIgnoredRelativePath(".env.example"), "Repository path policy excluded a safe environment template."); + Equal(false, RepositoryPathPolicy.IsIgnoredRelativePath("config/.env.production.template"), "Repository path policy excluded an environment-specific safe template."); + Equal(false, RepositoryPathPolicy.IsIgnoredRelativePath("src/Secrets.cs"), "Repository path policy excluded source code based only on its name."); + + var filter = new RepositoryPathFilter( + AllowPatterns: ["src/**"], + DenyPatterns: ["src/generated/**"]); + Equal(false, RepositoryPathPolicy.IsIgnoredRelativePath("src/App.cs", filter), "Repository path allow pattern did not include matching source."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("tests/AppTests.cs", filter), "Repository path allow pattern did not exclude a non-matching path."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("src/generated/App.g.cs", filter), "Repository path deny pattern did not override the allow pattern."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("src/.env", filter), "Allow pattern bypassed default known-secret exclusion."); + + var rootOnly = new RepositoryPathFilter(AllowPatterns: ["README.md"]); + Equal(false, RepositoryPathPolicy.IsIgnoredRelativePath("README.md", rootOnly), "Root-relative allow pattern did not match the root file."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("docs/README.md", rootOnly), "Root-relative allow pattern matched a nested basename."); + var recursiveBasename = new RepositoryPathFilter(AllowPatterns: ["**/README.md"]); + Equal(false, RepositoryPathPolicy.IsIgnoredRelativePath("docs/README.md", recursiveBasename), "Recursive basename allow pattern did not match a nested file."); + + var explicitSecretOverride = filter with { ExcludeKnownSecretFiles = false }; + Equal(false, RepositoryPathPolicy.IsIgnoredRelativePath("src/.env", explicitSecretOverride), "Explicit known-secret override did not change the default exclusion."); + Equal(true, RepositoryPathPolicy.IsIgnoredRelativePath("src/bin/Generated.cs", explicitSecretOverride), "Explicit secret override bypassed immutable generated-directory exclusions."); + + Equal( + RepositoryPathPolicy.StableFingerprint(new RepositoryPathFilter(["SRC/**", "docs/**"])), + RepositoryPathPolicy.StableFingerprint(new RepositoryPathFilter(["docs/**", "src/**"])), + "Equivalent path filters produced different stable fingerprints."); + + return Task.CompletedTask; +} + +static Task TestRepositoryPathPolicyRejectsUnsafePatterns() +{ + Throws( + () => RepositoryPathPolicy.StableFingerprint(new RepositoryPathFilter(["../outside/**"])), + "Repository path policy accepted parent traversal in an allow pattern."); + Throws( + () => RepositoryPathPolicy.StableFingerprint(new RepositoryPathFilter(DenyPatterns: [Path.GetFullPath("outside/**")])), + "Repository path policy accepted an absolute deny pattern."); + Throws( + () => RepositoryPathPolicy.StableFingerprint(new RepositoryPathFilter(DenyPatterns: ["C:\\outside\\**"])), + "Repository path policy accepted a Windows absolute deny pattern."); + Throws( + () => RepositoryPathPolicy.StableFingerprint(new RepositoryPathFilter([" "])), + "Repository path policy accepted an empty allow pattern."); + Throws( + () => RepositoryPathPolicy.StableFingerprint(new RepositoryPathFilter(["src//**"])), + "Repository path policy accepted an ambiguous repeated separator."); + + return Task.CompletedTask; +} + static Task TestEnvironmentFileLoaderFillsMissingVariables() { var root = Path.Combine(Path.GetTempPath(), $"codemesh-env-loader-test-{Guid.NewGuid():N}"); @@ -386,6 +505,8 @@ static Task TestWatchWorkflowDetectsFileChanges() var ignoredDirectory = Path.Combine(root, "bin"); var ignoredTemporaryDirectory = Path.Combine(root, ".tmp"); var ignoredSmokeDirectory = Path.Combine(root, ".codemesh-smoke"); + var filteredDirectory = Path.Combine(root, "tests"); + var secretFile = Path.Combine(root, ".env"); try { @@ -393,6 +514,7 @@ static Task TestWatchWorkflowDetectsFileChanges() Directory.CreateDirectory(ignoredDirectory); Directory.CreateDirectory(ignoredTemporaryDirectory); Directory.CreateDirectory(ignoredSmokeDirectory); + Directory.CreateDirectory(filteredDirectory); File.WriteAllText(file, "one"); var first = CodeMeshWatchWorkflow.CreateSnapshot(root); @@ -409,6 +531,19 @@ static Task TestWatchWorkflowDetectsFileChanges() Equal(false, CodeMeshWatchWorkflow.HasChanges(first, unchanged), "Watch workflow reported changes for identical snapshots."); Equal(true, CodeMeshWatchWorkflow.HasChanges(first, changed), "Watch workflow did not detect a changed file."); Equal(false, CodeMeshWatchWorkflow.HasChanges(changed, ignoredChange), "Watch workflow did not ignore generated directories."); + + var filter = new RepositoryPathFilter(AllowPatterns: ["src/**"]); + File.WriteAllText(secretFile, "TOKEN=not-a-real-secret"); + File.WriteAllText(Path.Combine(filteredDirectory, "SampleTests.cs"), "one"); + var filtered = CodeMeshWatchWorkflow.CreateSnapshot(root, filter); + File.WriteAllText(secretFile, "TOKEN=still-not-a-real-secret"); + File.WriteAllText(Path.Combine(filteredDirectory, "SampleTests.cs"), "one two"); + var filteredIgnoredChange = CodeMeshWatchWorkflow.CreateSnapshot(root, filter); + Equal(false, CodeMeshWatchWorkflow.HasChanges(filtered, filteredIgnoredChange), "Watch workflow reacted to secret or allow-filtered files."); + + File.WriteAllText(file, "one two three"); + var filteredSourceChange = CodeMeshWatchWorkflow.CreateSnapshot(root, filter); + Equal(true, CodeMeshWatchWorkflow.HasChanges(filteredIgnoredChange, filteredSourceChange), "Watch workflow missed an allowed source change."); } finally { @@ -865,6 +1000,47 @@ static async Task TestIngestionWritesParserOutput() NotNull(content, "Content was not written to content store."); } +static async Task TestMultiLanguageIngestionPublishesOneDeterministicSnapshot() +{ + var store = new InMemoryCodeMeshStore(); + var parser = new CompositeParserClient(new Dictionary + { + ["rust"] = new LanguageFixtureParserClient("rust", ".rs"), + ["python"] = new LanguageFixtureParserClient("python", ".py") + }); + var orchestrator = new IngestionOrchestrator( + parser, + store, + store, + store, + new NoEmbeddingProvider()); + + var first = await orchestrator.IngestAsync(new IngestionRequest( + Directory.GetCurrentDirectory(), + Language: "rust,python", + DryRun: true)); + var second = await orchestrator.IngestAsync(new IngestionRequest( + Directory.GetCurrentDirectory(), + Language: "python,rust", + DryRun: true)); + var filtered = await orchestrator.IngestAsync(new IngestionRequest( + Directory.GetCurrentDirectory(), + Language: "python,rust", + DryRun: true, + PathFilter: new RepositoryPathFilter(DenyPatterns: ["not-present/**"]))); + + Equal(2, first.NodeCount, "Composite ingestion did not retain both parser outputs."); + Equal(2, first.ContentCount, "Composite ingestion did not retain both source contents."); + Equal( + first.Repository.SnapshotId ?? string.Empty, + second.Repository.SnapshotId ?? string.Empty, + "Language order changed the atomic snapshot identity."); + Equal( + false, + string.Equals(first.Repository.SnapshotId, filtered.Repository.SnapshotId, StringComparison.Ordinal), + "Repository path filter did not change snapshot scope identity."); +} + static async Task TestIngestionRedactsSensitiveContentBeforeStoresAndEmbeddings() { var store = new InMemoryCodeMeshStore(); @@ -1338,6 +1514,210 @@ static async Task TestSummaryGeneratorRegeneratesWhenConfigurationChanges() Equal(1, fourth.SkippedCount, "Unchanged generation configuration did not skip the summary."); } +static async Task TestSummaryQualificationRunnerAndCompiler() +{ + const string canary = "summary-qualification-secret-canary"; + const string source = "private const string ApiKey = \"summary-qualification-secret-canary\";"; + var qualificationItem = new SummaryQualificationCorpusItem( + "secret-field", + "repo:fixture", + "0123456789abcdef", + "node:secret-field", + CodeMeshHash.Sha256Hex(source), + "csharp", + CodeNodeKind.Field, + "ApiKey", + "Fixture", + "SecretFixture.cs", + 1, + 1, + source, + "small", + "high", + ["security-sensitive", "small"], + ["The field stores an API key value."], + [], + ["The value is safe to publish."], + ["field"], + ["secret"], + "qualification", + "Stores an API key value.", + [], + [], + [], + [], + "Stores an API key value.", + [canary]); + var calibrationItem = qualificationItem with + { + Id = "calibration-secret-field", + NodeId = "node:calibration-secret-field", + Partition = "calibration" + }; + var suite = new SummaryQualificationSuite( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.SuiteKind, + "synthetic-summary-suite", + "1.0.0", + [calibrationItem, qualificationItem], + new SummaryQualificationGates( + MaximumP95LatencyMs: 1000, + MinimumNodesPerMinute: 1, + MaximumTotalTokens: 1000, + MaximumTokensPerSuccessfulNode: 200, + MaximumTotalDurationMs: 5000, + MaximumRetryTokenOverheadRatio: 0, + MaximumPeakHostMemoryMb: 512)); + var profile = new SummaryQualificationProfile( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.ProfileKind, + "deterministic-local", + "local-offline", + "local", + "test", + "summary-counting", + "fixture-v1", + "none", + "float32", + CodeMeshHash.Sha256Hex("fixture-template"), + 8192, + 4000, + 768, + 1536, + 3, + new Dictionary + { + ["temperature"] = "0", + ["topP"] = "1", + ["topK"] = "not-applicable", + ["seed"] = "1", + ["stopSequences"] = "none", + ["requestedReasoningEffort"] = "none", + ["effectiveReasoningEffort"] = "none", + ["providerGenerationConfigurationFingerprint"] = "fixture-configuration-v1", + ["retryPolicy"] = "none", + ["timeout"] = "10s", + ["concurrency"] = "1", + ["batching"] = "1" + }, + new Dictionary + { + ["operatingSystem"] = "test-os", + ["cpu"] = "test-cpu", + ["memory"] = "1 GiB", + ["accelerator"] = "none", + ["acceleratorMemory"] = "none", + ["servingRuntime"] = "test" + }); + var sourceIdentity = new SummaryQualificationSourceIdentity( + "fedcba9876543210", + false, + "main", + "https://example.invalid/codemesh", + "test-os", + "test-runtime"); + + var result = await new SummaryQualificationRunner(new CountingSummaryProvider("fixture-configuration-v1")) + .RunAsync(suite, profile, sourceIdentity); + + Equal(4, result.PrivateArchive.Samples.Count, "Qualification warm-up and repetitions were not executed."); + Equal(4, result.PrivateArchive.Samples.Count(sample => sample.Completed), "Deterministic qualification samples did not complete."); + Equal(false, result.ReviewPacket.Items.Any(review => review.Source.Contains(canary, StringComparison.Ordinal)), "The blinded review packet retained a secret canary."); + + SummaryQualificationReview Review(string reviewerId) + { + return new SummaryQualificationReview( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.ReviewKind, + result.PrivateArchive.RunId, + reviewerId, + true, + CodeMeshHash.Sha256Hex("resolved-review-calibration-v1"), + result.PrivateArchive.Samples + .Where(sample => string.Equals(sample.Phase, "measured", StringComparison.Ordinal)) + .Select(sample => new SummaryQualificationSampleReview( + sample.SampleId, + sample.ItemId, + 1, + 1, + 1, + 1, + false, + false, + true, + 0, + 0, + 1, + true, + true, + 1, + 0, + 0)).ToArray()); + } + + var baselineReportHash = CodeMeshHash.Sha256Hex("baseline-report"); + var candidateReportHash = CodeMeshHash.Sha256Hex("candidate-report"); + var retrievalAssessment = new SummaryQualificationRetrievalAssessment( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.RetrievalAssessmentKind, + result.PrivateArchive.RunId, + baselineReportHash, + candidateReportHash, + 1, + 0, + 1, + 0, + 0, + 0.9, + 0.9, + 1, + 1, + 0, + true); + var retrieval = new SummaryQualificationRetrievalEvidence( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.RetrievalEvidenceKind, + result.PrivateArchive.RunId, + result.PrivateArchive.SuiteHash, + result.PrivateArchive.PromptHash, + result.PrivateArchive.CorpusRepositoryIdentityHash, + CodeMeshHash.Sha256Hex("ranking-v1"), + "synthetic-live", + "repo:fixture", + "0123456789abcdef", + baselineReportHash, + candidateReportHash, + new SummaryRetrievalMetrics(0.9, 0.9, 0.9, 0.9, 1, 0, true, true), + new SummaryRetrievalMetrics(0.9, 0.9, 0.9, 0.9, 1, 0, true, true), + retrievalAssessment, + []); + var deployment = new SummaryQualificationDeploymentEvidence( + SummaryQualificationConstants.SchemaVersion, + SummaryQualificationConstants.DeploymentEvidenceKind, + result.PrivateArchive.RunId, + result.PrivateArchive.ProfileHash, + "deterministic fixture measurement", + 256, + null, + null, + null, + true); + var reviews = new[] { Review("reviewer-a"), Review("reviewer-b") }; + var report = SummaryQualificationCompiler.Compile(suite, result.PrivateArchive, reviews, retrieval, deployment); + Equal("qualified", report.Outcome, "A complete passing qualification run was not qualified."); + Equal(1d, report.Metrics.RequiredFactRecallByStratum["security-sensitive"], "Stratum recall was not calculated from item identities."); + Equal(560L, report.Metrics.TotalTokens, "Provider usage was not aggregated across warm-up and measured attempts."); + + Equal(1d, report.Metrics.IdenticalInputStability, "Repeated identical inputs were not compared for stability."); + Equal(256d, report.Metrics.PeakHostMemoryMb, "Deployment evidence was not included in the report."); + + var missingRetrieval = SummaryQualificationCompiler.Compile(suite, result.PrivateArchive, reviews, null, deployment); + Equal("invalid-run", missingRetrieval.Outcome, "Missing retrieval evidence did not invalidate qualification."); + Throws( + () => SummaryQualificationCompiler.Compare(report, report with { SuiteHash = "different-suite" }), + "Incompatible qualification reports were compared."); +} + static async Task TestForcedEmbeddingIngestionRebuildsUnchangedEmbeddings() { var root = Path.Combine(Path.GetTempPath(), $"codemesh-force-embeddings-test-{Guid.NewGuid():N}"); @@ -2174,6 +2554,9 @@ static async Task TestLmStudioSummaryProviderAcceptsNumericConfidence() "LM Studio summary text did not deserialize."); Equal("high", summary.Confidence, "LM Studio numeric confidence was not normalized."); Equal("normalization", summary.Tags[0], "LM Studio summary tags did not deserialize."); + Equal(96, summary.Usage?.InputTokens ?? -1, "LM Studio summary input-token usage was not captured."); + Equal(24, summary.Usage?.OutputTokens ?? -1, "LM Studio summary output-token usage was not captured."); + Equal(120, summary.Usage?.TotalTokens ?? -1, "LM Studio summary total-token usage was not captured."); if (!handler.LastRequestBody.Contains("\"response_format\":{\"type\":\"text\"}", StringComparison.Ordinal)) { throw new InvalidOperationException("LM Studio summary request response format changed."); @@ -2213,6 +2596,9 @@ static async Task TestOllamaSummaryProviderUsesRequestCompletionBudget() Equal(1, handler.RequestCount, "Expected one Ollama summary request."); Equal("Adds two integers and returns their sum.", summary.Summary, "Ollama summary text did not deserialize."); + Equal(72, summary.Usage?.InputTokens ?? -1, "Ollama summary input-token usage was not captured."); + Equal(18, summary.Usage?.OutputTokens ?? -1, "Ollama summary output-token usage was not captured."); + Equal(90, summary.Usage?.TotalTokens ?? -1, "Ollama summary total-token usage was not captured."); if (!handler.LastRequestBody.Contains("\"num_predict\":1024", StringComparison.Ordinal)) { throw new InvalidOperationException("Ollama summary request ignored the selected completion ceiling."); @@ -2667,8 +3053,14 @@ static async Task TestNeo4jGraphStoreRoundTripsGraph() await using var store = new Neo4jGraphStore(new Neo4jGraphStoreOptions(uri, user, password, database)); - using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15)); - var health = await store.CheckAsync(timeout.Token); + using var timeout = new CancellationTokenSource(); + CancellationToken OperationToken() + { + timeout.CancelAfter(TimeSpan.FromSeconds(15)); + return timeout.Token; + } + + var health = await store.CheckAsync(OperationToken()); if (health.State != CodeMesh.Domain.Health.ComponentState.Healthy) { var detail = health.Metadata.TryGetValue("detail", out var value) ? value : health.Message; @@ -2712,15 +3104,15 @@ await store.UpsertGraphAsync(new CodeGraphBatch( [source, target], [relationship], Array.Empty(), - now), timeout.Token); + now), OperationToken()); - var loaded = await store.GetNodeAsync(source.Id, timeout.Token); + var loaded = await store.GetNodeAsync(source.Id, OperationToken()); NotNull(loaded, "Neo4j graph store did not return the inserted node."); Equal(source.Id, loaded!.Id, "Neo4j node id changed."); Equal(source.Kind, loaded.Kind, "Neo4j node kind changed."); Equal(source.Metadata["test"], loaded.Metadata["test"], "Neo4j node metadata changed."); - var relationships = await store.GetRelationshipsAsync(source.Id, cancellationToken: timeout.Token); + var relationships = await store.GetRelationshipsAsync(source.Id, cancellationToken: OperationToken()); var loadedRelationship = relationships.SingleOrDefault(item => item.Id == relationship.Id); NotNull(loadedRelationship, "Neo4j graph store did not return the inserted relationship."); Equal(relationship.SourceNodeId, loadedRelationship!.SourceNodeId, "Neo4j relationship source changed."); @@ -2781,17 +3173,17 @@ await store.UpsertGraphAsync(new CodeGraphBatch( [collisionSourceA, collisionTargetA], [collisionRelationshipA], Array.Empty(), - now), timeout.Token); + now), OperationToken()); await store.UpsertGraphAsync(new CodeGraphBatch( "run:neo4j-collision-b", collisionRepositoryB, [collisionSourceB, collisionTargetB], [collisionRelationshipB], Array.Empty(), - now), timeout.Token); + now), OperationToken()); - var loadedCollisionA = await store.GetNodeAsync(sharedSourceId, collisionRepositoryA.Id, timeout.Token); - var loadedCollisionB = await store.GetNodeAsync(sharedSourceId, collisionRepositoryB.Id, timeout.Token); + var loadedCollisionA = await store.GetNodeAsync(sharedSourceId, collisionRepositoryA.Id, OperationToken()); + var loadedCollisionB = await store.GetNodeAsync(sharedSourceId, collisionRepositoryB.Id, OperationToken()); NotNull(loadedCollisionA, "Neo4j did not return repository A's colliding node."); NotNull(loadedCollisionB, "Neo4j did not return repository B's colliding node."); Equal("CollisionProjectA", loadedCollisionA!.Project, "Neo4j repository A node was overwritten by repository B."); @@ -2799,8 +3191,8 @@ await store.UpsertGraphAsync(new CodeGraphBatch( Equal("a", loadedCollisionA.Metadata["repoMarker"], "Neo4j repository A node metadata changed."); Equal("b", loadedCollisionB.Metadata["repoMarker"], "Neo4j repository B node metadata changed."); - var collisionRelationshipsA = await store.GetRelationshipsAsync(sharedSourceId, collisionRepositoryA.Id, cancellationToken: timeout.Token); - var collisionRelationshipsB = await store.GetRelationshipsAsync(sharedSourceId, collisionRepositoryB.Id, cancellationToken: timeout.Token); + var collisionRelationshipsA = await store.GetRelationshipsAsync(sharedSourceId, collisionRepositoryA.Id, cancellationToken: OperationToken()); + var collisionRelationshipsB = await store.GetRelationshipsAsync(sharedSourceId, collisionRepositoryB.Id, cancellationToken: OperationToken()); var loadedCollisionRelationshipA = collisionRelationshipsA.SingleOrDefault(item => item.Id == sharedRelationshipId); var loadedCollisionRelationshipB = collisionRelationshipsB.SingleOrDefault(item => item.Id == sharedRelationshipId); NotNull(loadedCollisionRelationshipA, "Neo4j did not return repository A's colliding relationship."); @@ -2827,18 +3219,18 @@ await store.UpsertGraphAsync(new CodeGraphBatch( [stale], Array.Empty(), Array.Empty(), - now), timeout.Token); + now), OperationToken()); await store.UpsertGraphAsync(new CodeGraphBatch( "run:neo4j-current", repository, [current], Array.Empty(), Array.Empty(), - now), timeout.Token); - await store.RemoveStaleGraphAsync(repository.Id, "run:neo4j-current", timeout.Token); + now), OperationToken()); + await store.RemoveStaleGraphAsync(repository.Id, "run:neo4j-current", OperationToken()); - IsNull(await store.GetNodeAsync(stale.Id, timeout.Token), "Neo4j graph refresh did not remove stale node."); - NotNull(await store.GetNodeAsync(current.Id, timeout.Token), "Neo4j graph refresh removed current node."); + IsNull(await store.GetNodeAsync(stale.Id, OperationToken()), "Neo4j graph refresh did not remove stale node."); + NotNull(await store.GetNodeAsync(current.Id, OperationToken()), "Neo4j graph refresh removed current node."); var incrementalRepository = new RepositoryRef($"repo:neo4j-incremental:{unique}", Directory.GetCurrentDirectory()); var incrementalStale = source with @@ -2867,10 +3259,10 @@ await store.UpsertGraphAsync(new CodeGraphBatch( [incrementalStale, incrementalCurrent], [incrementalRelationship], Array.Empty(), - now), timeout.Token); + now), OperationToken()); - var nodeStates = await store.GetNodeStatesAsync(incrementalRepository.Id, timeout.Token); - var relationshipStates = await store.GetRelationshipStatesAsync(incrementalRepository.Id, timeout.Token); + var nodeStates = await store.GetNodeStatesAsync(incrementalRepository.Id, OperationToken()); + var relationshipStates = await store.GetRelationshipStatesAsync(incrementalRepository.Id, OperationToken()); Equal(2, nodeStates.Count, "Neo4j incremental node inventory count changed."); Equal(1, relationshipStates.Count, "Neo4j incremental relationship inventory count changed."); @@ -2878,10 +3270,26 @@ await store.RemoveGraphExceptAsync( incrementalRepository.Id, new HashSet(StringComparer.Ordinal) { incrementalCurrent.Id }, new HashSet(StringComparer.Ordinal), - timeout.Token); - - IsNull(await store.GetNodeAsync(incrementalStale.Id, timeout.Token), "Neo4j incremental cleanup did not remove stale node."); - NotNull(await store.GetNodeAsync(incrementalCurrent.Id, timeout.Token), "Neo4j incremental cleanup removed current node."); + OperationToken()); + + IsNull(await store.GetNodeAsync(incrementalStale.Id, OperationToken()), "Neo4j incremental cleanup did not remove stale node."); + NotNull(await store.GetNodeAsync(incrementalCurrent.Id, OperationToken()), "Neo4j incremental cleanup removed current node."); + + var emptyIds = new HashSet(StringComparer.Ordinal); + foreach (var repositoryId in new[] + { + repository.Id, + collisionRepositoryA.Id, + collisionRepositoryB.Id, + incrementalRepository.Id + }) + { + await store.RemoveGraphExceptAsync( + repositoryId, + emptyIds, + emptyIds, + OperationToken()); + } } static async Task TestQdrantVectorStoreRoundTripsEmbeddings() @@ -3416,6 +3824,14 @@ public void Run(Downloader downloader) } """); + var hook = System.Security.SecurityElement.Escape(Path.Combine(AppContext.BaseDirectory, "CodeMesh.DesignTime.targets")); + foreach (var projectFile in Directory.EnumerateFiles(root, "*.csproj", SearchOption.AllDirectories)) + { + var projectText = await File.ReadAllTextAsync(projectFile); + await File.WriteAllTextAsync(projectFile, + projectText.Replace("", $"", StringComparison.Ordinal)); + } + var parser = new CSharpParseService(); var result = await parser.ParseAsync(new ParseRequest( root, @@ -3457,24 +3873,24 @@ static async Task TestCSharpParserSeparatesPartialDeclarations() try { await File.WriteAllTextAsync( - Path.Combine(root, "Customer.cs"), + Path.Combine(root, "Customer.Validation.cs"), """ namespace Sample; public sealed partial class Customer { - public string Name { get; set; } = ""; + public bool IsValid() => !string.IsNullOrWhiteSpace(Name); } """); await File.WriteAllTextAsync( - Path.Combine(root, "Customer.Validation.cs"), + Path.Combine(root, "Customer.cs"), """ namespace Sample; public sealed partial class Customer { - public bool IsValid() => !string.IsNullOrWhiteSpace(Name); + public string Name { get; set; } = ""; } """); @@ -3487,6 +3903,10 @@ public sealed partial class Customer new Dictionary())); Equal(1, result.Nodes.Count(node => string.Equals(node.Id, "csharp:type:Sample.Customer", StringComparison.Ordinal)), "Expected one logical Customer type node."); + Equal( + "Customer.cs", + result.Nodes.Single(node => string.Equals(node.Id, "csharp:type:Sample.Customer", StringComparison.Ordinal)).Span.FilePath, + "The logical Customer type should use the canonical main declaration path."); Equal(2, result.Nodes.Count(node => node.Kind == CodeNodeKind.Declaration && node.Metadata.TryGetValue("symbolId", out var symbolId) && @@ -3509,6 +3929,101 @@ public sealed partial class Customer } } +static async Task TestCSharpParserDesignTimeEditorConfiguration() +{ + var root = Directory.CreateTempSubdirectory("codemesh-editor-config-test-").FullName; + var hook = Path.Combine(AppContext.BaseDirectory, "CodeMesh.DesignTime.targets"); + if (!File.Exists(hook)) + { + throw new InvalidOperationException("The design-time hook must be included in parser output."); + } + + try + { + async Task ParseFixture(string label, bool fail) + { + var projectRoot = Path.Combine(root, label); + Directory.CreateDirectory(projectRoot); + var audit = Path.Combine(root, $"{label}.audit"); + var escapedHook = System.Security.SecurityElement.Escape(hook); + var escapedAudit = System.Security.SecurityElement.Escape(audit); + await File.WriteAllTextAsync(Path.Combine(projectRoot, "Same.csproj"), $$""" + + + net10.0 + {{label}} + {{label}} + + + + + + + + $([System.IO.File]::ReadAllText('$(GeneratedMSBuildEditorConfigFile)')) + + + + + + """); + await File.WriteAllTextAsync(Path.Combine(projectRoot, "Wanted.cs"), + $"namespace {label}; public sealed class Wanted {{ public int Value => 42; }}"); + return await new CSharpParseService().ParseAsync(new ParseRequest( + projectRoot, null, null, "csharp", new Dictionary())); + } + + var results = await Task.WhenAll(ParseFixture("First", false), ParseFixture("Second", false)); + foreach (var result in results) + { + HasNode(result, CodeNodeKind.Class, "Wanted"); + if (result.Diagnostics.Any(item => item.Code is "CMSHARP010" or "CMSHARP012")) + { + throw new InvalidOperationException("Editor-config fixtures must load with MSBuild."); + } + } + + var failed = await ParseFixture("Failed", true); + if (!failed.Diagnostics.Any(item => item.Message.Contains("Intentional design-time fixture failure", StringComparison.Ordinal))) + { + throw new InvalidOperationException("The deliberate project-load failure must remain visible."); + } + + var generatedPaths = new List(); + foreach (var label in new[] { "First", "Second", "Failed" }) + { + var lines = await File.ReadAllLinesAsync(Path.Combine(root, $"{label}.audit")); + var generated = lines[0]; + generatedPaths.Add(generated); + if (!lines.Any(line => line.Trim() == $"build_property.CodeMeshFixtureProperty = {label}")) + { + throw new InvalidOperationException($"Compiler-visible metadata was not preserved for {label}."); + } + + if (Path.GetRelativePath(root, generated).Split(Path.DirectorySeparatorChar)[0] != "..") + { + throw new InvalidOperationException("Generated configuration must be outside the repository."); + } + + if (File.Exists(generated) || Directory.Exists(Path.GetDirectoryName(generated))) + { + throw new InvalidOperationException("Request-owned configuration must be removed after workspace disposal."); + } + + if (Directory.EnumerateFiles(Path.Combine(root, label), "*.editorconfig", SearchOption.AllDirectories).Any()) + { + throw new InvalidOperationException("The hook must not write editor configuration inside the repository."); + } + } + + Equal(3, generatedPaths.Distinct(StringComparer.Ordinal).Count(), "Concurrent project output paths collided."); + } + finally + { + Directory.Delete(root, recursive: true); + } +} + static async Task TestCSharpParserHonorsProjectCompileItems() { var root = Path.Combine(Path.GetTempPath(), $"codemesh-parser-project-test-{Guid.NewGuid():N}"); @@ -3582,6 +4097,67 @@ public sealed class Excluded } } +static async Task TestCSharpParserAppliesRepositoryPathFilters() +{ + var root = Path.Combine(Path.GetTempPath(), $"codemesh-parser-path-filter-test-{Guid.NewGuid():N}"); + var externalRoot = Path.Combine(Path.GetTempPath(), $"codemesh-parser-path-filter-external-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(Path.Combine(root, "src", "private")); + Directory.CreateDirectory(Path.Combine(root, "tests")); + Directory.CreateDirectory(externalRoot); + + try + { + var projectPath = Path.Combine(root, "Filtered.csproj"); + await File.WriteAllTextAsync( + projectPath, + "net10.0"); + await File.WriteAllTextAsync( + Path.Combine(root, "src", "Visible.cs"), + "namespace Sample; public sealed class Visible { }"); + await File.WriteAllTextAsync( + Path.Combine(root, "src", "private", "Hidden.cs"), + "namespace Sample; public sealed class Hidden { }"); + await File.WriteAllTextAsync( + Path.Combine(root, "tests", "VisibleTests.cs"), + "namespace Sample.Tests; public sealed class VisibleTests { }"); + var externalProjectPath = Path.Combine(externalRoot, "External.csproj"); + await File.WriteAllTextAsync( + externalProjectPath, + "net10.0"); + + var result = await new CSharpParseService().ParseAsync(new ParseRequest( + root, + null, + projectPath, + "csharp", + new Dictionary(), + new RepositoryPathFilter( + AllowPatterns: ["src/**"], + DenyPatterns: ["src/private/**"]))); + + Equal(0, result.Diagnostics.Count(diagnostic => diagnostic.Severity == "error"), "C# parser returned errors for a filtered repository."); + HasNodeId(result, "csharp:file:src/Visible.cs"); + HasNodeId(result, "csharp:type:Sample.Visible"); + HasNoNodeId(result, "csharp:file:src/private/Hidden.cs", result); + HasNoNodeId(result, "csharp:type:Sample.Hidden", result); + HasNoNodeId(result, "csharp:file:tests/VisibleTests.cs", result); + + await ThrowsAsync( + () => new CSharpParseService().ParseAsync(new ParseRequest( + root, + null, + externalProjectPath, + "csharp", + new Dictionary())), + "C# parser opened an explicit project manifest outside the repository path policy."); + } + finally + { + Directory.Delete(root, recursive: true); + Directory.Delete(externalRoot, recursive: true); + } +} + static async Task TestDeploymentParserEmitsDockerfileAndComposeNodes() { var root = Path.Combine(Path.GetTempPath(), $"codemesh-deployment-parser-test-{Guid.NewGuid():N}"); @@ -3589,6 +4165,7 @@ static async Task TestDeploymentParserEmitsDockerfileAndComposeNodes() try { Directory.CreateDirectory(root); + Directory.CreateDirectory(Path.Combine(root, "private")); await File.WriteAllTextAsync( Path.Combine(root, "Dockerfile"), """ @@ -3624,6 +4201,9 @@ await File.WriteAllTextAsync( ports: - "5432:5432" """); + await File.WriteAllTextAsync( + Path.Combine(root, "private", "docker-compose.yml"), + "services: { hidden: { image: hidden:latest } }"); var parser = new DeploymentParseService(); var result = await parser.ParseAsync(new ParseRequest( @@ -3631,7 +4211,8 @@ await File.WriteAllTextAsync( null, null, "deployment", - new Dictionary())); + new Dictionary(), + new RepositoryPathFilter(DenyPatterns: ["private/**"]))); Equal("codemesh-parser-deployment", result.ParserName, "Deployment parser name changed."); Equal("deployment", result.Language, "Deployment parser language changed."); @@ -3644,6 +4225,7 @@ await File.WriteAllTextAsync( Equal(true, nodeIds.Contains("deployment:compose:docker-compose.yml"), "Deployment parser did not emit Compose file node."); Equal(true, nodeIds.Contains("deployment:compose-service:docker-compose.yml#api"), "Deployment parser did not emit api service node."); Equal(true, nodeIds.Contains("deployment:compose-service:docker-compose.yml#db"), "Deployment parser did not emit db service node."); + Equal(false, nodeIds.Any(id => id.Contains("private/", StringComparison.Ordinal)), "Deployment parser ignored the configured deny pattern."); var runtimeStage = result.Nodes.Single(node => node.Id == "deployment:dockerfile-stage:Dockerfile#stage-2"); Equal(CodeNodeKind.DockerfileStage, runtimeStage.Kind, "Deployment parser did not use DockerfileStage node kind."); @@ -3696,6 +4278,7 @@ static async Task TestMarkdownParserEmitsDocumentsSectionsAndContainment() try { Directory.CreateDirectory(root); + Directory.CreateDirectory(Path.Combine(root, "private")); var ignoredDirectories = new[] { ".venv", ".tmp", ".codemesh-smoke" }; foreach (var ignoredDirectory in ignoredDirectories) { @@ -3719,6 +4302,9 @@ Run the thing. Use local services. """); + await File.WriteAllTextAsync( + Path.Combine(root, "private", "Hidden.md"), + "# Hidden"); var parser = new MarkdownParseService(); var result = await parser.ParseAsync(new ParseRequest( @@ -3726,7 +4312,8 @@ Use local services. null, null, "markdown", - new Dictionary())); + new Dictionary(), + new RepositoryPathFilter(DenyPatterns: ["private/**"]))); Equal("codemesh-parser-markdown", result.ParserName, "Markdown parser name changed."); Equal(0, result.Diagnostics.Count(diagnostic => diagnostic.Severity == "error"), "Markdown parser emitted errors."); @@ -3734,6 +4321,7 @@ Use local services. HasNode(result, CodeNodeKind.Section, "Product"); HasNode(result, CodeNodeKind.Section, "Setup"); HasNode(result, CodeNodeKind.Section, "Local"); + Equal(false, result.Nodes.Any(node => node.Span.FilePath.StartsWith("private/", StringComparison.Ordinal)), "Markdown parser ignored the configured deny pattern."); Equal( false, result.Nodes.Any(node => ignoredDirectories.Any(directory => @@ -3795,8 +4383,20 @@ class Greeter: def greet(self, name: str) -> str: return normalize(name) + class GreeterContract: + def configure(self, *, enabled: bool) -> None: ... + + class GreeterAdapter: + def configure(self, *, enabled: bool) -> None: + return None + async def normalize(value: str) -> str: return value.strip().lower() + + def format_name( + value: str, + ) -> str: + return normalize(value) """); var parser = new PythonParseService(); @@ -3813,6 +4413,11 @@ async def normalize(value: str) -> str: HasNode(result, CodeNodeKind.Class, "Greeter"); HasNode(result, CodeNodeKind.Method, "greet"); HasNode(result, CodeNodeKind.Method, "normalize"); + HasNode(result, CodeNodeKind.Method, "format_name"); + Equal( + 2, + result.Nodes.Count(node => node.Kind == CodeNodeKind.Method && node.Name == "configure"), + "Python parser did not keep same-named methods in distinct inline-body classes."); HasRelationshipBetween( result, "python:file:service.py", @@ -3826,6 +4431,11 @@ async def normalize(value: str) -> str: var normalize = result.Nodes.Single(node => node.Id == "python:method:service.normalize"); Equal("True", normalize.Metadata["async"], "Python parser did not mark async functions."); + HasRelationshipBetween( + result, + "python:method:service.format_name", + normalize.Id, + CodeRelationshipKind.Invokes); } finally { @@ -3863,6 +4473,224 @@ static async Task TestPythonParserParsesAgentAccessProject() HasNode(result, CodeNodeKind.Method, "tool_manifest"); } +static async Task TestPythonParserEmitsImportsCallsAndTestCases() +{ + var root = Path.Combine(Path.GetTempPath(), $"codemesh-python-semantic-test-{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(root); + await File.WriteAllTextAsync( + Path.Combine(root, "native.py"), + """ + from _onenine_feed_hub_native import replay as native_replay + from _onenine_feed_hub_native import replay as native_replay + + def run_replay(): + return native_replay() + + def test_run_replay(): + return run_replay() + """); + + var result = await new PythonParseService().ParseAsync(new ParseRequest( + root, + null, + null, + "python", + new Dictionary())); + + var import = result.Nodes.Single(node => + node.Metadata.TryGetValue("pythonKind", out var kind) && kind == "import"); + var run = result.Nodes.Single(node => node.Name == "run_replay"); + var test = result.Nodes.Single(node => node.Name == "test_run_replay"); + Equal("_onenine_feed_hub_native", import.Metadata["importModule"], "Python import module metadata changed."); + Equal("replay", import.Metadata["importedName"], "Python imported-name metadata changed."); + Equal(CodeNodeKind.TestCase, test.Kind, "Python test function was not emitted as a test case."); + HasRelationshipBetween(result, run.Id, import.Id, CodeRelationshipKind.Invokes); + HasRelationshipBetween(result, test.Id, run.Id, CodeRelationshipKind.Invokes); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } +} + +static async Task TestRustParserEmitsDeclarationsRelationshipsTestsAndPyO3Exports() +{ + var root = Path.Combine(Path.GetTempPath(), $"codemesh-rust-parser-test-{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(Path.Combine(root, "src")); + Directory.CreateDirectory(Path.Combine(root, "target", "debug")); + await File.WriteAllTextAsync( + Path.Combine(root, "src", "lib.rs"), + """ + use pyo3::prelude::*; + + trait Runner { + fn run(&self); + } + + #[pyclass] + struct Replay; + + impl Runner for Replay { + fn run(&self) { + helper(); + } + } + + fn helper() {} + + #[pyfunction] + #[pyo3(name = "replay")] + fn replay_impl(value: Replay) -> Replay { + helper(); + value + } + + #[pymodule] + fn _onenine_feed_hub_native() {} + + #[test] + fn test_replay() { + replay_impl(Replay); + } + """); + await File.WriteAllTextAsync( + Path.Combine(root, "target", "debug", "generated.rs"), + "fn must_not_be_indexed() {}\n"); + + var result = await new RustParseService().ParseAsync(new ParseRequest( + root, + null, + null, + "rust", + new Dictionary())); + var repeatedResult = await new RustParseService().ParseAsync(new ParseRequest( + root, + null, + null, + "rust", + new Dictionary())); + + Equal("codemesh-parser-rust", result.ParserName, "Rust parser name changed."); + Equal(0, result.Diagnostics.Count(item => item.Severity == "error"), "Rust parser emitted errors."); + Equal( + string.Join('\n', result.Nodes.Select(node => node.Id).Order(StringComparer.Ordinal)), + string.Join('\n', repeatedResult.Nodes.Select(node => node.Id).Order(StringComparer.Ordinal)), + "Rust node identities were not deterministic."); + Equal( + string.Join('\n', result.Relationships.Select(relationship => relationship.Id).Order(StringComparer.Ordinal)), + string.Join('\n', repeatedResult.Relationships.Select(relationship => relationship.Id).Order(StringComparer.Ordinal)), + "Rust relationship identities were not deterministic."); + HasNode(result, CodeNodeKind.Interface, "Runner"); + HasNode(result, CodeNodeKind.Struct, "Replay"); + HasNode(result, CodeNodeKind.Method, "helper"); + HasNode(result, CodeNodeKind.TestCase, "test_replay"); + Equal(false, result.Nodes.Any(node => node.Name == "must_not_be_indexed"), "Rust parser indexed Cargo target output."); + + var implementation = result.Nodes.Single(node => + node.Metadata.TryGetValue("rustKind", out var kind) && kind == "impl"); + var traitNode = result.Nodes.Single(node => node.Name == "Runner" && node.Kind == CodeNodeKind.Interface); + HasRelationshipBetween(result, implementation.Id, traitNode.Id, CodeRelationshipKind.Implements); + + var replay = result.Nodes.Single(node => + node.Metadata.TryGetValue("pythonExportName", out var name) && name == "replay"); + Equal("function", replay.Metadata["pythonExportKind"], "Rust PyO3 function export metadata changed."); + var module = result.Nodes.Single(node => + node.Metadata.TryGetValue("pythonExportKind", out var kind) && kind == "module"); + Equal("_onenine_feed_hub_native", module.Metadata["pythonExportName"], "Rust PyO3 module name changed."); + HasRelationship(result, CodeRelationshipKind.Invokes); + HasRelationship(result, CodeRelationshipKind.UsesType); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } +} + +static async Task TestCompositeParserLinksPythonCallsToRustPyO3Exports() +{ + var root = Path.Combine(Path.GetTempPath(), $"codemesh-pyo3-boundary-test-{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(Path.Combine(root, "src")); + Directory.CreateDirectory(Path.Combine(root, "ignored")); + await File.WriteAllTextAsync( + Path.Combine(root, "bridge.py"), + """ + from _onenine_feed_hub_native import replay + + def run(): + replay() + return replay() + """); + await File.WriteAllTextAsync( + Path.Combine(root, "src", "lib.rs"), + """ + #[pyfunction] + fn replay() {} + + #[pymodule] + fn _onenine_feed_hub_native() {} + """); + await File.WriteAllTextAsync( + Path.Combine(root, "ignored", "skip.py"), + "def must_not_be_indexed(): pass\n"); + await File.WriteAllTextAsync( + Path.Combine(root, "ignored", "skip.rs"), + "fn must_not_be_indexed() {}\n"); + + var parser = new CompositeParserClient(new Dictionary + { + ["python"] = new LocalPythonFixtureClient(), + ["rust"] = new LocalRustFixtureClient() + }); + var result = await parser.ParseAsync(new ParseRequest( + root, + null, + null, + "python,rust", + new Dictionary(), + new RepositoryPathFilter(DenyPatterns: ["ignored/**"]))); + + var pythonRun = result.Nodes.Single(node => node.Language == "python" && node.Name == "run"); + var rustReplay = result.Nodes.Single(node => + node.Language == "rust" && + node.Metadata.TryGetValue("pythonExportName", out var name) && + name == "replay"); + var boundary = result.Relationships.SingleOrDefault(relationship => + relationship.SourceNodeId == pythonRun.Id && + relationship.TargetNodeId == rustReplay.Id && + relationship.Kind == CodeRelationshipKind.Invokes && + relationship.Metadata.TryGetValue("boundary", out var value) && + value == "pyo3"); + NotNull(boundary, "Composite parser did not link Python invocation to Rust PyO3 export."); + Equal(false, result.Nodes.Any(node => node.Name == "must_not_be_indexed"), "Composite parser did not apply the shared deny pattern."); + Equal( + 1, + result.Relationships.Count(relationship => + relationship.SourceNodeId == pythonRun.Id && + relationship.TargetNodeId == rustReplay.Id && + relationship.Kind == CodeRelationshipKind.Invokes), + "Composite parser did not coalesce repeated calls across the PyO3 boundary."); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } +} + static async Task TestYoutubeDownloaderSampleParser() { var sampleRoot = Environment.GetEnvironmentVariable("CODEMESH_SAMPLE_CSHARP_ROOT"); @@ -4289,6 +5117,92 @@ public Task ParseAsync(ParseRequest request, CancellationToken canc } } +sealed class LanguageFixtureParserClient(string language, string extension) : IParserClient +{ + public Task GetCapabilityAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(new ParserCapability( + $"fixture-{language}", + language, + "test", + [extension], + ["repositoryRoot"])); + } + + public Task ParseAsync( + ParseRequest request, + CancellationToken cancellationToken = default) + { + if (!string.Equals(language, request.Language, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "Composite parser did not isolate the language request."); + } + var now = DateTimeOffset.UtcNow; + var filePath = $"sample{extension}"; + var text = $"{language} fixture"; + var contentHash = CodeMeshHash.Sha256Hex(text); + var node = new CodeNode( + $"{language}:file:{filePath}", + $"{language}:file:{filePath}", + CodeNodeKind.File, + filePath, + language, + null, + new SourceSpan(filePath, 1, 1, 1, text.Length), + contentHash, + new Dictionary()); + var content = new CodeContent( + contentHash, + text, + "text/plain", + now, + now); + return Task.FromResult(new ParseResult( + $"fixture-{language}", + language, + [node], + Array.Empty(), + [content], + Array.Empty(), + now)); + } +} + +sealed class LocalPythonFixtureClient : IParserClient +{ + private readonly PythonParseService _parser = new(); + + public Task GetCapabilityAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(PythonParseService.Capability); + } + + public Task ParseAsync( + ParseRequest request, + CancellationToken cancellationToken = default) + { + return _parser.ParseAsync(request, cancellationToken); + } +} + +sealed class LocalRustFixtureClient : IParserClient +{ + private readonly RustParseService _parser = new(); + + public Task GetCapabilityAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(RustParseService.Capability); + } + + public Task ParseAsync( + ParseRequest request, + CancellationToken cancellationToken = default) + { + return _parser.ParseAsync(request, cancellationToken); + } +} + sealed class SecretParserClient : IParserClient { public const string SecretValue = "super-secret-value"; @@ -4561,7 +5475,9 @@ protected override async Task SendAsync( "message": { "role": "assistant", "content": "{\"summary\":\"Adds two integers and returns their sum.\",\"tags\":[\"addition\"],\"confidence\":\"high\"}" - } + }, + "prompt_eval_count": 72, + "eval_count": 18 } """, System.Text.Encoding.UTF8, @@ -4685,7 +5601,12 @@ protected override async Task SendAsync( }, "finish_reason": "stop" } - ] + ], + "usage": { + "prompt_tokens": 96, + "completion_tokens": 24, + "total_tokens": 120 + } } """, System.Text.Encoding.UTF8, diff --git a/tests/CodeMesh.Tests/packages.lock.json b/tests/CodeMesh.Tests/packages.lock.json index cca91eb..c35a6ae 100644 --- a/tests/CodeMesh.Tests/packages.lock.json +++ b/tests/CodeMesh.Tests/packages.lock.json @@ -261,6 +261,12 @@ "CodeMesh.Domain": "[1.0.0, )" } }, + "codemesh.parser.rust": { + "type": "Project", + "dependencies": { + "CodeMesh.Domain": "[1.0.0, )" + } + }, "codemesh.storage": { "type": "Project", "dependencies": { @@ -272,4 +278,4 @@ } } } -} \ No newline at end of file +} diff --git a/tools/check_github_workflows.py b/tools/check_github_workflows.py new file mode 100644 index 0000000..36359aa --- /dev/null +++ b/tools/check_github_workflows.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Validate read-only verification workflow security invariants.""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +IMMUTABLE_ACTION_RE = re.compile(r"^[^\s@]+@[0-9a-f]{40}\s+#\s+v\S+$") +USES_RE = re.compile(r"^(?P\s*)-?\s*uses:\s*(?P.+?)\s*$") + + +class WorkflowError(RuntimeError): + """Raised when a workflow violates the bounded verification contract.""" + + +def _permissions_are_read_only(lines: list[str]) -> bool: + found_top_level = False + for index, line in enumerate(lines): + if line.strip() != "permissions:": + continue + block_indent = len(line) - len(line.lstrip()) + found_top_level = found_top_level or block_indent == 0 + values: list[str] = [] + for following in lines[index + 1 :]: + if not following.strip(): + continue + indent = len(following) - len(following.lstrip()) + if indent <= block_indent: + break + if indent == block_indent + 2: + values.append(following.strip()) + if values != ["contents: read"]: + return False + return found_top_level + + +def _checkout_has_no_persisted_credentials(lines: list[str], index: int) -> bool: + action_line = lines[index] + action_indent = len(action_line) - len(action_line.lstrip()) + step_indent = ( + action_indent + if action_line.lstrip().startswith("- uses:") + else action_indent - 2 + ) + for following in lines[index + 1 :]: + if not following.strip(): + continue + indent = len(following) - len(following.lstrip()) + stripped = following.lstrip() + if indent <= step_indent and stripped.startswith(("- name:", "- uses:")): + break + if stripped == "persist-credentials: false": + return True + return False + + +def validate(path: Path) -> None: + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + if not _permissions_are_read_only(lines): + raise WorkflowError(f"{path}: top-level permissions must be contents: read") + if "${{ secrets." in text: + raise WorkflowError(f"{path}: verification workflow must not reference secrets") + + action_count = 0 + for index, line in enumerate(lines): + match = USES_RE.match(line) + if match is None: + continue + value = match.group("value") + if value.startswith("./"): + continue + action_count += 1 + if IMMUTABLE_ACTION_RE.fullmatch(value) is None: + raise WorkflowError( + f"{path}: mutable or uncommented action reference: {value}" + ) + if value.startswith( + "actions/checkout@" + ) and not _checkout_has_no_persisted_credentials(lines, index): + raise WorkflowError( + f"{path}: actions/checkout must set persist-credentials: false" + ) + if action_count == 0: + raise WorkflowError(f"{path}: no external action references found") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="+", type=Path) + args = parser.parse_args() + try: + for path in args.paths: + validate(path) + except (OSError, WorkflowError) as exc: + print(exc, file=sys.stderr) + return 1 + print(f"GitHub workflow checks passed: {len(args.paths)} file(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_version.py b/tools/check_version.py new file mode 100644 index 0000000..d022954 --- /dev/null +++ b/tools/check_version.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify synchronized source version declarations without importing the app.""" + +from __future__ import annotations + +import ast +import re +import tomllib +from pathlib import Path + + +def validate(root: Path) -> str: + version = (root / "VERSION").read_text(encoding="utf-8").strip() + if ( + re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", version) + is None + ): + raise ValueError("VERSION must contain a three-part numeric version.") + package = root / "agent-access" + project = tomllib.loads((package / "pyproject.toml").read_text(encoding="utf-8")) + module = ast.parse( + (package / "codemesh_agent_access/__init__.py").read_text(encoding="utf-8") + ) + exported = next( + ast.literal_eval(node.value) + for node in module.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "__version__" + for target in node.targets + ) + ) + lock = tomllib.loads((package / "uv.lock").read_text(encoding="utf-8")) + locked = next( + item["version"] + for item in lock["package"] + if item["name"] == project["project"]["name"] + ) + for name, actual in { + "pyproject.toml": project["project"]["version"], + "__version__": exported, + "uv.lock": locked, + }.items(): + if actual != version: + raise ValueError( + f"{name} version {actual!r} differs from VERSION {version!r}." + ) + return version + + +if __name__ == "__main__": + print( + f"Version declarations agree: {validate(Path(__file__).resolve().parents[1])}" + )