From 10c7fc3206724de49845eb822e3ac99d8540b86a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 16 Oct 2026 18:09:06 +0200 Subject: [PATCH] fix(pkg): remove the archive size caps from the npm, PyPI and Go wrappers v0.12.0 could not be installed through npm, pnpm, pip or the Go wrapper on any platform (#2657): each wrapper refused an archive member above 256 MiB, and every v0.12.0 binary is 331-335 MB. The v0.11.0 binaries were already about 300 MB, so the Go wrapper, which has had the cap since v0.10.0, was broken before; npm and PyPI gained it in v0.12.0. The archive is verified against the release's checksums.txt before it is unpacked, so a byte cap only ever refused what we ship. - npm, PyPI, Go: delete every byte-size cap (compressed archive, per member, aggregate, checksum manifest, download size). The checksum, the exact member set, path and link refusal, the member count, HTTPS and the --version check stay. - All three honour CBM_DOWNLOAD_URL like install.sh, for testing: https, or plain http to an exact loopback authority with no proxy and no redirect. - Smoke: scripts/smoke-wrappers.sh installs the three wrappers from the checkout against the release fixture and runs the binary each one installed. smoke-test.sh Phase 13w runs it; SMOKE_REQUIRE_WRAPPERS=1 makes it required on the PR smoke lanes and in the release smoke, which gates publishing. Other install paths found broken while checking every channel: - Homebrew: the formula was still at 0.10.3 and installed the dynamic Linux build, which needs glibc 2.38 (Ubuntu 22.04, Debian 12 and RHEL 9 lack it). It now installs 0.12.0 with the static -portable Linux assets and tracks the newest release in the version contract like the other surfaces. - Go: `go install .../pkg/go/cmd/codebase-memory-mcp@vX.Y.Z` never worked (no pkg/go/v* tags, so Go fetched the whole repository, over its 500 MB module limit), and @latest built main's wrapper, whose version constant lags each release. The release now also tags pkg/go/vX.Y.Z, and the wrapper installs its own module version (the constant only for local and untagged builds). - npm: a stable release also moves the `next` dist-tag to itself; `next` still pointed to 0.9.1-rc.1. - Docs: winget and Chocolatey were listed as install channels but are not published; they are removed until they are. Windows arm64 ships and is now listed. Fixes #2657. Signed-off-by: Martin Vogel --- .github/workflows/_smoke.yml | 3 + .github/workflows/pr.yml | 3 + .github/workflows/release.yml | 12 + README.md | 4 +- docs/index.html | 4 +- docs/llms.txt | 2 +- pkg/go/cmd/codebase-memory-mcp/main.go | 351 +++++++++--------- pkg/go/cmd/codebase-memory-mcp/main_test.go | 380 ++++++++++---------- pkg/homebrew/Formula/codebase-memory-mcp.rb | 16 +- pkg/npm/README.md | 2 + pkg/npm/install.js | 188 ++++------ pkg/npm/test/archive-limits.test.js | 251 +++++++------ pkg/npm/test/install-failure.test.js | 56 +++ pkg/pypi/README.md | 2 + pkg/pypi/src/codebase_memory_mcp/_cli.py | 244 +++++-------- pkg/pypi/tests/test_cli.py | 365 +++++++++++-------- scripts/ci/select-lanes.sh | 2 +- scripts/smoke-test.sh | 12 + scripts/smoke-wrappers.sh | 150 ++++++++ tests/test_version_metadata_contract.sh | 3 +- 20 files changed, 1132 insertions(+), 918 deletions(-) create mode 100644 scripts/smoke-wrappers.sh diff --git a/.github/workflows/_smoke.yml b/.github/workflows/_smoke.yml index 1dd85e1be5..5a20026da5 100644 --- a/.github/workflows/_smoke.yml +++ b/.github/workflows/_smoke.yml @@ -118,6 +118,7 @@ jobs: env: CBM_SMOKE_ARTIFACT_DIR: ${{ runner.temp }}/cbm-artifact SMOKE_REQUIRE_UI: "1" + SMOKE_REQUIRE_WRAPPERS: "1" - name: Security audits run: | @@ -224,6 +225,7 @@ jobs: env: SMOKE_ARCH: ${{ matrix.arch }} SMOKE_REQUIRE_UI: "1" + SMOKE_REQUIRE_WRAPPERS: "1" run: | export CBM_SMOKE_ARTIFACT_DIR="$(cygpath -u "$RUNNER_TEMP")/cbm-artifact" bash test-infrastructure/vm/vm-smoke.sh @@ -301,6 +303,7 @@ jobs: env: CBM_SMOKE_ARTIFACT_DIR: ${{ runner.temp }}/cbm-artifact SMOKE_REQUIRE_UI: "1" + SMOKE_REQUIRE_WRAPPERS: "1" # The -portable binary is what all linux install/update paths now deliver; # it MUST start on old glibc (Debian 11 / RHEL 8 / Ubuntu 20.04). Runs it diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 33cd4a5af8..0aa631e8ea 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -232,6 +232,7 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1000M + SMOKE_REQUIRE_WRAPPERS: "1" - name: Build prod + smoke (macOS) if: matrix.os == 'macos-14' @@ -242,6 +243,7 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1000M + SMOKE_REQUIRE_WRAPPERS: "1" - name: Build prod + smoke (Windows) if: matrix.os == 'windows-latest' @@ -252,6 +254,7 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1000M + SMOKE_REQUIRE_WRAPPERS: "1" # ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and # the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84e55a97cc..134b425e24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -283,9 +283,15 @@ jobs: # move the release tag. (A floating HEAD also broke the 0.8.0 release: # HEAD had become a commit touching .github/workflows/, and the App # token may not create refs pointing at workflow-modifying commits.) + # The Go wrapper is a nested module: `go install + # .../pkg/go/cmd/codebase-memory-mcp@vX.Y.Z` and @latest resolve its + # versions from pkg/go/v* tags only (without one, Go fetches the whole + # repository, which is over its 500 MB module limit). run: | git tag -f "$VERSION" "$GITHUB_SHA" git push origin "$VERSION" --force + git tag -f "pkg/go/$VERSION" "$GITHUB_SHA" + git push origin "pkg/go/$VERSION" --force - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: @@ -510,6 +516,12 @@ jobs: else npm publish --access public --provenance fi + # `next` is "the newest you can opt into": once a stable release + # exists, an older RC must not stay behind it (0.9.1-rc.1 did until + # v0.12.1). Re-runnable: dist-tag add is idempotent. + if [[ "$RELEASE_VERSION" != *-* ]]; then + npm dist-tag add "codebase-memory-mcp@$V" next + fi # ── PyPI ───────────────────────────────────────────────── - name: Setup Python diff --git a/README.md b/README.md index 27262a0790..8e5fd698fc 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ These captures show CBM Lens Beta on 12–13 October 2026, using public source f ## Why codebase-memory-mcp - **Indexing performance**: Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline: LZ4 compression, in-memory SQLite, fused Aho-Corasick pattern matching. Memory released after indexing. -- **Plug and play**: native executable plus authenticated release-owned assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64). The native install needs no Docker, language runtime, or API keys. Download → `install` → restart agent → done. +- **Plug and play**: native executable plus authenticated release-owned assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64). The native install needs no Docker, language runtime, or API keys. Download → `install` → restart agent → done. - **162 languages**: vendored tree-sitter grammars compiled into the binary. No separate parser installation. - **About 120× fewer tokens in the reported five-query comparison**: ~3,400 tokens vs ~412,000 via file-by-file search. See [Performance](#performance) for the scope and sources. - **45 supported automatic/conditional client surfaces**: `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries. @@ -360,7 +360,7 @@ The install script placed beside the binary is **reported, not deleted**: uninst - **Auto-sync**: Background watcher detects file changes in the project an MCP session is working in and re-indexes it automatically (other indexed repos: re-run `index_repository`) - **Route nodes**: REST endpoints are first-class graph entities - **CLI mode**: `codebase-memory-mcp cli search_graph --project my-project --name-pattern ".*Handler.*"` -- **Available on**: npm, PyPI, Homebrew, Scoop, Winget, Chocolatey, AUR, `go install` +- **Available on**: npm, PyPI, Homebrew, Scoop, AUR, `go install` ### Indexing modes diff --git a/docs/index.html b/docs/index.html index 2add87b467..e39ded17e5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -540,7 +540,7 @@

How do I install codebase-memory-mcp?

Warp MCP, JetBrains AI/ACP, GitHub Copilot coding agent, Jules, CodeRabbit, Replit, BLACKBOX AI, Plandex, and SWE-agent. Warp is counted above for its detected skill installation; its MCP connection remains manual. Windows users run install.ps1. Also available via npm, - pip, Homebrew, Scoop, Winget, Chocolatey, AUR, and go install. + pip, Homebrew, Scoop, AUR, and go install.

Lifecycle installation follows documented context contracts: Qoder uses SessionStart, @@ -805,7 +805,7 @@

Does it support semantic or natural-language code search?

Do I need Docker or a runtime?

The native install needs no language runtime. It is a native executable with authenticated release-owned - assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64); npm, PyPI, and Go wrapper + assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64); npm, PyPI, and Go wrapper installations naturally use their package runtime.

How does it stay up to date as I edit code?

diff --git a/docs/llms.txt b/docs/llms.txt index f3c23970b5..0d0c5a49cd 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -15,7 +15,7 @@ - Supported agents: 45 automatic/conditional client surfaces (39 automatically detected + 6 conditional/explicit): Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Grok Build, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Continue / cn, Visual Studio, TRAE, Roo Code, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob IDE, IBM Bob Shell, Pochi, Pi, Oh My Pi (omp), and Sourcegraph Cody. - Agent profiles: documented custom-agent formats receive Scout (fast/provisional), Verify (default/task-directed), and Auditor (bounded/full verification) definitions. Every direct tier checks exact path/scope coverage with check_index_coverage and falls back to source for flagged gaps; unsafe child-MCP formats use explicit parent handoff. Kiro and Junie use positive-allowlist Scout/Analysis server profiles (7/11 tools); Qoder combines named-server selection with exact tier-specific MCP tool IDs, while Factory uses exact registered IDs without additive whole-server exposure. Foreign Junie aliases are preserved and force parent handoff. - Performance: Linux kernel (28M LOC, 75K files) full index in 3 minutes → 4.81M nodes, 7.72M edges; Cypher queries in under 1ms. -- Distribution: verified runtime set (native executable plus authenticated release-owned assets); fully static Linux builds are available in the `-portable` archives. Also available through npm, PyPI, Homebrew, Scoop, Winget, Chocolatey, AUR, and `go install`. +- Distribution: verified runtime set (native executable plus authenticated release-owned assets); fully static Linux builds are available in the `-portable` archives. Also available through npm, PyPI, Homebrew, Scoop, AUR, and `go install`. ## Links diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index 6eec48a560..9f8291e87c 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -7,6 +7,12 @@ // Install: // // go install github.com/DeusData/codebase-memory-mcp/pkg/go/cmd/codebase-memory-mcp@latest +// +// For testing, CBM_DOWNLOAD_URL overrides the download base URL (default +// https://github.com/DeusData/codebase-memory-mcp/releases/download/v): +// the archive and checksums.txt are fetched from /. It +// must be https://, or plain http:// to exactly localhost, 127.0.0.1, or [::1] +// (optional port; no proxy, no redirects). package main import ( @@ -27,7 +33,9 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" + "runtime/debug" "sort" "strings" "syscall" @@ -39,16 +47,12 @@ const ( version = "0.12.0" windowsBinaryName = "codebase-memory-mcp.exe" - maxRedirects = 5 - requestTimeout = 2 * time.Minute - connectTimeout = 15 * time.Second - responseHeaderTimeout = 30 * time.Second - candidateTimeout = 15 * time.Second - maxChecksumManifestSize = 1024 * 1024 - maxReleaseArchiveSize = int64(256 * 1024 * 1024) - maxArchiveMembers = 64 - maxArchiveMemberSize = int64(256 * 1024 * 1024) - maxArchiveExpandedSize = int64(512 * 1024 * 1024) + maxRedirects = 5 + requestTimeout = 2 * time.Minute + connectTimeout = 15 * time.Second + responseHeaderTimeout = 30 * time.Second + candidateTimeout = 15 * time.Second + maxArchiveMembers = 64 runtimeSetLockName = ".codebase-memory-mcp-runtime.lock" runtimeSetLockWait = 45 * time.Second @@ -66,20 +70,6 @@ const ( runtimeBackupCleanupRemovedEvent = "cleanup-member-removed" ) -type archiveResourceLimits struct { - compressedBytes int64 - members int - memberBytes int64 - expandedBytes int64 -} - -var defaultArchiveResourceLimits = archiveResourceLimits{ - compressedBytes: maxReleaseArchiveSize, - members: maxArchiveMembers, - memberBytes: maxArchiveMemberSize, - expandedBytes: maxArchiveExpandedSize, -} - type runtimeSetLock struct { path string token string @@ -169,7 +159,7 @@ func ensureBinary() (string, error) { func binPath() string { return filepath.Join( - cacheDir(), version, binaryNameForOS(runtime.GOOS), + cacheDir(), installVersion, binaryNameForOS(runtime.GOOS), ) } @@ -375,10 +365,14 @@ func download(dest string) error { "codebase-memory-mcp-%s-%s%s.%s", platform, arch, portable, ext, ) - url := fmt.Sprintf("https://github.com/%s/releases/download/v%s/%s", repo, version, archive) - checksumURL := fmt.Sprintf("https://github.com/%s/releases/download/v%s/checksums.txt", repo, version) + base, loopback, err := downloadBase(os.Getenv("CBM_DOWNLOAD_URL")) + if err != nil { + return err + } + url := base + "/" + archive + checksumURL := base + "/checksums.txt" - fmt.Fprintf(os.Stderr, "codebase-memory-mcp: downloading v%s for %s/%s...\n", version, platform, arch) + fmt.Fprintf(os.Stderr, "codebase-memory-mcp: downloading v%s for %s/%s...\n", installVersion, platform, arch) tmp, err := os.MkdirTemp("", "cbm-install-*") if err != nil { @@ -387,13 +381,13 @@ func download(dest string) error { defer os.RemoveAll(tmp) archivePath := filepath.Join(tmp, "cbm."+ext) - if err := httpGet(url, archivePath); err != nil { + if err := httpGet(url, archivePath, loopback); err != nil { return fmt.Errorf("download failed: %w", err) } // A release binary is executable input, so checksum verification is a // mandatory precondition rather than a best-effort warning. - checksums, err := fetchChecksums(checksumURL) + checksums, err := fetchChecksums(checksumURL, loopback) if err != nil { return fmt.Errorf("checksum manifest unavailable: %w", err) } @@ -452,6 +446,64 @@ func download(dest string) error { return nil } +// loopbackHTTPURL is install.sh's pattern: plain HTTP only to an exact +// loopback authority, for local smoke tests. +// installVersion is the release this wrapper installs: its own module version +// when Go installed it as a tagged release (`go install +// .../pkg/go/cmd/codebase-memory-mcp@v0.12.1`, or @latest, which resolves to +// the newest pkg/go/v* tag), else the version constant (a local build, or a +// pseudo-version of an untagged commit). +var installVersion = releaseVersionFrom(moduleVersion(), version) + +var releaseTag = regexp.MustCompile(`^v([0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?)$`) + +func moduleVersion() string { + if info, ok := debug.ReadBuildInfo(); ok { + return info.Main.Version + } + return "" +} + +// releaseVersionFrom maps a module version to a release version: "v0.12.1" is +// "0.12.1"; "(devel)", a pseudo-version or "" fall back. +func releaseVersionFrom(module, fallback string) string { + if m := releaseTag.FindStringSubmatch(module); m != nil { + return m[1] + } + return fallback +} + +var loopbackHTTPURL = regexp.MustCompile( + `^http://(localhost|127\.0\.0\.1|\[::1\])(:[0-9]+)?([/?#].*)?$`, +) + +var loopbackHostnames = map[string]bool{ + "localhost": true, + "127.0.0.1": true, + "::1": true, +} + +// downloadBase returns the release download base and whether it is a +// loopback HTTP base. CBM_DOWNLOAD_URL replaces the GitHub release base (for +// testing) with install.sh's policy: HTTPS, or plain HTTP to an exact +// loopback authority. +func downloadBase(override string) (string, bool, error) { + if override == "" { + return fmt.Sprintf( + "https://github.com/%s/releases/download/v%s", repo, installVersion, + ), false, nil + } + if strings.HasPrefix(override, "https://") { + return override, false, nil + } + if loopbackHTTPURL.MatchString(override) { + return override, true, nil + } + return "", false, fmt.Errorf( + "refusing non-HTTPS download URL (CBM_DOWNLOAD_URL): %s", override, + ) +} + // validateURLScheme rejects non-https URLs before any fetch (defense-in-depth). func validateURLScheme(rawURL string) error { parsed, err := url.Parse(rawURL) @@ -461,6 +513,17 @@ func validateURLScheme(rawURL string) error { return nil } +// validateLoopbackURL accepts only plain HTTP to an exact loopback authority. +func validateLoopbackURL(rawURL string) error { + parsed, err := url.Parse(rawURL) + if err != nil || !loopbackHTTPURL.MatchString(rawURL) || + parsed.Scheme != "http" || parsed.User != nil || + !loopbackHostnames[parsed.Hostname()] { + return fmt.Errorf("refusing non-loopback HTTP URL: %s", rawURL) + } + return nil +} + // httpsOnlyClient returns an HTTP client that rejects non-HTTPS redirects. var httpsOnlyClient = &http.Client{ Timeout: requestTimeout, @@ -483,18 +546,40 @@ var httpsOnlyClient = &http.Client{ }, } -func httpGet(rawURL, dest string) error { - return httpGetWithLimit(rawURL, dest, maxReleaseArchiveSize) +// loopbackClient serves a loopback CBM_DOWNLOAD_URL: it uses no proxy and +// refuses every redirect, so no hop can leave the loopback authority. +var loopbackClient = &http.Client{ + Timeout: requestTimeout, + Transport: &http.Transport{ + Proxy: nil, + DialContext: (&net.Dialer{Timeout: connectTimeout}).DialContext, + ResponseHeaderTimeout: responseHeaderTimeout, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return fmt.Errorf("refusing redirect from loopback download: %s", req.URL) + }, } -func httpGetWithLimit(rawURL, dest string, maxBytes int64) error { +// downloadClient validates rawURL for its mode and returns the client for it. +func downloadClient(rawURL string, loopback bool) (*http.Client, error) { + if loopback { + if err := validateLoopbackURL(rawURL); err != nil { + return nil, err + } + return loopbackClient, nil + } if err := validateURLScheme(rawURL); err != nil { - return err + return nil, err } - if maxBytes <= 0 { - return fmt.Errorf("invalid compressed archive safety limit") + return httpsOnlyClient, nil +} + +func httpGet(rawURL, dest string, loopback bool) error { + client, err := downloadClient(rawURL, loopback) + if err != nil { + return err } - resp, err := httpsOnlyClient.Get(rawURL) //nolint:gosec + resp, err := client.Get(rawURL) //nolint:gosec if err != nil { return err } @@ -502,24 +587,12 @@ func httpGetWithLimit(rawURL, dest string, maxBytes int64) error { if resp.StatusCode != http.StatusOK { return fmt.Errorf("HTTP %d for %s", resp.StatusCode, rawURL) } - if resp.ContentLength > maxBytes { - return fmt.Errorf( - "release archive exceeds the %d-byte compressed safety limit", - maxBytes, - ) - } f, err := os.Create(dest) if err != nil { return err } - written, copyErr := io.Copy(f, io.LimitReader(resp.Body, maxBytes+1)) + _, copyErr := io.Copy(f, resp.Body) closeErr := f.Close() - if copyErr == nil && written > maxBytes { - copyErr = fmt.Errorf( - "release archive exceeds the %d-byte compressed safety limit", - maxBytes, - ) - } if copyErr != nil { _ = os.Remove(dest) return copyErr @@ -530,11 +603,12 @@ func httpGetWithLimit(rawURL, dest string, maxBytes int64) error { return closeErr } -func fetchChecksums(url string) (map[string]string, error) { - if err := validateURLScheme(url); err != nil { +func fetchChecksums(url string, loopback bool) (map[string]string, error) { + client, err := downloadClient(url, loopback) + if err != nil { return nil, err } - resp, err := httpsOnlyClient.Get(url) //nolint:gosec + resp, err := client.Get(url) //nolint:gosec if err != nil { return nil, err } @@ -542,13 +616,10 @@ func fetchChecksums(url string) (map[string]string, error) { if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } - body, err := io.ReadAll(io.LimitReader(resp.Body, maxChecksumManifestSize+1)) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } - if len(body) > maxChecksumManifestSize { - return nil, fmt.Errorf("checksums.txt exceeds the 1 MiB safety limit") - } result := make(map[string]string) for _, line := range strings.Split(string(body), "\n") { parts := strings.Fields(line) @@ -634,17 +705,7 @@ func validateArchiveMemberNames( return nil } -func validateArchiveResourceLimits(limits archiveResourceLimits) error { - if limits.compressedBytes <= 0 || limits.members <= 0 || - limits.memberBytes <= 0 || limits.expandedBytes <= 0 { - return fmt.Errorf("invalid archive resource safety limits") - } - return nil -} - -func requireCompressedArchiveWithinLimit( - archivePath string, limits archiveResourceLimits, -) error { +func requireRegularArchive(archivePath string) error { status, err := os.Stat(archivePath) if err != nil { return err @@ -652,91 +713,36 @@ func requireCompressedArchiveWithinLimit( if !status.Mode().IsRegular() { return fmt.Errorf("release archive is not a regular file") } - if status.Size() > limits.compressedBytes { - return fmt.Errorf( - "release archive exceeds the %d-byte compressed safety limit", - limits.compressedBytes, - ) - } return nil } -func accountDeclaredArchiveMember( - name string, size int64, expanded *int64, limits archiveResourceLimits, +// copyArchiveMember copies one member and requires its declared size. +func copyArchiveMember( + destination io.Writer, source io.Reader, name string, declaredSize int64, ) error { - if size < 0 { - return fmt.Errorf("archive member has a negative declared size: %q", name) - } - if size > limits.memberBytes { - return fmt.Errorf( - "archive member %q exceeds the %d-byte expanded safety limit", - name, limits.memberBytes, - ) - } - if *expanded > limits.expandedBytes-size { - return fmt.Errorf( - "archive exceeds the %d-byte aggregate expanded safety limit", - limits.expandedBytes, - ) - } - *expanded += size - return nil -} - -func copyArchiveMemberWithLimits( - destination io.Writer, - source io.Reader, - name string, - declaredSize int64, - actualExpanded *int64, - limits archiveResourceLimits, -) error { - remaining := limits.expandedBytes - *actualExpanded - copyLimit := limits.memberBytes - if remaining < copyLimit { - copyLimit = remaining - } - copied, err := io.Copy( - destination, io.LimitReader(source, copyLimit+1), - ) + copied, err := io.Copy(destination, source) if err != nil { return err } - if copied > limits.memberBytes { - return fmt.Errorf( - "archive member %q exceeds the %d-byte actual expanded safety limit", - name, limits.memberBytes, - ) - } - if copied > remaining { - return fmt.Errorf( - "archive exceeds the %d-byte aggregate actual expanded safety limit", - limits.expandedBytes, - ) - } if copied != declaredSize { return fmt.Errorf( "archive member %q actual size %d does not match declared size %d", name, copied, declaredSize, ) } - *actualExpanded += copied return nil } func tarGzMemberNames(archivePath string) ([]string, error) { - return tarGzMemberNamesWithLimits( - archivePath, defaultArchiveResourceLimits, - ) + return tarGzMemberNamesWithMemberLimit(archivePath, maxArchiveMembers) } -func tarGzMemberNamesWithLimits( - archivePath string, limits archiveResourceLimits, +// tarGzMemberNamesWithMemberLimit lists the root members; the entry count +// guard bounds the listing before the exact namespace check. +func tarGzMemberNamesWithMemberLimit( + archivePath string, maxMembers int, ) ([]string, error) { - if err := validateArchiveResourceLimits(limits); err != nil { - return nil, err - } - if err := requireCompressedArchiveWithinLimit(archivePath, limits); err != nil { + if err := requireRegularArchive(archivePath); err != nil { return nil, err } f, err := os.Open(archivePath) @@ -751,7 +757,6 @@ func tarGzMemberNamesWithLimits( defer gz.Close() tr := tar.NewReader(gz) var names []string - var declaredExpanded int64 for { hdr, err := tr.Next() if err == io.EOF { @@ -765,16 +770,11 @@ func tarGzMemberNamesWithLimits( "archive member is not a regular root file: %q", hdr.Name, ) } - if len(names) >= limits.members { + if len(names) >= maxMembers { return nil, fmt.Errorf( - "archive exceeds the %d-member safety limit", limits.members, + "archive exceeds the %d-member safety limit", maxMembers, ) } - if err := accountDeclaredArchiveMember( - hdr.Name, hdr.Size, &declaredExpanded, limits, - ); err != nil { - return nil, err - } names = append(names, hdr.Name) } return names, nil @@ -784,21 +784,21 @@ func extractTarGz( archivePath, destDir string, archiveNames, extractNames []string, ) ([]string, error) { - return extractTarGzWithLimits( + return extractTarGzWithMemberLimit( archivePath, destDir, archiveNames, extractNames, - defaultArchiveResourceLimits, + maxArchiveMembers, ) } -func extractTarGzWithLimits( +func extractTarGzWithMemberLimit( archivePath, destDir string, archiveNames, extractNames []string, - limits archiveResourceLimits, + maxMembers int, ) ([]string, error) { - names, err := tarGzMemberNamesWithLimits(archivePath, limits) + names, err := tarGzMemberNamesWithMemberLimit(archivePath, maxMembers) if err != nil { return nil, err } @@ -824,8 +824,6 @@ func extractTarGzWithLimits( tr := tar.NewReader(gz) extracted := make(map[string]struct{}, len(runtimeNames)) memberCount := 0 - var declaredExpanded int64 - var actualExpanded int64 for { hdr, err := tr.Next() if err == io.EOF { @@ -835,9 +833,9 @@ func extractTarGzWithLimits( return nil, err } memberCount++ - if memberCount > limits.members { + if memberCount > maxMembers { return nil, fmt.Errorf( - "archive exceeds the %d-member safety limit", limits.members, + "archive exceeds the %d-member safety limit", maxMembers, ) } if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { @@ -845,11 +843,6 @@ func extractTarGzWithLimits( "archive member is not a regular root file: %q", hdr.Name, ) } - if err := accountDeclaredArchiveMember( - hdr.Name, hdr.Size, &declaredExpanded, limits, - ); err != nil { - return nil, err - } if _, wanted := targets[hdr.Name]; wanted { outputPath := filepath.Join(destDir, hdr.Name) out, err := os.OpenFile( @@ -860,9 +853,7 @@ func extractTarGzWithLimits( if err != nil { return nil, err } - copyErr := copyArchiveMemberWithLimits( - out, tr, hdr.Name, hdr.Size, &actualExpanded, limits, - ) + copyErr := copyArchiveMember(out, tr, hdr.Name, hdr.Size) closeErr := out.Close() if copyErr != nil { _ = os.Remove(outputPath) @@ -873,8 +864,8 @@ func extractTarGzWithLimits( return nil, closeErr } extracted[hdr.Name] = struct{}{} - } else if err := copyArchiveMemberWithLimits( - io.Discard, tr, hdr.Name, hdr.Size, &actualExpanded, limits, + } else if err := copyArchiveMember( + io.Discard, tr, hdr.Name, hdr.Size, ); err != nil { return nil, err } @@ -889,24 +880,21 @@ func extractZip( archivePath, destDir string, archiveNames, extractNames []string, ) ([]string, error) { - return extractZipWithLimits( + return extractZipWithMemberLimit( archivePath, destDir, archiveNames, extractNames, - defaultArchiveResourceLimits, + maxArchiveMembers, ) } -func extractZipWithLimits( +func extractZipWithMemberLimit( archivePath, destDir string, archiveNames, extractNames []string, - limits archiveResourceLimits, + maxMembers int, ) ([]string, error) { - if err := validateArchiveResourceLimits(limits); err != nil { - return nil, err - } - if err := requireCompressedArchiveWithinLimit(archivePath, limits); err != nil { + if err := requireRegularArchive(archivePath); err != nil { return nil, err } r, err := zip.OpenReader(archivePath) @@ -914,14 +902,13 @@ func extractZipWithLimits( return nil, err } defer r.Close() - if len(r.File) > limits.members { + if len(r.File) > maxMembers { return nil, fmt.Errorf( - "archive exceeds the %d-member safety limit", limits.members, + "archive exceeds the %d-member safety limit", maxMembers, ) } names := make([]string, 0, len(r.File)) declaredSizes := make(map[string]int64, len(r.File)) - var declaredExpanded int64 for _, f := range r.File { name, err := validateWindowsZipMember(f.Name) if err != nil { @@ -932,18 +919,12 @@ func extractZipWithLimits( "archive member is not a regular root file: %q", f.Name, ) } - if f.UncompressedSize64 > uint64(limits.memberBytes) { + declaredSize := int64(f.UncompressedSize64) + if declaredSize < 0 { return nil, fmt.Errorf( - "archive member %q exceeds the %d-byte expanded safety limit", - name, limits.memberBytes, + "archive member has an invalid declared size: %q", name, ) } - declaredSize := int64(f.UncompressedSize64) - if err := accountDeclaredArchiveMember( - name, declaredSize, &declaredExpanded, limits, - ); err != nil { - return nil, err - } names = append(names, name) declaredSizes[name] = declaredSize } @@ -956,7 +937,6 @@ func extractZipWithLimits( targetNames[name] = struct{}{} } extracted := make(map[string]struct{}, len(runtimeNames)) - var actualExpanded int64 for index, target := range r.File { name := names[index] @@ -980,14 +960,7 @@ func extractZipWithLimits( } destination = out } - copyErr := copyArchiveMemberWithLimits( - destination, - rc, - name, - declaredSizes[name], - &actualExpanded, - limits, - ) + copyErr := copyArchiveMember(destination, rc, name, declaredSizes[name]) var closeOutErr error if out != nil { closeOutErr = out.Close() diff --git a/pkg/go/cmd/codebase-memory-mcp/main_test.go b/pkg/go/cmd/codebase-memory-mcp/main_test.go index 9a30c6525e..17a9f0e1a3 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main_test.go +++ b/pkg/go/cmd/codebase-memory-mcp/main_test.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -29,18 +30,18 @@ func (transport archiveTestRoundTripper) RoundTrip( return transport(request) } -type archiveCountingBody struct { - reader io.Reader - read int64 -} +// formerMemberCapBytes is the size cap the 0.12.0 wrappers enforced, which +// refused every release. +const formerMemberCapBytes = int64(256 * 1024 * 1024) -func (body *archiveCountingBody) Read(buffer []byte) (int, error) { - count, err := body.reader.Read(buffer) - body.read += int64(count) - return count, err -} +type zeroReader struct{} -func (*archiveCountingBody) Close() error { return nil } +func (zeroReader) Read(buffer []byte) (int, error) { + for index := range buffer { + buffer[index] = 0 + } + return len(buffer), nil +} func assertRuntimeTag(t *testing.T, directory, binaryName, tag string) { t.Helper() @@ -157,164 +158,167 @@ func writeTarGz(t *testing.T, archivePath string, names []string) { } } -func TestCompressedArchiveDownloadRejectsDeclaredAndActualOverflow(t *testing.T) { - const maxBytes = int64(8) +func TestDownloadIgnoresADeclaredLengthAboveTheFormerCap(t *testing.T) { priorClient := httpsOnlyClient defer func() { httpsOnlyClient = priorClient }() + httpsOnlyClient = &http.Client{Transport: archiveTestRoundTripper( + func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("contents")), + ContentLength: formerMemberCapBytes + 1, + Header: make(http.Header), + Request: request, + }, nil + }, + )} + destination := filepath.Join(t.TempDir(), "release.tar.gz") + if err := httpGet( + "https://example.invalid/release.tar.gz", destination, false, + ); err != nil { + t.Fatalf("download error = %v", err) + } + contents, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if string(contents) != "contents" { + t.Fatalf("download stored %q", contents) + } +} - t.Run("declared", func(t *testing.T) { - httpsOnlyClient = &http.Client{Transport: archiveTestRoundTripper( - func(request *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader("")), - ContentLength: maxBytes + 1, - Header: make(http.Header), - Request: request, - }, nil - }, - )} - destination := filepath.Join(t.TempDir(), "release.tar.gz") - err := httpGetWithLimit( - "https://example.invalid/release.tar.gz", destination, maxBytes, - ) - if err == nil || !strings.Contains(err.Error(), "compressed safety limit") { - t.Fatalf("compressed declared overflow error = %v", err) - } - if _, statErr := os.Stat(destination); !os.IsNotExist(statErr) { - t.Fatal("declared oversized archive created a download") - } - }) - - t.Run("actual", func(t *testing.T) { - body := &archiveCountingBody{reader: bytes.NewReader( - bytes.Repeat([]byte("x"), 1024), - )} - httpsOnlyClient = &http.Client{Transport: archiveTestRoundTripper( - func(request *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: http.StatusOK, - Body: body, - ContentLength: -1, - Header: make(http.Header), - Request: request, - }, nil - }, - )} - destination := filepath.Join(t.TempDir(), "release.tar.gz") - err := httpGetWithLimit( - "https://example.invalid/release.tar.gz", destination, maxBytes, - ) - if err == nil || !strings.Contains(err.Error(), "compressed safety limit") { - t.Fatalf("compressed actual overflow error = %v", err) +func TestTarArchiveRejectsMoreMembersThanTheLimit(t *testing.T) { + archivePath := filepath.Join(t.TempDir(), "members.tar.gz") + writeTarGz(t, archivePath, []string{"one", "two", "three"}) + _, err := tarGzMemberNamesWithMemberLimit(archivePath, 2) + if err == nil || !strings.Contains(err.Error(), "2-member safety limit") { + t.Fatalf("tar member count error = %v", err) + } +} + +func TestArchiveCopyAcceptsAMemberLargerThanTheFormerCap(t *testing.T) { + size := formerMemberCapBytes + 1024*1024 + if err := copyArchiveMember( + io.Discard, io.LimitReader(zeroReader{}, size), "codebase-memory-mcp", size, + ); err != nil { + t.Fatalf("copy of a %d-byte member error = %v", size, err) + } +} + +func TestArchiveCopyRequiresTheDeclaredSize(t *testing.T) { + var output bytes.Buffer + err := copyArchiveMember(&output, strings.NewReader("123"), "member", 2) + if err == nil || !strings.Contains(err.Error(), "does not match declared size") { + t.Fatalf("declared size mismatch error = %v", err) + } +} + +func TestDownloadBaseFollowsTheInstallShPolicy(t *testing.T) { + githubBase := "https://github.com/DeusData/codebase-memory-mcp/releases/download/v" + version + if base, loopback, err := downloadBase(""); err != nil || base != githubBase || loopback { + t.Fatalf("unset override = (%q, %v, %v), want the GitHub base", base, loopback, err) + } + for _, override := range []string{ + "https://mirror.example/releases/v1", + "https://127.0.0.1:8443/fixture", + } { + base, loopback, err := downloadBase(override) + if err != nil || base != override || loopback { + t.Errorf("https override %q = (%q, %v, %v)", override, base, loopback, err) } - if body.read != maxBytes+1 { - t.Fatalf("oversized response bytes consumed = %d, want %d", body.read, maxBytes+1) + } + for _, override := range []string{ + "http://localhost", + "http://localhost:8080/fixture", + "http://127.0.0.1:41234/standard", + "http://[::1]:9/x?y#z", + } { + base, loopback, err := downloadBase(override) + if err != nil || base != override || !loopback { + t.Errorf("loopback override %q = (%q, %v, %v)", override, base, loopback, err) } - if _, statErr := os.Stat(destination); !os.IsNotExist(statErr) { - t.Fatal("rejected compressed archive left a partial download") + } + for _, override := range []string{ + "http://example.com/releases", + "http://127.0.0.1.example.com/", + "http://localhost.example.com:8080/", + "http://user@localhost/", + "http://127.0.0.2/", + "HTTP://localhost/", + "file:///tmp/releases", + "ftp://localhost/releases", + "github.com/DeusData/codebase-memory-mcp", + } { + _, _, err := downloadBase(override) + if err == nil || !strings.Contains( + err.Error(), "refusing non-HTTPS download URL (CBM_DOWNLOAD_URL)", + ) { + t.Errorf("override %q error = %v, want a refusal", override, err) } - }) + } } -func testArchiveLimits( - members int, memberBytes, expandedBytes int64, -) archiveResourceLimits { - return archiveResourceLimits{ - compressedBytes: 1024 * 1024, - members: members, - memberBytes: memberBytes, - expandedBytes: expandedBytes, +func TestDownloadRefusesURLsOutsideTheirMode(t *testing.T) { + destination := filepath.Join(t.TempDir(), "release.tar.gz") + err := httpGet("http://127.0.0.1/release.tar.gz", destination, false) + if err == nil || !strings.Contains(err.Error(), "refusing non-https URL") { + t.Fatalf("plain HTTP without a loopback base error = %v", err) + } + err = httpGet("http://example.invalid/release.tar.gz", destination, true) + if err == nil || !strings.Contains(err.Error(), "refusing non-loopback HTTP URL") { + t.Fatalf("non-loopback URL with a loopback base error = %v", err) + } + if _, statErr := os.Stat(destination); !os.IsNotExist(statErr) { + t.Fatal("refused download created a file") } } -func TestTarArchiveRejectsMemberAndExpandedResourceOverflow(t *testing.T) { - root := t.TempDir() - tests := []struct { - name string - names []string - limits archiveResourceLimits - wantError string - }{ - { - name: "member count", - names: []string{"one", "two", "three"}, - limits: testArchiveLimits(2, 1024, 4096), - wantError: "member safety limit", - }, - { - name: "declared member bytes", - names: []string{"oversized"}, - limits: testArchiveLimits(4, 4, 4096), - wantError: "expanded safety limit", - }, - { - name: "declared aggregate bytes", - names: []string{"one", "two"}, - limits: testArchiveLimits(4, 1024, 15), - wantError: "aggregate expanded safety limit", +func TestLoopbackDownloadBypassesProxiesAndRefusesRedirects(t *testing.T) { + transport, ok := loopbackClient.Transport.(*http.Transport) + if !ok || transport.Proxy != nil { + t.Fatal("the loopback client must not use a proxy") + } + digest := strings.Repeat("a", 64) + server := httptest.NewServer(http.HandlerFunc( + func(response http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/fixture/archive": + _, _ = io.WriteString(response, "loopback contents") + case "/fixture/checksums.txt": + _, _ = io.WriteString(response, digest+" archive\n") + default: + http.Redirect( + response, request, "https://example.invalid/elsewhere", http.StatusFound, + ) + } }, + )) + defer server.Close() + root := t.TempDir() + + archivePath := filepath.Join(root, "archive") + if err := httpGet(server.URL+"/fixture/archive", archivePath, true); err != nil { + t.Fatalf("loopback download error = %v", err) } - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - archivePath := filepath.Join(root, strings.ReplaceAll(testCase.name, " ", "-")+".tar.gz") - writeTarGz(t, archivePath, testCase.names) - _, err := tarGzMemberNamesWithLimits(archivePath, testCase.limits) - if err == nil || !strings.Contains(err.Error(), testCase.wantError) { - t.Fatalf("tar resource overflow error = %v", err) - } - }) + contents, err := os.ReadFile(archivePath) + if err != nil { + t.Fatal(err) + } + if string(contents) != "loopback contents" { + t.Fatalf("loopback download stored %q", contents) + } + checksums, err := fetchChecksums(server.URL+"/fixture/checksums.txt", true) + if err != nil || checksums["archive"] != digest { + t.Fatalf("loopback checksums = %v, %v", checksums, err) } -} -func TestArchiveCopyRejectsActualMemberAggregateAndMetadataMismatch(t *testing.T) { - tests := []struct { - name string - contents string - declared int64 - actual int64 - limits archiveResourceLimits - wantError string - }{ - { - name: "actual member bytes", - contents: "12345", - declared: 5, - limits: testArchiveLimits(1, 4, 10), - wantError: "actual expanded safety limit", - }, - { - name: "actual aggregate bytes", - contents: "12", - declared: 2, - actual: 4, - limits: testArchiveLimits(1, 10, 5), - wantError: "aggregate actual expanded safety limit", - }, - { - name: "declared actual mismatch", - contents: "123", - declared: 2, - limits: testArchiveLimits(1, 10, 10), - wantError: "does not match declared size", - }, + redirected := filepath.Join(root, "redirected") + err = httpGet(server.URL+"/fixture/elsewhere", redirected, true) + if err == nil || !strings.Contains(err.Error(), "refusing redirect from loopback download") { + t.Fatalf("loopback redirect error = %v", err) } - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - actual := testCase.actual - var output bytes.Buffer - err := copyArchiveMemberWithLimits( - &output, - strings.NewReader(testCase.contents), - "member", - testCase.declared, - &actual, - testCase.limits, - ) - if err == nil || !strings.Contains(err.Error(), testCase.wantError) { - t.Fatalf("actual archive overflow error = %v", err) - } - }) + if _, statErr := os.Stat(redirected); !os.IsNotExist(statErr) { + t.Fatal("a refused loopback redirect created a file") } } @@ -342,52 +346,20 @@ func writeZip(t *testing.T, archivePath string, names []string) { } } -func TestZipArchiveRejectsMemberAndExpandedResourceOverflow(t *testing.T) { +func TestZipArchiveRejectsMoreMembersThanTheLimit(t *testing.T) { root := t.TempDir() - tests := []struct { - name string - names []string - limits archiveResourceLimits - wantError string - }{ - { - name: "member count", - names: []string{"one", "two", "three"}, - limits: testArchiveLimits(2, 1024, 4096), - wantError: "member safety limit", - }, - { - name: "declared member bytes", - names: []string{"oversized"}, - limits: testArchiveLimits(4, 4, 4096), - wantError: "expanded safety limit", - }, - { - name: "declared aggregate bytes", - names: []string{"one", "two"}, - limits: testArchiveLimits(4, 1024, 15), - wantError: "aggregate expanded safety limit", - }, + names := []string{"one", "two", "three"} + archivePath := filepath.Join(root, "members.zip") + writeZip(t, archivePath, names) + destination := filepath.Join(root, "extract") + if err := os.Mkdir(destination, 0755); err != nil { + t.Fatal(err) } - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - archivePath := filepath.Join(root, strings.ReplaceAll(testCase.name, " ", "-")+".zip") - writeZip(t, archivePath, testCase.names) - destination := filepath.Join(root, strings.ReplaceAll(testCase.name, " ", "-")+"-extract") - if err := os.Mkdir(destination, 0755); err != nil { - t.Fatal(err) - } - _, err := extractZipWithLimits( - archivePath, - destination, - testCase.names, - testCase.names[:1], - testCase.limits, - ) - if err == nil || !strings.Contains(err.Error(), testCase.wantError) { - t.Fatalf("zip resource overflow error = %v", err) - } - }) + _, err := extractZipWithMemberLimit( + archivePath, destination, names, names[:1], 2, + ) + if err == nil || !strings.Contains(err.Error(), "2-member safety limit") { + t.Fatalf("zip member count error = %v", err) } } @@ -1400,3 +1372,19 @@ func TestRuntimeSetLockSerializesProcesses(t *testing.T) { t.Fatalf("child never acquired the released runtime-set lock: %v", err) } } + +func TestReleaseVersionFromModuleVersion(t *testing.T) { + cases := []struct{ module, want string }{ + {"v0.12.1", "0.12.1"}, + {"v0.9.1-rc.1", "0.9.1-rc.1"}, + {"(devel)", "fallback"}, + {"", "fallback"}, + {"v0.0.0-20261015165448-ec6427c38176", "fallback"}, + {"v0.12.2-0.20261016120000-0123456789ab", "fallback"}, + } + for _, c := range cases { + if got := releaseVersionFrom(c.module, "fallback"); got != c.want { + t.Errorf("releaseVersionFrom(%q) = %q, want %q", c.module, got, c.want) + } + } +} diff --git a/pkg/homebrew/Formula/codebase-memory-mcp.rb b/pkg/homebrew/Formula/codebase-memory-mcp.rb index 85347d8e44..eb64bcf2f3 100644 --- a/pkg/homebrew/Formula/codebase-memory-mcp.rb +++ b/pkg/homebrew/Formula/codebase-memory-mcp.rb @@ -1,28 +1,30 @@ class CodebaseMemoryMcp < Formula desc "Fast code intelligence engine for AI coding agents" homepage "https://github.com/DeusData/codebase-memory-mcp" - version "0.10.3" + version "0.12.0" license "MIT" on_macos do on_arm do url "https://github.com/DeusData/codebase-memory-mcp/releases/download/v#{version}/codebase-memory-mcp-darwin-arm64.tar.gz" - sha256 "0ebf02328207d4c3d862c837b5e973de5bac808df92b0941737721d467287f7f" + sha256 "1be41a9dd6902e3633eb0b280189117dc3db3be623bdd3dd4c6ea74372b5a50b" end on_intel do url "https://github.com/DeusData/codebase-memory-mcp/releases/download/v#{version}/codebase-memory-mcp-darwin-amd64.tar.gz" - sha256 "1107fea28285823e1436e4f38a4e00a0b472d8a43c379da7dfd200c914a4b9dd" + sha256 "af3beeeaca53204faf350402ddacee04780a53d0fa510dc31941fb5c2202ff35" end end + # The static -portable build: the dynamic one needs glibc 2.38, which + # Ubuntu 22.04, Debian 12 and RHEL 9 do not have. on_linux do on_arm do - url "https://github.com/DeusData/codebase-memory-mcp/releases/download/v#{version}/codebase-memory-mcp-linux-arm64.tar.gz" - sha256 "967b9eababfdbd2ef1987c571d55bc7c028cd1db7f99279830634c58db311e32" + url "https://github.com/DeusData/codebase-memory-mcp/releases/download/v#{version}/codebase-memory-mcp-linux-arm64-portable.tar.gz" + sha256 "8e603f32c9c84b5dc5a6b4a0d23516817d5de199e30f8dcc9bdb128c39386cd8" end on_intel do - url "https://github.com/DeusData/codebase-memory-mcp/releases/download/v#{version}/codebase-memory-mcp-linux-amd64.tar.gz" - sha256 "74997fb0934e70a22f20c2e112fb4d883867dc1f01a7bcdc94cf86d13b5cbd31" + url "https://github.com/DeusData/codebase-memory-mcp/releases/download/v#{version}/codebase-memory-mcp-linux-amd64-portable.tar.gz" + sha256 "7024e585132760f383e3e25237e2b2ac6f5185f7e466f8dc66a085bb25f8d0be" end end diff --git a/pkg/npm/README.md b/pkg/npm/README.md index c12a2d9c63..63dc88f0be 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -17,6 +17,8 @@ npm install -g codebase-memory-mcp The runtime set for your platform is downloaded automatically at install time. There is one composition per platform and the graph UI is always included — the former `CBM_VARIANT=ui` opt-in is obsolete. +For testing, `CBM_DOWNLOAD_URL` overrides the download base URL (default `https://github.com/DeusData/codebase-memory-mcp/releases/download/v`): the archive and `checksums.txt` are fetched from `/`. It must be `https://`, or plain `http://` to exactly `localhost`, `127.0.0.1`, or `[::1]` (optional port; no proxy, no redirects). + Then configure your coding agents: ```bash diff --git a/pkg/npm/install.js b/pkg/npm/install.js index ada940f27f..f3ec0e9efa 100644 --- a/pkg/npm/install.js +++ b/pkg/npm/install.js @@ -3,6 +3,7 @@ // Postinstall script: downloads the platform-appropriate binary from GitHub Releases. // Runs automatically via `postinstall` in package.json. +const http = require('http'); const https = require('https'); const crypto = require('crypto'); const fs = require('fs'); @@ -17,19 +18,14 @@ const BIN_DIR = path.join(__dirname, 'bin'); const MAX_REDIRECTS = 5; const DOWNLOAD_HOP_TIMEOUT_MS = 120_000; const CANDIDATE_TIMEOUT_MS = 15_000; -const MAX_CHECKSUM_MANIFEST_BYTES = 1024 * 1024; -// Release archive resource limits. Keep in sync with the Go wrapper -// (pkg/go/cmd/codebase-memory-mcp/main.go) and the PyPI wrapper (_cli.py). -const MAX_RELEASE_ARCHIVE_BYTES = 256 * 1024 * 1024; +// Entry-count guard for the zip namespace check. Keep in sync with the Go +// wrapper (pkg/go/cmd/codebase-memory-mcp/main.go) and the PyPI wrapper. const MAX_ARCHIVE_MEMBERS = 64; -const MAX_ARCHIVE_MEMBER_BYTES = 256 * 1024 * 1024; -const MAX_ARCHIVE_EXPANDED_BYTES = 512 * 1024 * 1024; -const DEFAULT_ARCHIVE_LIMITS = Object.freeze({ - compressedBytes: MAX_RELEASE_ARCHIVE_BYTES, - members: MAX_ARCHIVE_MEMBERS, - memberBytes: MAX_ARCHIVE_MEMBER_BYTES, - expandedBytes: MAX_ARCHIVE_EXPANDED_BYTES, -}); +const DEFAULT_DOWNLOAD_BASE = `https://github.com/${REPO}/releases/download/v${VERSION}`; +// install.sh's loopback pattern: plain HTTP only to an exact loopback +// authority, for local smoke tests. +const LOOPBACK_HTTP_URL = /^http:\/\/(localhost|127\.0\.0\.1|\[::1\])(:[0-9]+)?([/?#].*)?$/; +const LOOPBACK_HOSTNAMES = new Set(['localhost', '127.0.0.1', '[::1]']); // Bytes of tar's own diagnostics kept for the failure message. const MAX_TAR_STDERR_CHARS = 4096; const WINDOWS_BINARY_NAME = 'codebase-memory-mcp.exe'; @@ -83,6 +79,32 @@ function validateUrl(rawUrl) { return parsed.href; } +// Plain HTTP is accepted only for an exact loopback authority, as in +// install.sh; such a fetch uses no proxy and follows no redirect. +function validateLoopbackUrl(rawUrl) { + let parsed = null; + try { + parsed = new URL(rawUrl); + } catch (_) { + // Refused below. + } + if (!LOOPBACK_HTTP_URL.test(rawUrl) || !parsed || parsed.protocol !== 'http:' || + !LOOPBACK_HOSTNAMES.has(parsed.hostname) || parsed.username || parsed.password) { + throw new Error(`Refusing non-loopback HTTP URL: ${rawUrl}`); + } + return parsed.href; +} + +// CBM_DOWNLOAD_URL replaces the release download base (for testing), with +// install.sh's policy: HTTPS, or plain HTTP to an exact loopback authority. +function resolveDownloadBase(env = process.env) { + const override = env.CBM_DOWNLOAD_URL; + if (!override) return { base: DEFAULT_DOWNLOAD_BASE, loopback: false }; + if (override.startsWith('https://')) return { base: override, loopback: false }; + if (LOOPBACK_HTTP_URL.test(override)) return { base: override, loopback: true }; + throw new Error(`Refusing non-HTTPS download URL (CBM_DOWNLOAD_URL): ${override}`); +} + function validateExactTarMemberListing(listing, expectedNames) { if (listing.includes('\0')) { throw new Error('tar member listing contains a NUL byte'); @@ -104,31 +126,16 @@ function validateExactTarMemberListing(listing, expectedNames) { } } -function validateArchiveLimits(limits) { - if (!limits || !(limits.compressedBytes > 0) || !(limits.members > 0) || - !(limits.memberBytes > 0) || !(limits.expandedBytes > 0)) { - throw new Error('invalid archive resource safety limits'); - } -} - -function requireCompressedArchiveWithinLimit(archivePath, limits) { - const status = fs.statSync(archivePath); - if (!status.isFile()) { +function requireRegularArchive(archivePath) { + if (!fs.statSync(archivePath).isFile()) { throw new Error('release archive is not a regular file'); } - if (status.size > limits.compressedBytes) { - throw new Error( - `release archive exceeds the ${limits.compressedBytes}-byte compressed safety limit`, - ); - } } -// Streams one member through `tar -xzOf` into a file this call creates, -// counting every byte against the per-member limit. The file is removed only -// when this call created it: a pre-existing target fails untouched. -function extractTarMemberWithLimit( - archivePath, target, memberName, limits, spawnFile = spawn, -) { +// Streams one member through `tar -xzOf` into a file this call creates. The +// file is removed only when this call created it: a pre-existing target +// fails untouched. +function extractTarMember(archivePath, target, memberName, spawnFile = spawn) { return new Promise((resolve, reject) => { let fd; try { @@ -164,12 +171,6 @@ function extractTarMemberWithLimit( child.stdout.on('data', (chunk) => { if (failure) return; written += chunk.length; - if (written > limits.memberBytes) { - fail(new Error( - `archive member ${JSON.stringify(memberName)} exceeds the ${limits.memberBytes}-byte expanded safety limit`, - )); - return; - } try { let offset = 0; while (offset < chunk.length) { @@ -209,29 +210,24 @@ function extractTarMemberWithLimit( async function extractExactTarArchive( archivePath, destPath, expectedNames, targetName, - runFile = execFileSync, limits = DEFAULT_ARCHIVE_LIMITS, spawnFile = spawn, + runFile = execFileSync, spawnFile = spawn, ) { - validateArchiveLimits(limits); - requireCompressedArchiveWithinLimit(archivePath, limits); - // The exact root namespace is the member bound: its four names lie well - // within MAX_ARCHIVE_MEMBERS, which the zip path enforces by count. - if (expectedNames.length > limits.members) { - throw new Error(`archive namespace exceeds the ${limits.members}-member safety limit`); - } + requireRegularArchive(archivePath); + // The exact root namespace is the member bound: the listing must name + // exactly the expected root files. const listing = runFile('tar', ['-tzf', archivePath], { encoding: 'utf8', - maxBuffer: MAX_CHECKSUM_MANIFEST_BYTES, windowsHide: true, }); validateExactTarMemberListing(listing, expectedNames); - // Only the fixed root executable is ever written, through the counted - // writer; the companions are listed but never extracted. + // Only the fixed root executable is ever written; the companions are + // listed but never extracted. const target = path.join(destPath, targetName); - await extractTarMemberWithLimit(archivePath, target, targetName, limits, spawnFile); + await extractTarMember(archivePath, target, targetName, spawnFile); fs.chmodSync(target, 0o755); } -function downloadHop(url, dest, maxBytes) { +function downloadHop(url, dest, loopback) { return new Promise((resolve, reject) => { let settled = false; let response = null; @@ -245,7 +241,11 @@ function downloadHop(url, dest, maxBytes) { else resolve(result); } - const req = https.get(url, (res) => { + // A loopback hop gets a fresh agent, so no proxy configuration applies. + const get = loopback + ? (target, callback) => http.get(target, { agent: false }, callback) + : (target, callback) => https.get(target, callback); + const req = get(url, (res) => { response = res; let piped = false; // Before the body is piped, a response error ends the hop directly; @@ -255,6 +255,11 @@ function downloadHop(url, dest, maxBytes) { const redirectCodes = new Set([301, 302, 303, 307, 308]); if (redirectCodes.has(res.statusCode)) { const location = res.headers.location; + if (loopback) { + res.destroy(); + finish(new Error(`Refusing redirect from loopback download ${url}`)); + return; + } if (!location) { res.destroy(); finish(new Error(`Redirect with no location for ${url}`)); @@ -277,20 +282,7 @@ function downloadHop(url, dest, maxBytes) { finish(new Error(`HTTP ${res.statusCode} for ${url}`)); return; } - const declared = Number(res.headers['content-length']); - if (maxBytes && Number.isFinite(declared) && declared > maxBytes) { - res.destroy(); - finish(new Error(`Download exceeds the ${maxBytes}-byte safety limit`)); - return; - } - let received = 0; - res.on('data', (chunk) => { - received += chunk.length; - if (maxBytes && received > maxBytes) { - res.destroy(new Error(`Download exceeds the ${maxBytes}-byte safety limit`)); - } - }); const file = fs.createWriteStream(dest, { flags: 'w' }); piped = true; pipeline(res, file, (err) => { @@ -317,10 +309,10 @@ function downloadHop(url, dest, maxBytes) { }); } -async function download(rawUrl, dest, maxBytes = 0) { - let url = validateUrl(rawUrl); +async function download(rawUrl, dest, loopback = false) { + let url = loopback ? validateLoopbackUrl(rawUrl) : validateUrl(rawUrl); for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects += 1) { - const result = await downloadHop(url, dest, maxBytes); + const result = await downloadHop(url, dest, loopback); if (!result.redirect) return; if (redirects === MAX_REDIRECTS) throw new Error('Too many redirects'); url = result.redirect; @@ -605,10 +597,9 @@ function installWindowsBinaryAtomically(sourceDir, destDir, verifier = verifyCan function extractZipOnWindows( archivePath, destPath, requiredNames, extractNames, - limits = DEFAULT_ARCHIVE_LIMITS, shell = 'powershell', + maxMembers = MAX_ARCHIVE_MEMBERS, shell = 'powershell', ) { - validateArchiveLimits(limits); - requireCompressedArchiveWithinLimit(archivePath, limits); + requireRegularArchive(archivePath); // -EncodedCommand is a constant program. Paths travel only through the child // environment and are consumed with -LiteralPath, so PowerShell never parses // user/TEMP path bytes as source code or wildcard syntax. @@ -616,10 +607,7 @@ function extractZipOnWindows( "$ErrorActionPreference = 'Stop'", 'Add-Type -AssemblyName System.IO.Compression.FileSystem', '$maxMembers = [int]$env:CBM_NPM_MAX_MEMBERS', - '$maxMemberBytes = [long]$env:CBM_NPM_MAX_MEMBER_BYTES', - '$maxExpandedBytes = [long]$env:CBM_NPM_MAX_EXPANDED_BYTES', '$memberCount = 0', - '$declaredExpanded = [long]0', '$zip = [System.IO.Compression.ZipFile]::OpenRead($env:CBM_NPM_ARCHIVE_PATH)', "$seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)", "$requiredNames = @($env:CBM_NPM_REQUIRED_NAMES.Split('|'))", @@ -629,11 +617,6 @@ function extractZipOnWindows( '$memberCount = $memberCount + 1; ' + 'if ($memberCount -gt $maxMembers) { ' + 'throw "archive exceeds the $maxMembers-member safety limit" }; ' + - 'if ($entry.Length -gt $maxMemberBytes) { ' + - 'throw "archive member $($entry.FullName) exceeds the $maxMemberBytes-byte expanded safety limit" }; ' + - 'if ($declaredExpanded -gt ($maxExpandedBytes - $entry.Length)) { ' + - 'throw "archive exceeds the $maxExpandedBytes-byte aggregate expanded safety limit" }; ' + - '$declaredExpanded = $declaredExpanded + $entry.Length; ' + "$name = $entry.FullName.Replace('\\', '/'); " + "$directory = $name.EndsWith('/'); " + "$segmentsPath = if ($directory) { $name.TrimEnd('/') } else { $name }; " + @@ -655,15 +638,13 @@ function extractZipOnWindows( 'if ($seen.Count -ne $requiredNames.Count) { ' + 'throw "archive does not match the exact release root-file allowlist" }', "$extractNames = @($env:CBM_NPM_EXTRACT_NAMES.Split('|'))", - '$actualExpanded = [long]0', '$buffer = New-Object byte[] 65536', '$extractZip = [System.IO.Compression.ZipFile]::OpenRead($env:CBM_NPM_ARCHIVE_PATH)', - // Copy by hand with a byte counter: ExtractToFile would write whatever the - // compressed stream yields, regardless of the declared size. + // Copy by hand so the bytes written can be compared with the declared + // size; ExtractToFile would accept whatever the compressed stream yields. 'try { foreach ($extractName in $extractNames) { ' + '$entry = @($extractZip.Entries | Where-Object { $_.FullName -ceq $extractName })[0]; ' + '$targetPath = Join-Path $env:CBM_NPM_DEST_PATH $extractName; ' + - '$remaining = $maxExpandedBytes - $actualExpanded; ' + '$copied = [long]0; ' + '$source = $entry.Open(); ' + '$target = [System.IO.File]::Open($targetPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write); ' + @@ -671,18 +652,13 @@ function extractZipOnWindows( '$count = $source.Read($buffer, 0, $buffer.Length); ' + 'if ($count -le 0) { break }; ' + '$copied = $copied + $count; ' + - 'if ($copied -gt $maxMemberBytes) { ' + - 'throw "archive member $extractName exceeds the $maxMemberBytes-byte actual expanded safety limit" }; ' + - 'if ($copied -gt $remaining) { ' + - 'throw "archive exceeds the $maxExpandedBytes-byte aggregate actual expanded safety limit" }; ' + '$target.Write($buffer, 0, $count) } ' + '} catch { $target.Dispose(); ' + 'Remove-Item -LiteralPath $targetPath -Force -ErrorAction SilentlyContinue; throw ' + '} finally { $target.Dispose(); $source.Dispose() }; ' + 'if ($copied -ne $entry.Length) { ' + 'Remove-Item -LiteralPath $targetPath -Force -ErrorAction SilentlyContinue; ' + - 'throw "archive member $extractName actual size $copied does not match declared size $($entry.Length)" }; ' + - '$actualExpanded = $actualExpanded + $copied ' + + 'throw "archive member $extractName actual size $copied does not match declared size $($entry.Length)" } ' + '} } finally { $extractZip.Dispose() }', ].join('; '); const encoded = Buffer.from(script, 'utf16le').toString('base64'); @@ -696,39 +672,30 @@ function extractZipOnWindows( CBM_NPM_DEST_PATH: destPath, CBM_NPM_REQUIRED_NAMES: requiredNames.join('|'), CBM_NPM_EXTRACT_NAMES: extractNames.join('|'), - CBM_NPM_MAX_MEMBERS: String(limits.members), - CBM_NPM_MAX_MEMBER_BYTES: String(limits.memberBytes), - CBM_NPM_MAX_EXPANDED_BYTES: String(limits.expandedBytes), + CBM_NPM_MAX_MEMBERS: String(maxMembers), }, encoding: 'utf8', stdio: ['ignore', 'inherit', 'pipe'], windowsHide: true, }); } catch (err) { - // Surface the script's own reason (for example which limit was hit) + // Surface the script's own reason (for example which check failed) // instead of a bare "Command failed". const detail = err && err.stderr ? String(err.stderr).trim() : ''; throw new Error(detail ? `zip extraction failed: ${detail}` : err.message); } } -// Fetch checksums.txt and verify the archive hash. -async function verifyChecksum(archivePath, archiveName) { - const url = `https://github.com/${REPO}/releases/download/v${VERSION}/checksums.txt`; +// Fetch checksums.txt from the same download base and verify the archive hash. +async function verifyChecksum(archivePath, archiveName, source) { + const url = `${source.base}/checksums.txt`; const tmpChecksums = archivePath + '.checksums'; try { - await download(url, tmpChecksums, MAX_CHECKSUM_MANIFEST_BYTES); - const manifestSize = fs.statSync(tmpChecksums).size; - if (manifestSize > MAX_CHECKSUM_MANIFEST_BYTES) { - throw new Error('checksums.txt exceeds the 1 MiB safety limit'); - } + await download(url, tmpChecksums, source.loopback); const expected = parseExpectedChecksum( fs.readFileSync(tmpChecksums, 'utf-8'), archiveName, ); - const actual = crypto - .createHash('sha256') - .update(fs.readFileSync(archivePath)) - .digest('hex'); + const actual = fileSha256(archivePath); if (expected !== actual) { throw new Error( `Checksum mismatch for ${archiveName}:\n expected: ${expected}\n actual: ${actual}`, @@ -772,6 +739,7 @@ async function main() { } } + const source = resolveDownloadBase(); fs.mkdirSync(BIN_DIR, { recursive: true }); // Linux ships a fully-static "-portable" build; the standard linux binary @@ -782,7 +750,7 @@ async function main() { // always embedded. The old CBM_VARIANT=ui opt-in pointed at ui-* archives // that no longer exist, so honoring it could only 404 (#1538). const archive = `codebase-memory-mcp-${platform}-${arch}${variant}.${ext}`; - const url = `https://github.com/${REPO}/releases/download/v${VERSION}/${archive}`; + const url = `${source.base}/${archive}`; process.stdout.write(`codebase-memory-mcp: downloading v${VERSION} for ${platform}/${arch}...\n`); @@ -790,8 +758,8 @@ async function main() { const tmpArchive = path.join(tmpDir, `cbm.${ext}`); try { - await download(url, tmpArchive, MAX_RELEASE_ARCHIVE_BYTES); - await verifyChecksum(tmpArchive, archive); + await download(url, tmpArchive, source.loopback); + await verifyChecksum(tmpArchive, archive, source); // Validate the complete archive namespace, then extract only executables. if (ext === 'tar.gz') { @@ -872,7 +840,6 @@ if (require.main === module || module.parent == null) { } module.exports = { - DEFAULT_ARCHIVE_LIMITS, UNIX_ARCHIVE_NAMES, WINDOWS_ARCHIVE_NAMES, WINDOWS_BINARY_NAME, @@ -880,6 +847,7 @@ module.exports = { extractExactTarArchive, extractZipOnWindows, installWindowsBinaryAtomically, + resolveDownloadBase, validateExactTarMemberListing, windowsBinaryReady, }; diff --git a/pkg/npm/test/archive-limits.test.js b/pkg/npm/test/archive-limits.test.js index 998f772e63..581c6e807a 100644 --- a/pkg/npm/test/archive-limits.test.js +++ b/pkg/npm/test/archive-limits.test.js @@ -3,6 +3,7 @@ const assert = require('node:assert/strict'); const { EventEmitter } = require('node:events'); const fs = require('node:fs'); +const http = require('node:http'); const os = require('node:os'); const path = require('node:path'); const { Readable } = require('node:stream'); @@ -10,20 +11,23 @@ const test = require('node:test'); const vm = require('node:vm'); const { - DEFAULT_ARCHIVE_LIMITS, UNIX_ARCHIVE_NAMES, WINDOWS_ARCHIVE_NAMES, WINDOWS_BINARY_NAME, + download, extractExactTarArchive, extractZipOnWindows, + resolveDownloadBase, } = require('../install.js'); const { TAR_PATH_SKIP, buildZip, patternBytes, writeTarGz, } = require('./archive-fixtures.js'); const EXECUTABLE = 'codebase-memory-mcp'; -// Spans several pipe chunks, so an over-limit member is cut off mid-stream. +// Spans several pipe chunks, so the counted writer sees a multi-chunk stream. const LARGE_EXECUTABLE = patternBytes(300 * 1024); +// The size cap the 0.12.0 wrappers enforced, which refused every release. +const FORMER_MEMBER_CAP_BYTES = 256 * 1024 * 1024; function withScratch(callback) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cbm-npm-archive-test-')); @@ -32,10 +36,6 @@ function withScratch(callback) { .finally(() => fs.rmSync(root, { recursive: true, force: true })); } -function limitsWith(overrides) { - return { ...DEFAULT_ARCHIVE_LIMITS, ...overrides }; -} - function releaseEntries(names, executableData) { return names.map((name) => ({ name, @@ -61,59 +61,6 @@ function fakeTarListing(names) { }; } -function neverLists() { - throw new Error('the listing must not run'); -} - -test('tar extraction refuses an archive larger than the compressed limit', { skip: TAR_PATH_SKIP }, () => - withScratch(async (root) => { - const archive = writeTarGz(root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES)); - const destination = makeDestination(root); - - await assert.rejects( - extractExactTarArchive( - archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE, - neverLists, limitsWith({ compressedBytes: 16 }), - ), - /16-byte compressed safety limit/, - ); - assert.deepEqual(fs.readdirSync(destination), []); - })); - -test('tar extraction refuses an exact namespace wider than the member limit', { skip: TAR_PATH_SKIP }, () => - withScratch(async (root) => { - const archive = writeTarGz(root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES)); - const destination = makeDestination(root); - - await assert.rejects( - extractExactTarArchive( - archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE, - neverLists, limitsWith({ members: 2 }), - ), - /2-member safety limit/, - ); - assert.deepEqual(fs.readdirSync(destination), []); - })); - -test('tar extraction stops an over-limit member in the counted writer and leaves no file', { skip: TAR_PATH_SKIP }, () => - withScratch(async (root) => { - const archive = writeTarGz( - root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES, LARGE_EXECUTABLE), - ); - const destination = makeDestination(root); - - // The real listing passes (names only); the limit trips while tar is - // still streaming the executable into the writer. - await assert.rejects( - extractExactTarArchive( - archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE, - undefined, limitsWith({ memberBytes: 64 * 1024 }), - ), - /"codebase-memory-mcp" exceeds the 65536-byte expanded safety limit/, - ); - assert.deepEqual(fs.readdirSync(destination), []); - })); - test('tar extraction leaves a pre-existing target untouched', { skip: TAR_PATH_SKIP }, () => withScratch(async (root) => { const archive = writeTarGz(root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES)); @@ -181,7 +128,7 @@ test('tar extraction installs the executable byte-identical with the executable assert.deepEqual(fs.readdirSync(destination), [EXECUTABLE]); })); -function loadInstallerWithHttps(fakeHttps) { +function loadInstallerWithTransports({ fakeHttps, fakeHttp }) { const installPath = path.join(__dirname, '..', 'install.js'); const source = fs.readFileSync(installPath, 'utf8'); const sandbox = { @@ -197,7 +144,8 @@ function loadInstallerWithHttps(fakeHttps) { process, require: (specifier) => { if (specifier === './package.json') return { version: '0.0.0-test' }; - if (specifier === 'https') return fakeHttps; + if (specifier === 'https' && fakeHttps) return fakeHttps; + if (specifier === 'http' && fakeHttp) return fakeHttp; return require(specifier); }, setTimeout, @@ -206,10 +154,13 @@ function loadInstallerWithHttps(fakeHttps) { return sandbox.module.exports; } -// Answers every request with one HTTP 200 response shaped by `configure`. -function fakeHttpsServing(configure) { +// Answers every request with one HTTP 200 response shaped by `configure`; +// `requests` records each (url, options) the installer issued. +function fakeTransportServing(configure, requests = []) { return { - get(url, callback) { + get(url, ...rest) { + const callback = rest[rest.length - 1]; + requests.push({ url, options: rest.length > 1 ? rest[0] : undefined }); const request = new EventEmitter(); request.destroy = (err) => { if (err) request.emit('error', err); }; setImmediate(() => { @@ -224,50 +175,146 @@ function fakeHttpsServing(configure) { }; } -test('download refuses a response whose declared length exceeds the limit', () => +function refusingTransport() { + return { + get(url) { + throw new Error(`unexpected request to ${url}`); + }, + }; +} + +test('download stores a body whose declared length exceeds the former 256 MiB cap', () => withScratch(async (root) => { - const { download } = loadInstallerWithHttps(fakeHttpsServing((response) => { - response.headers['content-length'] = '9'; - response.push(null); - })); + const { download: fakeDownload } = loadInstallerWithTransports({ + fakeHttps: fakeTransportServing((response) => { + response.headers['content-length'] = String(FORMER_MEMBER_CAP_BYTES + 1); + response.push('payload'); + response.push(null); + }), + }); const destination = path.join(root, 'release.tar.gz'); - await assert.rejects( - download('https://example.invalid/release.tar.gz', destination, 8), - /8-byte safety limit/, - ); - assert.equal(fs.existsSync(destination), false); + await fakeDownload('https://example.invalid/release.tar.gz', destination); + + assert.equal(fs.readFileSync(destination, 'utf8'), 'payload'); })); -test('download refuses a body that exceeds the limit and removes the partial file', () => +test('download refuses plain HTTP unless the base is a loopback override', () => withScratch(async (root) => { - const { download } = loadInstallerWithHttps(fakeHttpsServing((response) => { - response.push(Buffer.alloc(1024, 0x78)); - response.push(null); - })); + const { download: fakeDownload } = loadInstallerWithTransports({ + fakeHttps: refusingTransport(), + fakeHttp: refusingTransport(), + }); const destination = path.join(root, 'release.tar.gz'); await assert.rejects( - download('https://example.invalid/release.tar.gz', destination, 8), - /8-byte safety limit/, + fakeDownload('http://127.0.0.1/release.tar.gz', destination), + /Refusing non-HTTPS or credentialed URL/, + ); + await assert.rejects( + fakeDownload('http://example.invalid/release.tar.gz', destination, true), + /Refusing non-loopback HTTP URL/, ); assert.equal(fs.existsSync(destination), false); })); -test('download within the limit stores the body', () => +test('a loopback download uses a fresh agent, so no proxy applies', () => withScratch(async (root) => { - const { download } = loadInstallerWithHttps(fakeHttpsServing((response) => { - response.headers['content-length'] = '7'; - response.push('payload'); - response.push(null); - })); + const requests = []; + const { download: fakeDownload } = loadInstallerWithTransports({ + fakeHttps: refusingTransport(), + fakeHttp: fakeTransportServing((response) => { + response.push('payload'); + response.push(null); + }, requests), + }); const destination = path.join(root, 'release.tar.gz'); - await download('https://example.invalid/release.tar.gz', destination, 1024); + await fakeDownload('http://[::1]:8080/release.tar.gz', destination, true); assert.equal(fs.readFileSync(destination, 'utf8'), 'payload'); + assert.equal(requests.length, 1); + assert.equal(requests[0].url, 'http://[::1]:8080/release.tar.gz'); + assert.equal(requests[0].options.agent, false); })); +function withLoopbackServer(handler, callback) { + const server = http.createServer(handler); + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => resolve()); + }).then(() => Promise.resolve() + .then(() => callback(`http://127.0.0.1:${server.address().port}`)) + .finally(() => new Promise((resolve) => server.close(resolve)))); +} + +test('a loopback download fetches from a local server and refuses its redirects', () => + withScratch((root) => withLoopbackServer((request, response) => { + if (request.url === '/release/codebase-memory-mcp.tar.gz') { + response.writeHead(200, { 'content-type': 'application/octet-stream' }); + response.end('loopback payload'); + return; + } + response.writeHead(302, { location: 'https://example.invalid/elsewhere' }); + response.end(); + }, async (origin) => { + const archive = path.join(root, 'release.tar.gz'); + await download(`${origin}/release/codebase-memory-mcp.tar.gz`, archive, true); + assert.equal(fs.readFileSync(archive, 'utf8'), 'loopback payload'); + + const redirected = path.join(root, 'redirected.tar.gz'); + await assert.rejects( + download(`${origin}/release/checksums.txt`, redirected, true), + /Refusing redirect from loopback download/, + ); + assert.equal(fs.existsSync(redirected), false); + }))); + +test('CBM_DOWNLOAD_URL follows install.sh: HTTPS or exact-loopback HTTP only', () => { + const { version } = require('../package.json'); + const githubBase = `https://github.com/DeusData/codebase-memory-mcp/releases/download/v${version}`; + assert.deepEqual(resolveDownloadBase({}), { base: githubBase, loopback: false }); + assert.deepEqual( + resolveDownloadBase({ CBM_DOWNLOAD_URL: '' }), { base: githubBase, loopback: false }, + ); + + for (const base of [ + 'https://mirror.example/releases/v1', + 'https://127.0.0.1:8443/fixture', + ]) { + assert.deepEqual( + resolveDownloadBase({ CBM_DOWNLOAD_URL: base }), { base, loopback: false }, base, + ); + } + for (const base of [ + 'http://localhost', + 'http://localhost:8080/fixture', + 'http://127.0.0.1:41234/standard', + 'http://[::1]:9/x?y#z', + ]) { + assert.deepEqual( + resolveDownloadBase({ CBM_DOWNLOAD_URL: base }), { base, loopback: true }, base, + ); + } + for (const base of [ + 'http://example.com/releases', + 'http://127.0.0.1.example.com/', + 'http://localhost.example.com:8080/', + 'http://user@localhost/', + 'http://127.0.0.2/', + 'HTTP://localhost/', + 'file:///tmp/releases', + 'ftp://localhost/releases', + 'github.com/DeusData/codebase-memory-mcp', + ]) { + assert.throws( + () => resolveDownloadBase({ CBM_DOWNLOAD_URL: base }), + /Refusing non-HTTPS download URL \(CBM_DOWNLOAD_URL\)/, + base, + ); + } +}); + // The Windows path runs a constant PowerShell program through powershell.exe, // which only a Windows host provides (pwsh is not a wrapper dependency, and // no PowerShell is installed on the Unix wrapper hosts), so these cases run @@ -288,33 +335,14 @@ test('Windows zip extraction refuses an archive with more members than the limit assert.throws( () => extractZipOnWindows( - archive, destination, ['one', 'two', 'three'], ['one'], - limitsWith({ members: 2 }), + archive, destination, ['one', 'two', 'three'], ['one'], 2, ), /2-member safety limit/, ); assert.deepEqual(fs.readdirSync(destination), []); })); -test('Windows zip extraction refuses a member whose declared size exceeds the limit', { skip: WINDOWS_ONLY }, () => - withScratch(async (root) => { - const archive = path.join(root, 'member.zip'); - fs.writeFileSync(archive, buildZip([ - { name: WINDOWS_BINARY_NAME, data: Buffer.alloc(65, 0x78) }, - ])); - const destination = makeDestination(root); - - assert.throws( - () => extractZipOnWindows( - archive, destination, [WINDOWS_BINARY_NAME], [WINDOWS_BINARY_NAME], - limitsWith({ memberBytes: 64 }), - ), - /64-byte expanded safety limit/, - ); - assert.deepEqual(fs.readdirSync(destination), []); - })); - -test('Windows zip extraction stops a member that yields more bytes than it declares', { skip: WINDOWS_ONLY }, () => +test('Windows zip extraction rejects a member that yields more bytes than it declares', { skip: WINDOWS_ONLY }, () => withScratch(async (root) => { const archive = path.join(root, 'longer.zip'); fs.writeFileSync(archive, buildZip([ @@ -325,14 +353,13 @@ test('Windows zip extraction stops a member that yields more bytes than it decla assert.throws( () => extractZipOnWindows( archive, destination, [WINDOWS_BINARY_NAME], [WINDOWS_BINARY_NAME], - limitsWith({ memberBytes: 64 }), ), - /64-byte actual expanded safety limit/, + /actual size \d+ does not match declared size 10/, ); assert.deepEqual(fs.readdirSync(destination), []); })); -test('Windows zip extraction within the limits installs only the executable', { skip: WINDOWS_ONLY }, () => +test('Windows zip extraction installs only the executable', { skip: WINDOWS_ONLY }, () => withScratch(async (root) => { const archive = path.join(root, 'release.zip'); fs.writeFileSync(archive, buildZip(releaseEntries(WINDOWS_ARCHIVE_NAMES))); diff --git a/pkg/npm/test/install-failure.test.js b/pkg/npm/test/install-failure.test.js index 3d539ec0f3..211caa8255 100644 --- a/pkg/npm/test/install-failure.test.js +++ b/pkg/npm/test/install-failure.test.js @@ -2,6 +2,7 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); +const os = require('node:os'); const path = require('node:path'); const test = require('node:test'); const vm = require('node:vm'); @@ -51,3 +52,58 @@ test('postinstall exits nonzero when installation fails', async () => { 'npm postinstall reported success after a deterministic installation failure', ); }); + +test('postinstall refuses a non-loopback http CBM_DOWNLOAD_URL before any download', async () => { + const installPath = path.join(__dirname, '..', 'install.js'); + const source = fs.readFileSync(installPath, 'utf8'); + // A fresh package directory: no cached binary, and nothing may be created. + const packageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cbm-npm-override-test-')); + const output = { stdout: '', stderr: '' }; + const observed = { exitCode: undefined }; + const fakeProcess = { + platform: 'linux', + arch: 'x64', + env: { CBM_DOWNLOAD_URL: 'http://example.com/releases' }, + pid: 12345, + stdout: { write: (text) => { output.stdout += String(text); } }, + stderr: { write: (text) => { output.stderr += String(text); } }, + exit: (code) => { observed.exitCode = code; }, + exitCode: undefined, + }; + const refusingTransport = { + get(url) { throw new Error(`unexpected request to ${url}`); }, + }; + + try { + const sandbox = { + Buffer, + URL, + __dirname: packageDir, + __filename: path.join(packageDir, 'install.js'), + clearTimeout, + console, + exports: {}, + module: { exports: {} }, + process: fakeProcess, + require: (specifier) => { + if (specifier === './package.json') return { version: '0.0.0-test' }; + if (specifier === 'https' || specifier === 'http') return refusingTransport; + return require(specifier); + }, + setTimeout, + }; + + vm.runInNewContext(source, sandbox, { filename: installPath }); + await new Promise((resolve) => setImmediate(resolve)); + + assert.match( + output.stderr, + /install failed .*Refusing non-HTTPS download URL \(CBM_DOWNLOAD_URL\): http:\/\/example\.com\/releases/, + ); + assert.equal(observed.exitCode, 1); + assert.doesNotMatch(output.stdout, /downloading/); + assert.deepEqual(fs.readdirSync(packageDir), []); + } finally { + fs.rmSync(packageDir, { recursive: true, force: true }); + } +}); diff --git a/pkg/pypi/README.md b/pkg/pypi/README.md index a2047d8762..577a426611 100644 --- a/pkg/pypi/README.md +++ b/pkg/pypi/README.md @@ -16,6 +16,8 @@ pipx install codebase-memory-mcp There is one composition per platform: the graph UI ships in every build, so no variant selection is needed. +For testing, `CBM_DOWNLOAD_URL` overrides the download base URL (default `https://github.com/DeusData/codebase-memory-mcp/releases/download/v`): the archive and `checksums.txt` are fetched from `/`. It must be `https://`, or plain `http://` to exactly `localhost`, `127.0.0.1`, or `[::1]` (optional port; no proxy, no redirects). + ## Usage ```bash diff --git a/pkg/pypi/src/codebase_memory_mcp/_cli.py b/pkg/pypi/src/codebase_memory_mcp/_cli.py index cae54b3431..8321bc173e 100644 --- a/pkg/pypi/src/codebase_memory_mcp/_cli.py +++ b/pkg/pypi/src/codebase_memory_mcp/_cli.py @@ -1,6 +1,5 @@ """Downloads codebase-memory-mcp on first run, then runs its native entry point.""" -import collections import errno import hashlib import json @@ -39,26 +38,19 @@ # file://, ftp://, and custom schemes — a redirect or tainted URL source # could otherwise turn a download into an arbitrary-local-file read. _ALLOWED_SCHEMES = frozenset({"https"}) +# install.sh's loopback pattern: plain HTTP only to an exact loopback +# authority, for local smoke tests (CBM_DOWNLOAD_URL). +_LOOPBACK_HTTP_URL = re.compile( + r"http://(localhost|127\.0\.0\.1|\[::1\])(:[0-9]+)?([/?#].*)?" +) +_LOOPBACK_HOSTNAMES = frozenset({"localhost", "127.0.0.1", "::1"}) _MAX_REDIRECTS = 5 _NETWORK_TIMEOUT_SECONDS = 120 _CANDIDATE_TIMEOUT_SECONDS = 15 -_MAX_CHECKSUM_MANIFEST_BYTES = 1024 * 1024 -# Release archive resource limits. Keep in sync with the Go wrapper -# (pkg/go/cmd/codebase-memory-mcp/main.go) and the npm wrapper (install.js). -_MAX_RELEASE_ARCHIVE_BYTES = 256 * 1024 * 1024 +# Entry-count guard for the archive namespace check. Keep in sync with the +# Go wrapper (pkg/go/cmd/codebase-memory-mcp/main.go) and the npm wrapper. _MAX_ARCHIVE_MEMBERS = 64 -_MAX_ARCHIVE_MEMBER_BYTES = 256 * 1024 * 1024 -_MAX_ARCHIVE_EXPANDED_BYTES = 512 * 1024 * 1024 _ARCHIVE_COPY_CHUNK_BYTES = 64 * 1024 -_ArchiveLimits = collections.namedtuple( - "_ArchiveLimits", "compressed_bytes members member_bytes expanded_bytes" -) -_DEFAULT_ARCHIVE_LIMITS = _ArchiveLimits( - compressed_bytes=_MAX_RELEASE_ARCHIVE_BYTES, - members=_MAX_ARCHIVE_MEMBERS, - member_bytes=_MAX_ARCHIVE_MEMBER_BYTES, - expanded_bytes=_MAX_ARCHIVE_EXPANDED_BYTES, -) _REDIRECT_CODES = frozenset({301, 302, 303, 307, 308}) _RUNTIME_LOCK_NAME = ".codebase-memory-mcp-runtime.lock" _RUNTIME_LOCK_WAIT_SECONDS = 45 @@ -79,13 +71,50 @@ class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Expose redirects to _download_https for explicit per-hop validation.""" + """Expose redirects to _download_file for explicit per-hop validation.""" def redirect_request(self, req, fp, code, msg, headers, newurl): return None _HTTPS_OPENER = urllib.request.build_opener(_NoRedirectHandler()) +# A loopback fetch never goes through a proxy. +_LOOPBACK_OPENER = urllib.request.build_opener( + urllib.request.ProxyHandler({}), _NoRedirectHandler() +) + + +def _download_base(version: str, environ=None): + """Return (base, loopback) for release downloads. + + CBM_DOWNLOAD_URL replaces the release download base (for testing), with + install.sh's policy: HTTPS, or plain HTTP to an exact loopback authority. + """ + env = os.environ if environ is None else environ + override = env.get("CBM_DOWNLOAD_URL") + if not override: + return f"https://github.com/{REPO}/releases/download/v{version}", False + if override.startswith("https://"): + return override, False + if _LOOPBACK_HTTP_URL.fullmatch(override): + return override, True + sys.exit( + f"codebase-memory-mcp: refusing non-HTTPS download URL " + f"(CBM_DOWNLOAD_URL): {override}" + ) + + +def _validate_loopback_url(url: str) -> None: + """Reject anything but plain HTTP to an exact loopback authority.""" + parsed = urllib.parse.urlparse(url) + if ( + not _LOOPBACK_HTTP_URL.fullmatch(url) + or parsed.scheme != "http" + or parsed.hostname not in _LOOPBACK_HOSTNAMES + or parsed.username is not None + or parsed.password is not None + ): + sys.exit(f"codebase-memory-mcp: refusing non-loopback HTTP URL: {url}") def _validate_url_scheme(url: str) -> None: @@ -103,23 +132,33 @@ def _validate_url_scheme(url: str) -> None: ) -def _download_https(url: str, dest: str, max_bytes: int = 0) -> None: - """Download through a bounded HTTPS-only opener into dest.""" +def _validate_download_url(url: str, loopback: bool) -> None: + if loopback: + _validate_loopback_url(url) + else: + _validate_url_scheme(url) + + +def _download_file(url: str, dest: str, loopback: bool = False) -> None: + """Download into dest: HTTPS with checked redirects, or one loopback hop.""" + opener = _LOOPBACK_OPENER if loopback else _HTTPS_OPENER current_url = url for redirect_count in range(_MAX_REDIRECTS + 1): - _validate_url_scheme(current_url) + _validate_download_url(current_url, loopback) request = urllib.request.Request( current_url, headers={"User-Agent": "codebase-memory-mcp-installer"} ) try: - response = _HTTPS_OPENER.open( - request, timeout=_NETWORK_TIMEOUT_SECONDS - ) + response = opener.open(request, timeout=_NETWORK_TIMEOUT_SECONDS) except urllib.error.HTTPError as exc: if exc.code not in _REDIRECT_CODES: raise location = exc.headers.get("Location") exc.close() + if loopback: + raise RuntimeError( + f"refusing redirect from loopback download: {current_url}" + ) if not location: raise RuntimeError(f"redirect has no Location: {current_url}") if redirect_count == _MAX_REDIRECTS: @@ -129,18 +168,8 @@ def _download_https(url: str, dest: str, max_bytes: int = 0) -> None: continue with response: - _validate_url_scheme(response.geturl()) - declared = (response.headers.get("Content-Length") or "").strip() - # Only plain ASCII digits are a length (str.isdigit also accepts - # characters that int() rejects); anything else is left to the - # byte counter below. - if max_bytes and re.fullmatch(r"[0-9]+", declared): - if int(declared) > max_bytes: - raise RuntimeError( - f"download exceeds the {max_bytes}-byte safety limit" - ) + _validate_download_url(response.geturl(), loopback) deadline = time.monotonic() + _NETWORK_TIMEOUT_SECONDS - total = 0 try: with open(dest, "wb") as out: while True: @@ -151,12 +180,6 @@ def _download_https(url: str, dest: str, max_bytes: int = 0) -> None: chunk = response.read(65536) if not chunk: return - total += len(chunk) - if max_bytes and total > max_bytes: - raise RuntimeError( - f"download exceeds the {max_bytes}-byte " - f"safety limit" - ) out.write(chunk) except BaseException: # Never leave a partial download behind for a later step to @@ -236,106 +259,51 @@ def _validate_archive_names(names, archive_names, casefold: bool = False): ) -def _validate_archive_limits(limits) -> None: - if ( - limits.compressed_bytes <= 0 - or limits.members <= 0 - or limits.member_bytes <= 0 - or limits.expanded_bytes <= 0 - ): - sys.exit("codebase-memory-mcp: invalid archive resource safety limits") - - -def _require_compressed_archive_within_limit(archive_path: str, limits) -> None: - status = os.stat(archive_path) - if not stat.S_ISREG(status.st_mode): +def _require_regular_archive(archive_path: str) -> None: + if not stat.S_ISREG(os.stat(archive_path).st_mode): sys.exit("codebase-memory-mcp: release archive is not a regular file") - if status.st_size > limits.compressed_bytes: - sys.exit( - f"codebase-memory-mcp: release archive exceeds the " - f"{limits.compressed_bytes}-byte compressed safety limit" - ) -def _require_member_count_within_limit(count: int, limits) -> None: - if count > limits.members: +def _require_member_count_within_limit(count: int, max_members: int) -> None: + if count > max_members: sys.exit( - f"codebase-memory-mcp: archive exceeds the {limits.members}-member " + f"codebase-memory-mcp: archive exceeds the {max_members}-member " f"safety limit" ) -def _account_declared_archive_member( - name: str, size: int, declared_expanded: int, limits -) -> int: - """Check one member's declared size; return the new declared total.""" +def _require_declared_size(name: str, size: int) -> None: if size < 0: sys.exit( f"codebase-memory-mcp: archive member has a negative declared " f"size: {name!r}" ) - if size > limits.member_bytes: - sys.exit( - f"codebase-memory-mcp: archive member {name!r} exceeds the " - f"{limits.member_bytes}-byte expanded safety limit" - ) - if declared_expanded > limits.expanded_bytes - size: - sys.exit( - f"codebase-memory-mcp: archive exceeds the {limits.expanded_bytes}-byte " - f"aggregate expanded safety limit" - ) - return declared_expanded + size -def _copy_archive_member_within_limits( - source, output, name: str, declared_size: int, actual_expanded: int, limits -) -> int: - """Copy a member while counting bytes; return the new actual total. +def _copy_archive_member(source, output, name: str, declared_size: int) -> int: + """Copy a member, then require the declared size; return the bytes copied. - Reads stop one byte past the smaller of the member and remaining aggregate - limits, so a member that yields more than it declares can never fill the - disk. The caller removes the target on failure. + The caller removes the target on failure. """ - remaining = limits.expanded_bytes - actual_expanded - copy_limit = min(limits.member_bytes, remaining) copied = 0 while True: - wanted = min(_ARCHIVE_COPY_CHUNK_BYTES, copy_limit + 1 - copied) - if wanted <= 0: - break - chunk = source.read(wanted) + chunk = source.read(_ARCHIVE_COPY_CHUNK_BYTES) if not chunk: break copied += len(chunk) - if copied > copy_limit: - break output.write(chunk) - if copied > limits.member_bytes: - sys.exit( - f"codebase-memory-mcp: archive member {name!r} exceeds the " - f"{limits.member_bytes}-byte actual expanded safety limit" - ) - if copied > remaining: - sys.exit( - f"codebase-memory-mcp: archive exceeds the {limits.expanded_bytes}-byte " - f"aggregate actual expanded safety limit" - ) if copied != declared_size: sys.exit( f"codebase-memory-mcp: archive member {name!r} actual size {copied} " f"does not match declared size {declared_size}" ) - return actual_expanded + copied + return copied -def _write_archive_member_within_limits( - source, target: str, name: str, declared_size: int, actual_expanded: int, limits -) -> int: +def _write_archive_member(source, target: str, name: str, declared_size: int) -> int: try: with open(target, "xb") as output: - return _copy_archive_member_within_limits( - source, output, name, declared_size, actual_expanded, limits - ) + return _copy_archive_member(source, output, name, declared_size) except BaseException: _remove_quietly(target) raise @@ -347,36 +315,29 @@ def _safe_extract_tar( archive_names=(), extract_names=(), ui: bool = False, - limits=_DEFAULT_ARCHIVE_LIMITS, + max_members: int = _MAX_ARCHIVE_MEMBERS, ): """Validate a tar namespace, then extract only its authenticated runtime set.""" - _validate_archive_limits(limits) members = tf.getmembers() - _require_member_count_within_limit(len(members), limits) - declared_expanded = 0 + _require_member_count_within_limit(len(members), max_members) for member in members: if not member.isfile(): sys.exit( f"codebase-memory-mcp: refusing unsafe tar entry " f"(not a regular root file: {member.name!r})" ) - declared_expanded = _account_declared_archive_member( - member.name, member.size, declared_expanded, limits - ) + _require_declared_size(member.name, member.size) _validate_archive_names([member.name for member in members], archive_names) runtime_names = list(extract_names) by_name = {member.name: member for member in members} dest_abs = os.path.abspath(dest) - actual_expanded = 0 for name in runtime_names: source = tf.extractfile(by_name[name]) if source is None: sys.exit(f"codebase-memory-mcp: could not read archive member: {name}") target = os.path.join(dest_abs, name) with source: - actual_expanded = _write_archive_member_within_limits( - source, target, name, by_name[name].size, actual_expanded, limits - ) + _write_archive_member(source, target, name, by_name[name].size) return tuple(runtime_names) @@ -386,7 +347,7 @@ def _safe_extract_zip( archive_names=(), extract_names=(), ui: bool = False, - limits=_DEFAULT_ARCHIVE_LIMITS, + max_members: int = _MAX_ARCHIVE_MEMBERS, ): """Extract a zipfile after validating its Windows-style namespace. @@ -394,16 +355,12 @@ def _safe_extract_zip( members before extraction prevents archive order from selecting the binary that a portable package-manager shim eventually executes. """ - _validate_archive_limits(limits) dest_abs = os.path.abspath(dest) names = [] infos = zf.infolist() - _require_member_count_within_limit(len(infos), limits) - declared_expanded = 0 + _require_member_count_within_limit(len(infos), max_members) for info in infos: - declared_expanded = _account_declared_archive_member( - info.filename, info.file_size, declared_expanded, limits - ) + _require_declared_size(info.filename, info.file_size) raw_name = info.filename name = raw_name.replace("\\", "/") segments_name = name[:-1] if name.endswith("/") else name @@ -437,30 +394,24 @@ def _safe_extract_zip( # Extract only the validated runtime files. This avoids relying on # platform-specific zip path rewriting and always creates regular files. runtime_names = list(extract_names) - actual_expanded = 0 for name in runtime_names: target = os.path.join(dest_abs, name) with zf.open(name) as source: - actual_expanded = _write_archive_member_within_limits( - source, - target, - name, - zf.getinfo(name).file_size, - actual_expanded, - limits, - ) + _write_archive_member(source, target, name, zf.getinfo(name).file_size) return tuple(runtime_names) -def _verify_checksum(archive_path: str, archive_name: str, version: str) -> None: - """Verify SHA256 checksum against checksums.txt from the release.""" - url = f"https://github.com/{REPO}/releases/download/v{version}/checksums.txt" +def _verify_checksum( + archive_path: str, archive_name: str, base: str, loopback: bool = False +) -> None: + """Verify SHA256 checksum against checksums.txt from the same base.""" + url = f"{base}/checksums.txt" tmp_path = None try: - _validate_url_scheme(url) + _validate_download_url(url, loopback) with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp: tmp_path = tmp.name - _download_https(url, tmp_path, _MAX_CHECKSUM_MANIFEST_BYTES) + _download_file(url, tmp_path, loopback) expected = None with open(tmp_path, encoding="utf-8") as f: for line in f: @@ -1344,8 +1295,9 @@ def _download(version: str) -> Path: # have no such variant. Keep in sync with install.sh / install.js / cli.c. portable = "-portable" if os_name == "linux" else "" archive = f"codebase-memory-mcp-{os_name}-{arch}{portable}.{ext}" - url = f"https://github.com/{REPO}/releases/download/v{version}/{archive}" - _validate_url_scheme(url) + base, loopback = _download_base(version) + url = f"{base}/{archive}" + _validate_download_url(url, loopback) dest = _bin_path(version) dest.parent.mkdir(parents=True, exist_ok=True) @@ -1358,7 +1310,7 @@ def _download(version: str) -> Path: with tempfile.TemporaryDirectory() as tmp: tmp_archive = os.path.join(tmp, f"cbm.{ext}") try: - _download_https(url, tmp_archive, _MAX_RELEASE_ARCHIVE_BYTES) + _download_file(url, tmp_archive, loopback) except (OSError, RuntimeError, urllib.error.URLError) as e: sys.exit( f"codebase-memory-mcp: download failed ({e})\n" @@ -1366,10 +1318,8 @@ def _download(version: str) -> Path: f"See https://github.com/{REPO}/releases for available versions." ) - _verify_checksum(tmp_archive, archive, version) - _require_compressed_archive_within_limit( - tmp_archive, _DEFAULT_ARCHIVE_LIMITS - ) + _verify_checksum(tmp_archive, archive, base, loopback) + _require_regular_archive(tmp_archive) bin_name = ( _WINDOWS_BINARY_NAME if os_name == "windows" else "codebase-memory-mcp" diff --git a/pkg/pypi/tests/test_cli.py b/pkg/pypi/tests/test_cli.py index b6d62f7d2a..ed8f33f52b 100644 --- a/pkg/pypi/tests/test_cli.py +++ b/pkg/pypi/tests/test_cli.py @@ -1,5 +1,7 @@ +import contextlib import errno import hashlib +import http.server import io import json import os @@ -840,9 +842,10 @@ def test_windows_zip_rejects_symlink_metadata(self): class _FakeHttpsResponse: """Minimal stand-in for the opener's response: headers, geturl, read.""" - def __init__(self, body: bytes, headers=None): + def __init__(self, body: bytes, headers=None, fail_after_first_chunk=False): self._body = body self._offset = 0 + self._fail_after_first_chunk = fail_after_first_chunk self.headers = dict(headers or {}) def __enter__(self): @@ -855,6 +858,8 @@ def geturl(self): return "https://example.invalid/release.tar.gz" def read(self, size=-1): + if self._fail_after_first_chunk and self._offset: + raise OSError("connection reset") if size is None or size < 0: size = len(self._body) - self._offset chunk = self._body[self._offset : self._offset + size] @@ -862,8 +867,31 @@ def read(self, size=-1): return chunk -def _limits_with(**overrides): - return _cli._DEFAULT_ARCHIVE_LIMITS._replace(**overrides) +class _ZeroSource: + """Reader that yields `size` zero bytes without holding them.""" + + def __init__(self, size: int): + self._remaining = size + + def read(self, size=-1): + if size is None or size < 0: + size = self._remaining + count = min(size, self._remaining) + self._remaining -= count + return bytes(count) + + +class _CountingSink: + def __init__(self): + self.written = 0 + + def write(self, chunk): + self.written += len(chunk) + return len(chunk) + + +# The size cap the 0.12.0 wrappers enforced, which refused every release. +_FORMER_MEMBER_CAP_BYTES = 256 * 1024 * 1024 def _write_tar_gz(archive, entries): @@ -884,58 +912,32 @@ def _release_entries(names): return [(name, f"payload of {name}".encode()) for name in names] -class ArchiveLimitTests(unittest.TestCase): +class ArchiveExtractionTests(unittest.TestCase): URL = "https://example.invalid/release.tar.gz" - def _download(self, response, destination, max_bytes): + def _download(self, response, destination): with mock.patch.object(_cli._HTTPS_OPENER, "open", return_value=response): - _cli._download_https(self.URL, destination, max_bytes) - - def test_download_rejects_a_declared_length_over_the_limit(self): - with tempfile.TemporaryDirectory() as root: - destination = os.path.join(root, "release.tar.gz") - response = _FakeHttpsResponse(b"", {"Content-Length": "9"}) - with self.assertRaises(RuntimeError) as raised: - self._download(response, destination, 8) - self.assertIn("8-byte safety limit", str(raised.exception)) - self.assertFalse(os.path.exists(destination)) + _cli._download_file(self.URL, destination) - def test_download_rejects_a_body_over_the_limit_and_removes_the_partial_file( + def test_download_stores_a_body_whose_declared_length_exceeds_the_former_cap( self, ): with tempfile.TemporaryDirectory() as root: destination = os.path.join(root, "release.tar.gz") - response = _FakeHttpsResponse(b"x" * 1024) - with self.assertRaises(RuntimeError) as raised: - self._download(response, destination, 8) - self.assertIn("8-byte safety limit", str(raised.exception)) - self.assertFalse(os.path.exists(destination)) - - def test_download_within_the_limit_stores_the_body(self): - with tempfile.TemporaryDirectory() as root: - destination = os.path.join(root, "release.tar.gz") - response = _FakeHttpsResponse(b"payload", {"Content-Length": "7"}) - self._download(response, destination, 1024) + response = _FakeHttpsResponse( + b"payload", {"Content-Length": str(_FORMER_MEMBER_CAP_BYTES + 1)} + ) + self._download(response, destination) with open(destination, "rb") as stored: self.assertEqual(stored.read(), b"payload") - def test_download_counts_bytes_when_the_declared_length_is_not_a_number(self): - # A superscript two passes str.isdigit but not int(); such values and - # plain junk must fall through to the byte counter, never crash. - for declared in ("²", "abc", "", "1e3", "-1"): - with self.subTest(declared=declared), tempfile.TemporaryDirectory() as root: - destination = os.path.join(root, "release.tar.gz") - response = _FakeHttpsResponse(b"x" * 1024, {"Content-Length": declared}) - with self.assertRaises(RuntimeError) as raised: - self._download(response, destination, 8) - self.assertIn("8-byte safety limit", str(raised.exception)) - self.assertFalse(os.path.exists(destination)) + def test_download_removes_the_partial_file_when_the_body_fails(self): with tempfile.TemporaryDirectory() as root: destination = os.path.join(root, "release.tar.gz") - response = _FakeHttpsResponse(b"payload", {"Content-Length": "²"}) - self._download(response, destination, 1024) - with open(destination, "rb") as stored: - self.assertEqual(stored.read(), b"payload") + response = _FakeHttpsResponse(b"x" * 200000, fail_after_first_chunk=True) + with self.assertRaises(OSError): + self._download(response, destination) + self.assertFalse(os.path.exists(destination)) def test_unix_tar_rejects_more_members_than_the_limit(self): with tempfile.TemporaryDirectory() as root: @@ -949,50 +951,12 @@ def test_unix_tar_rejects_more_members_than_the_limit(self): str(destination), ("one", "two", "three"), ("one",), - limits=_limits_with(members=2), + max_members=2, ) self.assertIn("2-member safety limit", str(raised.exception)) self.assertFalse((destination / "one").exists()) - def test_unix_tar_rejects_a_member_declared_over_the_limit(self): - with tempfile.TemporaryDirectory() as root: - archive = Path(root) / "member.tar.gz" - _write_tar_gz(archive, [("codebase-memory-mcp", b"x" * 65)]) - destination = Path(root) / "extract" - destination.mkdir() - with tarfile.open(archive) as tf, self.assertRaises(SystemExit) as raised: - _cli._safe_extract_tar( - tf, - str(destination), - ("codebase-memory-mcp",), - ("codebase-memory-mcp",), - limits=_limits_with(member_bytes=64), - ) - self.assertIn( - "'codebase-memory-mcp' exceeds the 64-byte expanded safety limit", - str(raised.exception), - ) - self.assertFalse((destination / "codebase-memory-mcp").exists()) - - def test_unix_tar_rejects_a_declared_aggregate_over_the_limit(self): - with tempfile.TemporaryDirectory() as root: - archive = Path(root) / "aggregate.tar.gz" - _write_tar_gz(archive, [("one", b"a" * 40), ("two", b"b" * 40)]) - destination = Path(root) / "extract" - destination.mkdir() - with tarfile.open(archive) as tf, self.assertRaises(SystemExit) as raised: - _cli._safe_extract_tar( - tf, - str(destination), - ("one", "two"), - ("one",), - limits=_limits_with(member_bytes=64, expanded_bytes=64), - ) - self.assertIn( - "64-byte aggregate expanded safety limit", str(raised.exception) - ) - - def test_unix_tar_within_the_limits_extracts_only_the_executable(self): + def test_unix_tar_extracts_only_the_executable(self): with tempfile.TemporaryDirectory() as root: archive = Path(root) / "release.tar.gz" _write_tar_gz(archive, _release_entries(_cli._UNIX_ARCHIVE_NAMES)) @@ -1012,67 +976,29 @@ def test_unix_tar_within_the_limits_extracts_only_the_executable(self): ) self.assertEqual(os.listdir(destination), ["codebase-memory-mcp"]) - def test_archive_copy_stops_a_member_that_yields_more_than_the_limit(self): - source = io.BytesIO(b"x" * 100) - output = io.BytesIO() - with self.assertRaises(SystemExit) as raised: - _cli._copy_archive_member_within_limits( - source, - output, - "member", - 100, - 0, - _limits_with(member_bytes=64, expanded_bytes=1024), - ) - self.assertIn( - "'member' exceeds the 64-byte actual expanded safety limit", - str(raised.exception), + def test_archive_copy_accepts_a_member_larger_than_the_former_cap(self): + size = _FORMER_MEMBER_CAP_BYTES + 1024 * 1024 + sink = _CountingSink() + copied = _cli._copy_archive_member( + _ZeroSource(size), sink, "codebase-memory-mcp", size ) - # Reading stopped one byte past the limit; nothing beyond it was kept. - self.assertEqual(source.tell(), 65) - self.assertLessEqual(len(output.getvalue()), 64) + self.assertEqual(copied, size) + self.assertEqual(sink.written, size) def test_archive_copy_rejects_a_member_that_yields_more_than_it_declares(self): - source = io.BytesIO(b"x" * 100) with self.assertRaises(SystemExit) as raised: - _cli._copy_archive_member_within_limits( - source, - io.BytesIO(), - "member", - 10, - 0, - _limits_with(member_bytes=1024, expanded_bytes=1024), - ) + _cli._copy_archive_member(io.BytesIO(b"x" * 100), io.BytesIO(), "member", 10) self.assertIn( "'member' actual size 100 does not match declared size 10", str(raised.exception), ) - def test_archive_copy_rejects_an_aggregate_over_the_actual_limit(self): - with self.assertRaises(SystemExit) as raised: - _cli._copy_archive_member_within_limits( - io.BytesIO(b"abcd"), - io.BytesIO(), - "member", - 4, - 3, - _limits_with(member_bytes=10, expanded_bytes=5), - ) - self.assertIn( - "5-byte aggregate actual expanded safety limit", str(raised.exception) - ) - def test_archive_member_write_removes_the_partial_file_on_failure(self): with tempfile.TemporaryDirectory() as root: target = os.path.join(root, "codebase-memory-mcp") with self.assertRaises(SystemExit): - _cli._write_archive_member_within_limits( - io.BytesIO(b"x" * 100), - target, - "codebase-memory-mcp", - 10, - 0, - _limits_with(member_bytes=1024, expanded_bytes=1024), + _cli._write_archive_member( + io.BytesIO(b"x" * 100), target, "codebase-memory-mcp", 10 ) self.assertFalse(os.path.exists(target)) @@ -1088,28 +1014,11 @@ def test_windows_zip_rejects_more_members_than_the_limit(self): str(destination), ("one", "two", "three"), ("one",), - limits=_limits_with(members=2), + max_members=2, ) self.assertIn("2-member safety limit", str(raised.exception)) - def test_windows_zip_rejects_a_member_declared_over_the_limit(self): - with tempfile.TemporaryDirectory() as root: - archive = Path(root) / "member.zip" - _write_zip(archive, [(_cli._WINDOWS_BINARY_NAME, b"x" * 65)]) - destination = Path(root) / "extract" - destination.mkdir() - with zipfile.ZipFile(archive) as zf, self.assertRaises(SystemExit) as raised: - _cli._safe_extract_zip( - zf, - str(destination), - (_cli._WINDOWS_BINARY_NAME,), - (_cli._WINDOWS_BINARY_NAME,), - limits=_limits_with(member_bytes=64), - ) - self.assertIn("64-byte expanded safety limit", str(raised.exception)) - self.assertFalse((destination / _cli._WINDOWS_BINARY_NAME).exists()) - - def test_windows_zip_within_the_limits_extracts_only_the_executable(self): + def test_windows_zip_extracts_only_the_executable(self): with tempfile.TemporaryDirectory() as root: archive = Path(root) / "release.zip" _write_zip(archive, _release_entries(_cli._WINDOWS_ARCHIVE_NAMES)) @@ -1129,3 +1038,161 @@ def test_windows_zip_within_the_limits_extracts_only_the_executable(self): ) self.assertEqual(os.listdir(destination), [_cli._WINDOWS_BINARY_NAME]) + +@contextlib.contextmanager +def _loopback_server(routes): + """Serve `routes` ({path: bytes}) on 127.0.0.1; any other path redirects.""" + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = routes.get(self.path) + if body is None: + self.send_response(302) + self.send_header("Location", "https://example.invalid/elsewhere") + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + thread.join() + + +class DownloadBaseTests(unittest.TestCase): + GITHUB_BASE = "https://github.com/DeusData/codebase-memory-mcp/releases/download/v1.2.3" + + def test_unset_or_empty_uses_the_github_release_base(self): + self.assertEqual(_cli._download_base("1.2.3", {}), (self.GITHUB_BASE, False)) + self.assertEqual( + _cli._download_base("1.2.3", {"CBM_DOWNLOAD_URL": ""}), + (self.GITHUB_BASE, False), + ) + + def test_https_override_is_accepted(self): + for base in ( + "https://mirror.example/releases/v1", + "https://127.0.0.1:8443/fixture", + ): + with self.subTest(base=base): + self.assertEqual( + _cli._download_base("1.2.3", {"CBM_DOWNLOAD_URL": base}), + (base, False), + ) + + def test_exact_loopback_http_override_is_accepted(self): + for base in ( + "http://localhost", + "http://localhost:8080/fixture", + "http://127.0.0.1:41234/standard", + "http://[::1]:9/x?y#z", + ): + with self.subTest(base=base): + self.assertEqual( + _cli._download_base("1.2.3", {"CBM_DOWNLOAD_URL": base}), + (base, True), + ) + + def test_non_loopback_http_and_other_schemes_are_refused(self): + for base in ( + "http://example.com/releases", + "http://127.0.0.1.example.com/", + "http://localhost.example.com:8080/", + "http://user@localhost/", + "http://127.0.0.2/", + "HTTP://localhost/", + "file:///tmp/releases", + "ftp://localhost/releases", + "github.com/DeusData/codebase-memory-mcp", + ): + with self.subTest(base=base), self.assertRaises(SystemExit) as raised: + _cli._download_base("1.2.3", {"CBM_DOWNLOAD_URL": base}) + self.assertIn( + "refusing non-HTTPS download URL (CBM_DOWNLOAD_URL)", + str(raised.exception), + ) + + def test_https_download_refuses_plain_http(self): + with tempfile.TemporaryDirectory() as root: + with self.assertRaises(SystemExit): + _cli._download_file( + "http://127.0.0.1/release.tar.gz", os.path.join(root, "a") + ) + + def test_loopback_download_refuses_a_non_loopback_url(self): + with tempfile.TemporaryDirectory() as root: + with self.assertRaises(SystemExit) as raised: + _cli._download_file( + "http://example.invalid/release.tar.gz", + os.path.join(root, "a"), + loopback=True, + ) + self.assertIn("refusing non-loopback HTTP URL", str(raised.exception)) + + def test_loopback_download_bypasses_proxies_and_refuses_redirects(self): + routes = {"/fixture/archive": b"loopback payload"} + # A configured proxy that is not listening: using it would fail. + proxy_env = {"http_proxy": "http://127.0.0.1:9", "HTTP_PROXY": "http://127.0.0.1:9"} + with _loopback_server(routes) as origin, tempfile.TemporaryDirectory() as root, \ + mock.patch.dict(os.environ, proxy_env): + archive = os.path.join(root, "archive") + _cli._download_file(f"{origin}/fixture/archive", archive, loopback=True) + with open(archive, "rb") as stored: + self.assertEqual(stored.read(), b"loopback payload") + + redirected = os.path.join(root, "redirected") + with self.assertRaises(RuntimeError) as raised: + _cli._download_file( + f"{origin}/fixture/checksums.txt", redirected, loopback=True + ) + self.assertIn("refusing redirect from loopback download", str(raised.exception)) + self.assertFalse(os.path.exists(redirected)) + + def test_download_fetches_archive_and_checksums_from_the_override(self): + os_name = _cli._os_name() + arch = _cli._arch() + if os_name == "windows": + ext, names = "zip", _cli._WINDOWS_ARCHIVE_NAMES + else: + ext, names = "tar.gz", _cli._UNIX_ARCHIVE_NAMES + portable = "-portable" if os_name == "linux" else "" + archive_name = f"codebase-memory-mcp-{os_name}-{arch}{portable}.{ext}" + with tempfile.TemporaryDirectory() as root: + fixture = Path(root) / archive_name + if ext == "zip": + _write_zip(fixture, _release_entries(names)) + else: + _write_tar_gz(fixture, _release_entries(names)) + archive_bytes = fixture.read_bytes() + digest = hashlib.sha256(archive_bytes).hexdigest() + routes = { + f"/fixture/{archive_name}": archive_bytes, + "/fixture/checksums.txt": f"{digest} {archive_name}\n".encode(), + } + cache = Path(root) / "cache" + with _loopback_server(routes) as origin, mock.patch.dict( + os.environ, {"CBM_DOWNLOAD_URL": f"{origin}/fixture"} + ), mock.patch.object(_cli, "_cache_dir", return_value=cache), \ + mock.patch.object(_cli, "_verify_candidate"), \ + contextlib.redirect_stderr(io.StringIO()): + installed = _cli._download("9.9.9") + binary_name = ( + _cli._WINDOWS_BINARY_NAME + if os_name == "windows" + else "codebase-memory-mcp" + ) + self.assertEqual( + installed.read_bytes(), f"payload of {binary_name}".encode() + ) diff --git a/scripts/ci/select-lanes.sh b/scripts/ci/select-lanes.sh index b21083082a..04abb18805 100755 --- a/scripts/ci/select-lanes.sh +++ b/scripts/ci/select-lanes.sh @@ -128,7 +128,7 @@ SMOKE = {"scripts/smoke-test.sh", "scripts/smoke-local.sh", "scripts/smoke-fixture-server.py", "scripts/smoke-invariants.sh", "test-infrastructure/vm/vm-smoke.sh", "test-infrastructure/vm/windows-user-path-guard.ps1", - "scripts/test-windows.ps1"} + "scripts/test-windows.ps1", "scripts/smoke-wrappers.sh"} LINT_CONFIG = {"scripts/memory-core-baseline.txt", "scripts/lint-memory-core.py", "scripts/hooks/pre-commit"} NON_GATING = {"scripts/package-release.sh", "scripts/benchmark-index.sh", diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 7887c3146e..c1a3f3ec72 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -3996,6 +3996,18 @@ else echo "SKIP Phase 13: no install script available for this platform" fi +# ── Phase 13w: package-wrapper install E2E ── +# The npm, PyPI and Go wrappers install the same release archive as above, +# from this checkout, the way a user's package manager runs them (#2657: a +# wrapper cap refused every shipped binary). SMOKE_REQUIRE_WRAPPERS=1 (CI) +# makes a missing node/python/go toolchain a failure, not a skip. +echo "" +echo "--- Phase 13w: package-wrapper install E2E ---" +if ! bash "$REPO_ROOT/scripts/smoke-wrappers.sh" "$SMOKE_DOWNLOAD_URL"; then + echo "FAIL 13w: a package wrapper could not install the release binary" + exit 1 +fi + else echo "" echo "=== Phase 12-13: SKIPPED (SMOKE_DOWNLOAD_URL not set) ===" diff --git a/scripts/smoke-wrappers.sh b/scripts/smoke-wrappers.sh new file mode 100644 index 0000000000..43e7b0821c --- /dev/null +++ b/scripts/smoke-wrappers.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# smoke-wrappers.sh — install the npm, PyPI and Go wrappers from this checkout +# against a release fixture and run the binary each one installed. +# +# The wrappers download the release archive themselves. This runs each one the +# way a user's package manager does, against CBM_DOWNLOAD_URL (the test +# override install.sh honours too) serving the real release archives, so a +# wrapper that cannot install what we ship fails here instead of on users +# (#2657: a 256 MiB archive-member cap against 331 MB binaries). +# +# Usage: scripts/smoke-wrappers.sh DOWNLOAD_URL [npm] [pypi] [go] +# DOWNLOAD_URL base URL serving the release archives and checksums.txt: +# https, or plain http to a loopback authority +# (no wrapper names = all three) +# Environment: +# SMOKE_REQUIRE_WRAPPERS=1 a missing toolchain fails instead of skipping (CI) +# Exit: 0 = every wrapper installed and ran its binary (or was skipped) · +# 1 = a wrapper failed · 2 = usage error. +set -euo pipefail + +case "${BASH_SOURCE[0]}" in */*) cd "${BASH_SOURCE[0]%/*}/.." ;; *) cd .. ;; esac +ROOT="$PWD" +URL="${1:-}" +[ -n "$URL" ] || { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } +shift +WRAPPERS=("$@") +[ ${#WRAPPERS[@]} -gt 0 ] || WRAPPERS=(npm pypi go) +REQUIRE="${SMOKE_REQUIRE_WRAPPERS:-0}" +EXE="" +case "${OSTYPE:-}" in msys* | cygwin* | win32*) EXE=".exe" ;; esac + +WORK="${TMPDIR:-/tmp}/cbm-smoke-wrappers.$$.$RANDOM" +mkdir -m 700 "$WORK" +trap 'rm -rf "$WORK"' EXIT +FAILED=0 + +missing() { # wrapper, what + if [ "$REQUIRE" = "1" ]; then + echo "FAIL $1: SMOKE_REQUIRE_WRAPPERS=1 but $2 is not available" + FAILED=1 + else + echo "SKIP $1: $2 not available on this host" + fi +} + +# A fresh HOME, cache and profile per wrapper: nothing a wrapper writes leaks +# into the next one, and a binary cached by an earlier run cannot stand in. +isolated() { # wrapper dir, command... + local dir="$1" + shift + mkdir -p "$dir/home" "$dir/cache" "$dir/localappdata" + HOME="$dir/home" USERPROFILE="$dir/home" XDG_CACHE_HOME="$dir/cache" \ + LOCALAPPDATA="$dir/localappdata" CBM_CACHE_DIR="$dir/cbm-cache" \ + CBM_DOWNLOAD_URL="$URL" "$@" +} + +ran() { # wrapper, output file, rc + # The binary's own answer ("codebase-memory-mcp 0.12.0"), not a wrapper + # message ("codebase-memory-mcp: downloading ..."), proves it ran. + if [ "$3" -eq 0 ] && grep -E -q '^codebase-memory-mcp [^:]+$' "$2"; then + echo "OK $1: installed and ran the release binary: $(grep -E '^codebase-memory-mcp [^:]+$' "$2" | tail -1)" + else + echo "FAIL $1: the installed wrapper did not run the release binary (rc=$3):" + sed 's/^/ /' "$2" | tail -20 + FAILED=1 + fi +} + +smoke_npm() { + if ! command -v node >/dev/null 2>&1 || ! command -v npm >/dev/null 2>&1; then + missing npm "node/npm" + return + fi + local dir="$WORK/npm" tgz="" line modules rc=0 + mkdir -p "$dir/prefix" + (cd "$ROOT/pkg/npm" && npm pack --silent --pack-destination "$dir") >"$dir/pack.out" + while IFS= read -r line; do [ -n "$line" ] && tgz="$line"; done <"$dir/pack.out" + # A global install from the packed tarball runs postinstall (install.js) + # exactly as `npm i -g codebase-memory-mcp` does. + isolated "$dir" npm install --global --prefix "$dir/prefix" --cache "$dir/cache/npm" \ + --foreground-scripts --no-audit --no-fund "$dir/$tgz" >"$dir/install.log" 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + ran npm "$dir/install.log" "$rc" + return + fi + local modules + npm root --global --prefix "$dir/prefix" >"$dir/root.out" + IFS= read -r modules <"$dir/root.out" + isolated "$dir" node "$modules/codebase-memory-mcp/bin.js" --version >"$dir/run.log" 2>&1 || + rc=$? + ran npm "$dir/run.log" "$rc" +} + +smoke_pypi() { + local py="" + if command -v python3 >/dev/null 2>&1; then + py=python3 + elif command -v python >/dev/null 2>&1; then + py=python + fi + if [ -z "$py" ]; then + missing pypi "python3" + return + fi + local dir="$WORK/pypi" rc=0 + mkdir -p "$dir" + "$py" -m venv "$dir/venv" + local vpy="$dir/venv/bin/python" + [ -x "$vpy" ] || vpy="$dir/venv/Scripts/python.exe" + "$vpy" -m pip install --quiet --disable-pip-version-check "$ROOT/pkg/pypi" \ + >"$dir/install.log" 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + ran pypi "$dir/install.log" "$rc" + return + fi + # The console script's entry point; the first run downloads the binary. + isolated "$dir" "$vpy" -c \ + 'import sys, codebase_memory_mcp; sys.argv = ["codebase-memory-mcp", "--version"]; codebase_memory_mcp.main()' \ + >"$dir/run.log" 2>&1 || rc=$? + ran pypi "$dir/run.log" "$rc" +} + +smoke_go() { + if ! command -v go >/dev/null 2>&1; then + missing go "go" + return + fi + local dir="$WORK/go" rc=0 + mkdir -p "$dir" + (cd "$ROOT/pkg/go" && GOTOOLCHAIN="${GOTOOLCHAIN:-auto}" \ + go build -o "$dir/codebase-memory-mcp$EXE" ./cmd/codebase-memory-mcp) \ + >"$dir/install.log" 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + ran go "$dir/install.log" "$rc" + return + fi + # The first run downloads the binary into the wrapper's cache. + isolated "$dir" "$dir/codebase-memory-mcp$EXE" --version >"$dir/run.log" 2>&1 || rc=$? + ran go "$dir/run.log" "$rc" +} + +for wrapper in "${WRAPPERS[@]}"; do + case "$wrapper" in + npm) smoke_npm ;; + pypi) smoke_pypi ;; + go) smoke_go ;; + *) echo "unknown wrapper: $wrapper" >&2; exit 2 ;; + esac +done +exit "$FAILED" diff --git a/tests/test_version_metadata_contract.sh b/tests/test_version_metadata_contract.sh index 05655a1486..b74203bf6b 100644 --- a/tests/test_version_metadata_contract.sh +++ b/tests/test_version_metadata_contract.sh @@ -108,7 +108,7 @@ SURFACES=( "pkg/chocolatey/codebase-memory-mcp.nuspec|release" "pkg/chocolatey/tools/chocolateyInstall.ps1|release" "pkg/scoop/codebase-memory-mcp.json|release" - "pkg/homebrew/Formula/codebase-memory-mcp.rb|pin:0.10.3" + "pkg/homebrew/Formula/codebase-memory-mcp.rb|release" "pkg/aur/PKGBUILD|pin:0.8.1" "pkg/aur/.SRCINFO|pin:0.8.1" ) @@ -118,7 +118,6 @@ SURFACES=( # (gh release download v --pattern checksums.txt), replace the version # AND every sha256 it pins, then move the entry to "release" above. PIN_REASONS=( - "pkg/homebrew/Formula/codebase-memory-mcp.rb|pins 4 per-asset sha256 (darwin/linux x arm/intel); last re-pinned for v0.10.3" "pkg/aur/PKGBUILD|pins sha256sums_x86_64 + sha256sums_aarch64; last re-pinned for v0.8.1" "pkg/aur/.SRCINFO|generated from PKGBUILD, so it must move with it, not before it" )