From e93b1cce6a5e30c169cef693596935abfabda4eb Mon Sep 17 00:00:00 2001 From: Naved Date: Wed, 29 Jul 2026 21:15:01 -0700 Subject: [PATCH] refactor: extract managed binary installation infrastructure Refs #1055 --- src/eslint-suppressions.json | 5 - .../__tests__/semble-downloader.spec.ts | 42 +- .../code-index/semble/semble-downloader.ts | 412 +++--------------- .../managed-binary/__tests__/archive.spec.ts | 93 ++++ .../managed-binary/__tests__/download.spec.ts | 161 +++++++ .../managed-binary/__tests__/install.spec.ts | 88 ++++ src/services/managed-binary/archive.ts | 105 +++++ src/services/managed-binary/download.ts | 149 +++++++ src/services/managed-binary/install.ts | 131 ++++++ 9 files changed, 804 insertions(+), 382 deletions(-) create mode 100644 src/services/managed-binary/__tests__/archive.spec.ts create mode 100644 src/services/managed-binary/__tests__/download.spec.ts create mode 100644 src/services/managed-binary/__tests__/install.spec.ts create mode 100644 src/services/managed-binary/archive.ts create mode 100644 src/services/managed-binary/download.ts create mode 100644 src/services/managed-binary/install.ts diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index 608e190d04..4141e6a259 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -1499,11 +1499,6 @@ "count": 3 } }, - "services/code-index/semble/semble-downloader.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 1 - } - }, "services/code-index/shared/__tests__/validation-helpers.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 4 diff --git a/src/services/code-index/semble/__tests__/semble-downloader.spec.ts b/src/services/code-index/semble/__tests__/semble-downloader.spec.ts index 7a3eee9a70..8261bfd307 100644 --- a/src/services/code-index/semble/__tests__/semble-downloader.spec.ts +++ b/src/services/code-index/semble/__tests__/semble-downloader.spec.ts @@ -263,12 +263,14 @@ describe("semble-downloader", () => { ) // Version file should be written expect(fs.writeFile).toHaveBeenCalledWith( - path.join("/storage", "semble", ".semble-version"), + path.join("/storage", "semble.new", ".semble-version"), "v0.4.1", "utf-8", ) // Archive should be cleaned up (version-prefixed local cache path) - expect(fs.unlink).toHaveBeenCalledWith(path.join("/storage", "v0.4.1-semble-linux-x64-fast.tar.gz")) + expect(fs.rm).toHaveBeenCalledWith(path.join("/storage", "v0.4.1-semble-linux-x64-fast.tar.gz"), { + force: true, + }) } finally { if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform) if (originalArch) Object.defineProperty(process, "arch", originalArch) @@ -325,7 +327,9 @@ describe("semble-downloader", () => { try { await expect(downloadSemble("/storage")).rejects.toThrow("Failed to download semble") - expect(fs.unlink).toHaveBeenCalledWith(path.join("/storage", "v0.4.1-semble-linux-arm64-fast.tar.gz")) + expect(fs.rm).toHaveBeenCalledWith(path.join("/storage", "v0.4.1-semble-linux-arm64-fast.tar.gz"), { + force: true, + }) // Should clean up staging directory, not the original expect(fs.rm).toHaveBeenCalledWith(path.join("/storage", "semble.new"), { recursive: true, @@ -589,7 +593,7 @@ describe("semble-downloader", () => { }) // Archive cleanup fails but should not throw (only archive removal after extraction) - ;(fs.unlink as any).mockRejectedValue(new Error("unlink cleanup failed")) + ;(fs.rm as any).mockRejectedValueOnce(new Error("archive cleanup failed")) try { const result = await downloadSemble("/storage") @@ -641,7 +645,7 @@ describe("semble-downloader", () => { expect(https.get).toHaveBeenCalledWith(expect.stringContaining("v0.4.1"), expect.any(Function)) // Should write the new version file expect(fs.writeFile).toHaveBeenCalledWith( - path.join("/storage", "semble", ".semble-version"), + path.join("/storage", "semble.new", ".semble-version"), "v0.4.1", "utf-8", ) @@ -700,19 +704,23 @@ describe("semble-downloader", () => { ) // The stale archive is removed before the fresh download to guarantee // a clean package is verified against the new checksum. - expect(fs.unlink).toHaveBeenCalledWith(versionedArchive) + expect(fs.rm).toHaveBeenCalledWith(versionedArchive, { force: true }) // The prior-version archive (v0.4.0-*) is swept by cleanupStaleArchives // after a successful install, so a version upgrade doesn't accumulate // orphaned packages on disk. - expect(fs.unlink).toHaveBeenCalledWith(path.join("/storage", "v0.4.0-semble-linux-x64-fast.tar.gz")) + expect(fs.rm).toHaveBeenCalledWith(path.join("/storage", "v0.4.0-semble-linux-x64-fast.tar.gz"), { + force: true, + }) // The legacy unversioned archive (pre-v0.4.0 cache layout) is also // swept, covering the v0.3.1 → v0.4.1 upgrade path. - expect(fs.unlink).toHaveBeenCalledWith(path.join("/storage", "semble-linux-x64-fast.tar.gz")) + expect(fs.rm).toHaveBeenCalledWith(path.join("/storage", "semble-linux-x64-fast.tar.gz"), { + force: true, + }) // Unrelated files in the storage dir must not be touched. expect(fs.unlink).not.toHaveBeenCalledWith(path.join("/storage", "unrelated-file.txt")) // The new version file is recorded expect(fs.writeFile).toHaveBeenCalledWith( - path.join("/storage", "semble", ".semble-version"), + path.join("/storage", "semble.new", ".semble-version"), "v0.4.1", "utf-8", ) @@ -789,7 +797,7 @@ describe("semble-downloader", () => { ) // Should write version file again expect(fs.writeFile).toHaveBeenCalledWith( - path.join("/storage", "semble", ".semble-version"), + path.join("/storage", "semble.new", ".semble-version"), "v0.4.1", "utf-8", ) @@ -831,7 +839,7 @@ describe("semble-downloader", () => { ) // Should write version file expect(fs.writeFile).toHaveBeenCalledWith( - path.join("/storage", "semble", ".semble-version"), + path.join("/storage", "semble.new", ".semble-version"), "v0.4.1", "utf-8", ) @@ -903,8 +911,12 @@ describe("semble-downloader", () => { const currentArchive = path.join("/storage", "v0.4.1-semble-linux-x64-fast.tar.gz") // Stale versioned + legacy unversioned archives are swept - expect(fs.unlink).toHaveBeenCalledWith(path.join("/storage", "v0.4.0-semble-linux-x64-fast.tar.gz")) - expect(fs.unlink).toHaveBeenCalledWith(path.join("/storage", "semble-linux-x64-fast.tar.gz")) + expect(fs.rm).toHaveBeenCalledWith(path.join("/storage", "v0.4.0-semble-linux-x64-fast.tar.gz"), { + force: true, + }) + expect(fs.rm).toHaveBeenCalledWith(path.join("/storage", "semble-linux-x64-fast.tar.gz"), { + force: true, + }) // The current archive is never swept by cleanupStaleArchives (it is // excluded by the currentArchivePath guard). It is unlinked only by // the pre-download partial-archive cleanup and the post-install @@ -913,8 +925,8 @@ describe("semble-downloader", () => { // Sanity: the current archive path is never passed to the stale sweep. // It is unlinked exactly twice (pre-download cleanup + post-install // archive cleanup), never via cleanupStaleArchives. - const currentUnlinks = (fs.unlink as any).mock.calls.filter((c: any[]) => c[0] === currentArchive) - expect(currentUnlinks.length).toBe(2) + const currentRemovals = (fs.rm as any).mock.calls.filter((c: any[]) => c[0] === currentArchive) + expect(currentRemovals.length).toBe(2) } finally { if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform) if (originalArch) Object.defineProperty(process, "arch", originalArch) diff --git a/src/services/code-index/semble/semble-downloader.ts b/src/services/code-index/semble/semble-downloader.ts index fc8a8e2a33..836e1946e1 100644 --- a/src/services/code-index/semble/semble-downloader.ts +++ b/src/services/code-index/semble/semble-downloader.ts @@ -1,10 +1,9 @@ import * as fs from "fs/promises" import * as path from "path" -import * as https from "https" -import { createWriteStream } from "fs" -import { createHash } from "crypto" -import { createReadStream } from "fs" -import { spawn } from "child_process" + +import { extractTarGzArchive, extractZipArchive } from "../../managed-binary/archive" +import { downloadBinaryFile, verifySha256Checksum } from "../../managed-binary/download" +import { ensureManagedBinaryInstalled, getManagedBinaryPaths } from "../../managed-binary/install" /** * Supported platform/arch combinations for the semble standalone executable. @@ -47,19 +46,14 @@ export const SEMBLE_SHA256: Record = { * Throws if the checksum does not match. */ export async function verifyChecksum(filePath: string, expected: string): Promise { - const hash = createHash("sha256") - await new Promise((resolve, reject) => { - const stream = createReadStream(filePath) - stream.on("data", (chunk) => hash.update(chunk)) - stream.on("end", resolve) - stream.on("error", reject) - }) - const actual = hash.digest("hex") - if (actual !== expected) { - throw new Error( - `Checksum mismatch for ${path.basename(filePath)}: expected ${expected.slice(0, 12)}…, got ${actual.slice(0, 12)}…`, - ) - } + await verifySha256Checksum( + filePath, + expected, + (actual) => + new Error( + `Checksum mismatch for ${path.basename(filePath)}: expected ${expected.slice(0, 12)}…, got ${actual.slice(0, 12)}…`, + ), + ) } /** @@ -87,64 +81,6 @@ function getArchiveInfo(platform?: string, arch?: string): { archive: string; bi return SEMBLE_ARCHIVES[`${p}-${a}`] } -/** - * Reads the locally installed version from the version metadata file. - * Returns undefined if no version file exists (first install or legacy). - */ -async function getInstalledVersion(storageDir: string): Promise { - try { - const versionPath = path.join(storageDir, "semble", VERSION_FILE) - const version = (await fs.readFile(versionPath, "utf-8")).trim() - return version || undefined - } catch { - return undefined - } -} - -/** - * Writes the version metadata file after a successful download. - */ -async function writeInstalledVersion(storageDir: string, version: string): Promise { - const versionPath = path.join(storageDir, "semble", VERSION_FILE) - await fs.writeFile(versionPath, version, "utf-8") -} - -/** - * Best-effort removal of archive files left over from previous semble versions. - * - * Because the local archive cache path is version-prefixed (see `downloadSemble`), - * upgrading SEMBLE_VERSION leaves the prior version's archive orphaned on disk. - * This sweeps those stale packages so a version upgrade doesn't accumulate them. - * - * Matches both the version-prefixed cache names (`${version}-${archiveName}`, - * used since v0.4.0) and the legacy unversioned cache name (`${archiveName}`, - * used before v0.4.0), so a v0.3.1 → v0.4.1 upgrade also clears the legacy file. - * The current archive path is always preserved. - * - * Errors are swallowed since this is purely cosmetic cleanup. - */ -async function cleanupStaleArchives( - storageDir: string, - archiveName: string, - currentArchivePath: string, -): Promise { - try { - const entries = await fs.readdir(storageDir) - const suffix = `-${archiveName}` - await Promise.all( - entries - .filter( - (name) => - (name === archiveName || name.endsWith(suffix)) && - path.join(storageDir, name) !== currentArchivePath, - ) - .map((name) => fs.unlink(path.join(storageDir, name)).catch(() => {})), - ) - } catch { - // ignore — storage dir may not be listable yet - } -} - /** * Downloads and extracts the semble archive for the current platform. * @@ -164,130 +100,48 @@ export async function downloadSemble(storageDir: string): Promise + downloadBinaryFile(url, archivePath, { + name: "Semble", + trustedDomains: TRUSTED_DOWNLOAD_DOMAINS, + timeoutMs: 120_000, + }), + verifyArchive: (archivePath) => verifyChecksum(archivePath, expectedChecksum), + extractArchive: async (archivePath, stagingDir) => { + if (info.archive.endsWith(".tar.gz")) { + await extractTarGzArchive(archivePath, stagingDir) + } else if (info.archive.endsWith(".zip")) { + await extractZipArchive(archivePath, stagingDir) + } + }, + }) + + console.log(`[SembleDownloader] Successfully installed semble ${SEMBLE_VERSION} to ${paths.binaryPath}`) + return result } /** @@ -309,174 +163,8 @@ export async function getSembleBinaryPath(storageDir: string): Promise { - return new Promise((resolve, reject) => { - const args = ["-xzf", archivePath, "-C", destDir, "--no-same-owner"] - // GNU tar: --no-overwrite-dir adds defense-in-depth against ../relative traversal. - // macOS bsdtar strips absolute paths by default. - if (process.platform === "linux") { - args.push("--no-overwrite-dir") - } - const child = spawn("tar", args, { - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }) - - let stderr = "" - child.stderr?.on("data", (data: Buffer) => { - stderr += data.toString() - }) - - child.on("error", (err) => reject(err)) - child.on("close", (code) => { - if (code === 0) { - resolve() - } else { - reject(new Error(`tar extraction failed (code ${code}): ${stderr.trim()}`)) - } - }) - }) -} - -/** - * Escapes a string for use inside a PowerShell single-quoted literal. - * In PowerShell, the only special character in a single-quoted string is the - * apostrophe itself, which is escaped by doubling it. - */ -function escapePowerShellLiteral(value: string): string { - return value.replace(/'/g, "''") -} - -/** - * Extracts a .zip archive into the destination directory. - * Uses PowerShell on Windows, unzip on other platforms. - */ -function extractZip(archivePath: string, destDir: string): Promise { - return new Promise((resolve, reject) => { - let child - - if (process.platform === "win32") { - child = spawn( - "powershell", - [ - "-NoProfile", - "-Command", - `Expand-Archive -Path '${escapePowerShellLiteral(archivePath)}' -DestinationPath '${escapePowerShellLiteral(destDir)}' -Force`, - ], - { shell: false, stdio: ["ignore", "pipe", "pipe"] }, - ) - } else { - child = spawn("unzip", ["-o", archivePath, "-d", destDir], { - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }) - } - - let stderr = "" - child.stderr?.on("data", (data: Buffer) => { - stderr += data.toString() - }) - - child.on("error", (err) => reject(err)) - child.on("close", (code) => { - if (code === 0) { - resolve() - } else { - reject(new Error(`zip extraction failed (code ${code}): ${stderr.trim()}`)) - } - }) - }) -} - /** * Trusted domains for following redirects during semble binary download. * GitHub releases redirect to objects.githubusercontent.com for the actual download. */ const TRUSTED_DOWNLOAD_DOMAINS = ["github.com", "objects.githubusercontent.com", "release-assets.githubusercontent.com"] - -/** - * Validates that a URL belongs to a trusted domain. - * Uses domain-boundary aware matching to prevent suffix-based bypasses - * (e.g. "evilgithub.com" does NOT match "github.com"). - */ -function isTrustedDownloadUrl(url: string): boolean { - try { - const parsed = new URL(url) - const h = parsed.hostname - return parsed.protocol === "https:" && TRUSTED_DOWNLOAD_DOMAINS.some((d) => h === d || h.endsWith("." + d)) - } catch { - return false - } -} - -/** - * Downloads a file from the given URL to the destination path. - * Follows redirects (GitHub releases use 302 redirects to CDN). - * Only follows redirects to trusted domains to prevent redirect-based attacks. - */ -function downloadFile(url: string, destPath: string, maxRedirects = 5): Promise { - return new Promise((resolve, reject) => { - if (maxRedirects <= 0) { - reject(new Error("Too many redirects")) - return - } - - const request = https.get(url, (response) => { - // Follow redirects - if ( - response.statusCode && - response.statusCode >= 300 && - response.statusCode < 400 && - response.headers.location - ) { - response.destroy() - const redirectUrl = response.headers.location - if (!isTrustedDownloadUrl(redirectUrl)) { - reject( - new Error( - `Redirect to untrusted domain blocked: ${redirectUrl}. Only ${TRUSTED_DOWNLOAD_DOMAINS.join(", ")} are allowed.`, - ), - ) - return - } - downloadFile(redirectUrl, destPath, maxRedirects - 1) - .then(resolve) - .catch(reject) - return - } - - if (response.statusCode !== 200) { - response.destroy() - reject(new Error(`HTTP ${response.statusCode}: Failed to download ${url}`)) - return - } - - const file = createWriteStream(destPath) - response.pipe(file) - - file.on("finish", () => { - file.close() - resolve() - }) - - file.on("error", (err) => { - file.close() - reject(err) - }) - }) - - request.on("error", reject) - request.on("timeout", () => { - request.destroy() - reject(new Error("Download timed out")) - }) - - // 2 minute timeout for download - request.setTimeout(120_000) - }) -} diff --git a/src/services/managed-binary/__tests__/archive.spec.ts b/src/services/managed-binary/__tests__/archive.spec.ts new file mode 100644 index 0000000000..b100c5adc4 --- /dev/null +++ b/src/services/managed-binary/__tests__/archive.spec.ts @@ -0,0 +1,93 @@ +import { EventEmitter } from "events" +import { PassThrough } from "stream" + +import { spawn } from "child_process" + +import { + escapePowerShellLiteral, + extractSingleFileTarXzArchive, + extractSingleFileZipArchive, + extractTarGzArchive, + runProcess, +} from "../archive" + +vi.mock("child_process", () => ({ spawn: vi.fn() })) + +const mockSpawn = vi.mocked(spawn) + +function createChild() { + return Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + kill: vi.fn(), + }) +} + +describe("managed binary archive utilities", () => { + beforeEach(() => mockSpawn.mockReset()) + + it("runs processes without a shell and returns their output", async () => { + const child = createChild() + mockSpawn.mockReturnValue(child as unknown as ReturnType) + const processResult = runProcess("tool", ["--version"]) + child.stdout.write("1.2.3") + child.emit("close", 0) + + await expect(processResult).resolves.toEqual({ stdout: "1.2.3", stderr: "" }) + expect(mockSpawn).toHaveBeenCalledWith("tool", ["--version"], { + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }) + }) + + it("escapes PowerShell single-quoted literals", () => { + expect(escapePowerShellLiteral("C:\\it's\\archive.zip")).toBe("C:\\it''s\\archive.zip") + }) + + it("extracts tar.gz archives with hardened flags", async () => { + const child = createChild() + mockSpawn.mockReturnValue(child as unknown as ReturnType) + const extraction = extractTarGzArchive("/tmp/archive.tar.gz", "/tmp/output") + child.emit("close", 0) + await extraction + + expect(mockSpawn).toHaveBeenCalledWith( + "tar", + expect.arrayContaining(["-xzf", "/tmp/archive.tar.gz", "-C", "/tmp/output", "--no-same-owner"]), + expect.objectContaining({ shell: false }), + ) + }) + + it("validates a single-file tar.xz layout before extraction", async () => { + const listing = createChild() + const extraction = createChild() + mockSpawn.mockReturnValueOnce(listing as unknown as ReturnType) + mockSpawn.mockReturnValueOnce(extraction as unknown as ReturnType) + const result = extractSingleFileTarXzArchive("/tmp/archive.tar.xz", "/tmp/output", "binary", "Tool") + listing.stdout.write("./binary\n") + listing.emit("close", 0) + await new Promise((resolve) => setImmediate(resolve)) + extraction.emit("close", 0) + await result + + expect(mockSpawn).toHaveBeenNthCalledWith( + 2, + "tar", + ["-xJf", "/tmp/archive.tar.xz", "-C", "/tmp/output", "binary"], + expect.any(Object), + ) + }) + + it("builds a single-entry-validated PowerShell ZIP extraction", async () => { + const child = createChild() + mockSpawn.mockReturnValue(child as unknown as ReturnType) + const extraction = extractSingleFileZipArchive("C:\\archive.zip", "C:\\output", "binary.exe", "Tool") + child.emit("close", 0) + await extraction + + const script = mockSpawn.mock.calls[0][1][3] + expect(script).toContain("$entries.Count -ne 1") + expect(script).toContain("binary.exe") + expect(script).toContain("Tool archive has an unexpected layout") + }) +}) diff --git a/src/services/managed-binary/__tests__/download.spec.ts b/src/services/managed-binary/__tests__/download.spec.ts new file mode 100644 index 0000000000..4c4ea195e5 --- /dev/null +++ b/src/services/managed-binary/__tests__/download.spec.ts @@ -0,0 +1,161 @@ +import { EventEmitter } from "events" +import { createReadStream, createWriteStream } from "fs" +import { get } from "https" +import type { IncomingMessage, RequestOptions } from "http" + +import { + assertSizeWithinLimit, + downloadBinaryFile, + isTrustedHttpsUrl, + resolveTrustedRedirect, + verifySha256Checksum, +} from "../download" + +vi.mock("crypto", () => ({ + createHash: vi.fn(() => ({ + update: vi.fn(), + digest: vi.fn(() => "actual-checksum"), + })), +})) + +vi.mock("fs", () => ({ + createReadStream: vi.fn(), + createWriteStream: vi.fn(), +})) + +vi.mock("https", () => ({ get: vi.fn() })) + +const trustedDomains = ["github.com", "objects.githubusercontent.com"] +const mockGet = vi.mocked(get) +const mockCreateReadStream = vi.mocked(createReadStream) +const mockCreateWriteStream = vi.mocked(createWriteStream) + +function createRequest(): EventEmitter & { setTimeout: ReturnType; destroy: ReturnType } { + return Object.assign(new EventEmitter(), { setTimeout: vi.fn(), destroy: vi.fn() }) +} + +function createResponse(statusCode: number, headers: Record = {}) { + return Object.assign(new EventEmitter(), { + statusCode, + headers, + destroy: vi.fn(), + pipe: vi.fn(), + }) +} + +describe("managed binary downloads", () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it("validates HTTPS URLs against hostname boundaries", () => { + expect(isTrustedHttpsUrl("https://github.com/release", trustedDomains)).toBe(true) + expect(isTrustedHttpsUrl("https://cdn.objects.githubusercontent.com/release", trustedDomains)).toBe(true) + expect(isTrustedHttpsUrl("http://github.com/release", trustedDomains)).toBe(false) + expect(isTrustedHttpsUrl("https://evilgithub.com/release", trustedDomains)).toBe(false) + expect(isTrustedHttpsUrl("not a URL", trustedDomains)).toBe(false) + }) + + it("resolves relative redirects and rejects unsafe or exhausted redirects", () => { + const options = { name: "Example", trustedDomains } + expect(resolveTrustedRedirect("https://github.com/release", "/asset", 5, options)).toBe( + "https://github.com/asset", + ) + expect(() => + resolveTrustedRedirect("https://github.com/release", "https://example.com/asset", 5, options), + ).toThrow("Example download redirected to an untrusted host") + expect(() => resolveTrustedRedirect("https://github.com/release", "/asset", 0, options)).toThrow( + "Too many Example download redirects", + ) + }) + + it("enforces configurable archive size limits", () => { + expect(() => assertSizeWithinLimit(10, 10, "Example")).not.toThrow() + expect(() => assertSizeWithinLimit(11, 10, "Example")).toThrow( + "Example archive exceeds the download size limit", + ) + }) + + it("reports the actual SHA-256 value through a caller-defined mismatch error", async () => { + const input = new EventEmitter() + mockCreateReadStream.mockReturnValue(input as ReturnType) + const verification = verifySha256Checksum( + "/tmp/archive", + "expected-checksum", + (actual) => new Error(`checksum mismatch: ${actual}`), + ) + input.emit("data", Buffer.from("archive")) + input.emit("end") + await expect(verification).rejects.toThrow("checksum mismatch: actual-checksum") + }) + + it("follows a trusted redirect and applies destination security options", async () => { + const requestOne = createRequest() + const requestTwo = createRequest() + const redirect = createResponse(302, { location: "/asset" }) + const success = createResponse(200, { "content-length": "7" }) + const output = Object.assign(new EventEmitter(), { close: vi.fn() }) + mockCreateWriteStream.mockReturnValue(output as unknown as ReturnType) + + mockGet + .mockImplementationOnce((_url, optionsOrCallback, optionalCallback) => { + const callback = + typeof optionsOrCallback === "function" + ? optionsOrCallback + : (optionalCallback as ((response: IncomingMessage) => void) | undefined) + setImmediate(() => callback?.(redirect as unknown as IncomingMessage)) + return requestOne as unknown as ReturnType + }) + .mockImplementationOnce((_url, optionsOrCallback, optionalCallback) => { + const callback = + typeof optionsOrCallback === "function" + ? optionsOrCallback + : (optionalCallback as ((response: IncomingMessage) => void) | undefined) + setImmediate(() => callback?.(success as unknown as IncomingMessage)) + return requestTwo as unknown as ReturnType + }) + + const download = downloadBinaryFile("https://github.com/release", "/tmp/archive", { + name: "Example", + trustedDomains, + timeoutMs: 1_000, + maxBytes: 10, + exclusiveDestination: true, + }) + await new Promise((resolve) => setImmediate(resolve)) + await new Promise((resolve) => setImmediate(resolve)) + output.emit("finish") + await download + + expect(mockGet).toHaveBeenNthCalledWith(2, "https://github.com/asset", expect.any(Function)) + expect(mockCreateWriteStream).toHaveBeenCalledWith("/tmp/archive", { flags: "wx", mode: 0o600 }) + expect(requestOne.setTimeout).toHaveBeenCalledWith(1_000, expect.any(Function)) + expect(requestTwo.setTimeout).toHaveBeenCalledWith(1_000, expect.any(Function)) + }) + + it("rejects an oversized declared response before opening the destination", async () => { + const request = createRequest() + const response = createResponse(200, { "content-length": "11" }) + mockGet.mockImplementation( + ( + _url: string | URL, + optionsOrCallback: RequestOptions | ((response: IncomingMessage) => void), + optionalCallback?: (response: IncomingMessage) => void, + ) => { + const callback = typeof optionsOrCallback === "function" ? optionsOrCallback : optionalCallback + setImmediate(() => callback?.(response as unknown as IncomingMessage)) + return request as unknown as ReturnType + }, + ) + + await expect( + downloadBinaryFile("https://github.com/release", "/tmp/archive", { + name: "Example", + trustedDomains, + timeoutMs: 1_000, + maxBytes: 10, + }), + ).rejects.toThrow("Example archive exceeds the download size limit") + expect(mockCreateWriteStream).not.toHaveBeenCalled() + }) +}) diff --git a/src/services/managed-binary/__tests__/install.spec.ts b/src/services/managed-binary/__tests__/install.spec.ts new file mode 100644 index 0000000000..0041706470 --- /dev/null +++ b/src/services/managed-binary/__tests__/install.spec.ts @@ -0,0 +1,88 @@ +import { access, chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises" +import { tmpdir } from "os" +import * as path from "path" + +import { ensureManagedBinaryInstalled, getManagedBinaryPaths, type ManagedBinaryInstallOptions } from "../install" + +describe("managed binary installation", () => { + let tempDir: string + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), "managed-binary-")) + }) + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }) + }) + + function createOptions(overrides: Partial = {}): ManagedBinaryInstallOptions { + return { + storageDir: tempDir, + id: "example", + version: "v1.2.3", + versionFile: ".example-version", + archiveName: "example.tar.gz", + binaryName: "example", + download: vi.fn(), + verifyArchive: vi.fn(), + extractArchive: vi.fn(), + ...overrides, + } + } + + it("derives one consistent mutable installation layout", () => { + expect(getManagedBinaryPaths(createOptions())).toEqual({ + installRoot: path.join(tempDir, "example"), + binaryPath: path.join(tempDir, "example", "example"), + versionPath: path.join(tempDir, "example", ".example-version"), + stagingDir: path.join(tempDir, "example.new"), + stagedBinaryPath: path.join(tempDir, "example.new", "example"), + archivePath: path.join(tempDir, "v1.2.3-example.tar.gz"), + }) + }) + + it("reuses a current executable without invoking update callbacks", async () => { + const options = createOptions() + const paths = getManagedBinaryPaths(options) + await mkdir(paths.installRoot, { recursive: true }) + await writeFile(paths.binaryPath, "current") + await writeFile(paths.versionPath, options.version) + if (process.platform !== "win32") await chmod(paths.binaryPath, 0o600) + + await expect(ensureManagedBinaryInstalled(options)).resolves.toBe(paths.binaryPath) + expect(options.download).not.toHaveBeenCalled() + }) + + it("deduplicates concurrent installations", () => { + const options = createOptions({ download: () => new Promise(() => {}) }) + expect(ensureManagedBinaryInstalled(options)).toBe(ensureManagedBinaryInstalled(options)) + }) + + it("coordinates update, metadata promotion, and cleanup", async () => { + const calls: string[] = [] + const options = createOptions({ + download: async (archivePath) => { + calls.push("download") + await writeFile(archivePath, "archive") + }, + verifyArchive: async () => { + calls.push("verify") + }, + extractArchive: async (_archivePath, stagingDir) => { + calls.push("extract") + await writeFile(path.join(stagingDir, "example"), "binary") + }, + validateBinary: async () => { + calls.push("validate") + }, + }) + const paths = getManagedBinaryPaths(options) + + await expect(ensureManagedBinaryInstalled(options)).resolves.toBe(paths.binaryPath) + expect(calls).toEqual(["download", "verify", "extract", "validate"]) + expect(await readFile(paths.binaryPath, "utf8")).toBe("binary") + expect(await readFile(paths.versionPath, "utf8")).toBe(options.version) + await expect(access(paths.archivePath)).rejects.toThrow() + await expect(access(paths.stagingDir)).rejects.toThrow() + }) +}) diff --git a/src/services/managed-binary/archive.ts b/src/services/managed-binary/archive.ts new file mode 100644 index 0000000000..5c3f26a7fa --- /dev/null +++ b/src/services/managed-binary/archive.ts @@ -0,0 +1,105 @@ +import { spawn } from "child_process" +import * as path from "path" + +export interface ProcessResult { + stdout: string + stderr: string +} + +export function runProcess(executable: string, args: string[], timeoutMs = 30_000): Promise { + return new Promise((resolve, reject) => { + const child = spawn(executable, args, { shell: false, stdio: ["ignore", "pipe", "pipe"] }) + let stdout = "" + let stderr = "" + const timer = setTimeout(() => { + child.kill("SIGKILL") + reject(new Error(`${path.basename(executable)} timed out`)) + }, timeoutMs) + child.stdout?.on("data", (chunk: Buffer) => (stdout += chunk.toString())) + child.stderr?.on("data", (chunk: Buffer) => (stderr += chunk.toString())) + child.on("error", (error) => { + clearTimeout(timer) + reject(error) + }) + child.on("close", (code) => { + clearTimeout(timer) + if (code === 0) { + resolve({ stdout, stderr }) + } else { + reject(new Error(stderr.trim() || `Process exited with code ${code}`)) + } + }) + }) +} + +export function escapePowerShellLiteral(value: string): string { + return value.replace(/'/g, "''") +} + +export async function extractTarGzArchive(archivePath: string, destination: string): Promise { + const args = ["-xzf", archivePath, "-C", destination, "--no-same-owner"] + if (process.platform === "linux") { + args.push("--no-overwrite-dir") + } + await runProcess("tar", args) +} + +export async function extractTarXzArchive(archivePath: string, destination: string): Promise { + const args = ["-xJf", archivePath, "-C", destination, "--no-same-owner"] + if (process.platform === "linux") { + args.push("--no-overwrite-dir") + } + await runProcess("tar", args) +} + +export async function extractZipArchive(archivePath: string, destination: string): Promise { + if (process.platform === "win32") { + await runProcess("powershell", [ + "-NoProfile", + "-Command", + `Expand-Archive -Path '${escapePowerShellLiteral(archivePath)}' -DestinationPath '${escapePowerShellLiteral(destination)}' -Force`, + ]) + return + } + + await runProcess("unzip", ["-o", archivePath, "-d", destination]) +} + +export async function extractSingleFileZipArchive( + archivePath: string, + destination: string, + expectedFile: string, + archiveName: string, +): Promise { + const outputPath = path.join(destination, expectedFile) + const script = [ + "$ErrorActionPreference = 'Stop'", + "Add-Type -AssemblyName System.IO.Compression.FileSystem", + `$archive = [System.IO.Compression.ZipFile]::OpenRead('${escapePowerShellLiteral(archivePath)}')`, + "try {", + " $entries = @($archive.Entries | Where-Object { -not [string]::IsNullOrEmpty($_.Name) })", + ` if ($entries.Count -ne 1 -or $entries[0].FullName -ne '${escapePowerShellLiteral(expectedFile)}') { throw '${escapePowerShellLiteral(archiveName)} archive has an unexpected layout' }`, + ` [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entries[0], '${escapePowerShellLiteral(outputPath)}', $false)`, + "} finally { $archive.Dispose() }", + ].join("; ") + + await runProcess("powershell", ["-NoProfile", "-NonInteractive", "-Command", script]) +} + +export async function extractSingleFileTarXzArchive( + archivePath: string, + destination: string, + expectedFile: string, + archiveName: string, +): Promise { + const listing = await runProcess("tar", ["-tJf", archivePath]) + const entries = listing.stdout + .split(/\r?\n/) + .map((entry) => entry.trim().replace(/^\.\//, "")) + .filter(Boolean) + if (entries.length !== 1 || entries[0] !== expectedFile) { + throw new Error(`${archiveName} archive has an unexpected layout`) + } + + await runProcess("tar", ["-xJf", archivePath, "-C", destination, expectedFile]) +} diff --git a/src/services/managed-binary/download.ts b/src/services/managed-binary/download.ts new file mode 100644 index 0000000000..c17eaed909 --- /dev/null +++ b/src/services/managed-binary/download.ts @@ -0,0 +1,149 @@ +import { createHash } from "crypto" +import { createReadStream, createWriteStream } from "fs" +import * as https from "https" + +export interface BinaryDownloadOptions { + name: string + trustedDomains: readonly string[] + timeoutMs: number + maxBytes?: number + maxRedirects?: number + exclusiveDestination?: boolean +} + +export function isTrustedHttpsUrl(url: string, trustedDomains: readonly string[]): boolean { + try { + const parsed = new URL(url) + return ( + parsed.protocol === "https:" && + trustedDomains.some((domain) => parsed.hostname === domain || parsed.hostname.endsWith(`.${domain}`)) + ) + } catch { + return false + } +} + +export function resolveTrustedRedirect( + url: string, + location: string | undefined, + redirectsRemaining: number, + options: Pick, +): string { + if (redirectsRemaining <= 0 || !location) { + throw new Error(`Too many ${options.name} download redirects`) + } + + const nextUrl = new URL(location, url).toString() + if (!isTrustedHttpsUrl(nextUrl, options.trustedDomains)) { + throw new Error(`${options.name} download redirected to an untrusted host (untrusted domain)`) + } + + return nextUrl +} + +export function assertSizeWithinLimit(size: number, maxBytes: number, name: string): void { + if (size > maxBytes) { + throw new Error(`${name} archive exceeds the download size limit`) + } +} + +export async function verifySha256Checksum( + filePath: string, + expected: string, + createMismatchError: (actual: string) => Error, +): Promise { + const hash = createHash("sha256") + await new Promise((resolve, reject) => { + const input = createReadStream(filePath) + input.on("data", (chunk) => hash.update(chunk)) + input.on("end", resolve) + input.on("error", reject) + }) + + const actual = hash.digest("hex") + if (actual !== expected) { + throw createMismatchError(actual) + } +} + +export function downloadBinaryFile(url: string, destination: string, options: BinaryDownloadOptions): Promise { + return downloadBinaryFileWithRedirects(url, destination, options, options.maxRedirects ?? 5) +} + +function downloadBinaryFileWithRedirects( + url: string, + destination: string, + options: BinaryDownloadOptions, + redirectsRemaining: number, +): Promise { + return new Promise((resolve, reject) => { + if (!isTrustedHttpsUrl(url, options.trustedDomains)) { + reject(new Error(`${options.name} download redirected to an untrusted host (untrusted domain)`)) + return + } + + const request = https.get(url, (response) => { + const status = response.statusCode ?? 0 + if ([301, 302, 303, 307, 308].includes(status)) { + response.destroy() + let nextUrl: string + try { + nextUrl = resolveTrustedRedirect(url, response.headers.location, redirectsRemaining, options) + } catch (error) { + reject(error) + return + } + downloadBinaryFileWithRedirects(nextUrl, destination, options, redirectsRemaining - 1).then( + resolve, + reject, + ) + return + } + + if (status !== 200) { + response.destroy() + reject(new Error(`${options.name} download failed with HTTP ${status}`)) + return + } + + const declaredSize = Number(response.headers["content-length"] ?? 0) + if (options.maxBytes !== undefined) { + try { + assertSizeWithinLimit(declaredSize, options.maxBytes, options.name) + } catch (error) { + response.destroy() + reject(error) + return + } + } + + let received = 0 + const output = createWriteStream( + destination, + options.exclusiveDestination ? { flags: "wx", mode: 0o600 } : undefined, + ) + response.on("data", (chunk: Buffer) => { + received += chunk.length + if (options.maxBytes !== undefined) { + try { + assertSizeWithinLimit(received, options.maxBytes, options.name) + } catch (error) { + response.destroy() + request.destroy(error as Error) + reject(error) + } + } + }) + response.on("error", reject) + response.pipe(output) + output.on("finish", () => { + output.close() + resolve() + }) + output.on("error", reject) + }) + + request.setTimeout(options.timeoutMs, () => request.destroy(new Error(`${options.name} download timed out`))) + request.on("error", reject) + }) +} diff --git a/src/services/managed-binary/install.ts b/src/services/managed-binary/install.ts new file mode 100644 index 0000000000..5257cfdf9f --- /dev/null +++ b/src/services/managed-binary/install.ts @@ -0,0 +1,131 @@ +import * as fs from "fs/promises" +import * as path from "path" + +const installationPromises = new Map>() + +export interface ManagedBinaryInstallOptions { + storageDir: string + id: string + version: string + versionFile: string + archiveName: string + binaryName: string + download: (archivePath: string) => Promise + verifyArchive: (archivePath: string) => Promise + extractArchive: (archivePath: string, stagingDir: string) => Promise + validateBinary?: (stagedBinaryPath: string) => Promise + errorPrefix?: string +} + +export interface ManagedBinaryPaths { + installRoot: string + binaryPath: string + versionPath: string + stagingDir: string + stagedBinaryPath: string + archivePath: string +} + +export function getManagedBinaryPaths( + options: Pick< + ManagedBinaryInstallOptions, + "storageDir" | "id" | "version" | "versionFile" | "archiveName" | "binaryName" + >, +): ManagedBinaryPaths { + const installRoot = path.join(options.storageDir, options.id) + const stagingDir = path.join(options.storageDir, `${options.id}.new`) + return { + installRoot, + binaryPath: path.join(installRoot, options.binaryName), + versionPath: path.join(installRoot, options.versionFile), + stagingDir, + stagedBinaryPath: path.join(stagingDir, options.binaryName), + archivePath: path.join(options.storageDir, `${options.version}-${options.archiveName}`), + } +} + +async function readInstalledVersion(versionPath: string): Promise { + try { + const version = (await fs.readFile(versionPath, "utf8")).trim() + return version || undefined + } catch { + return undefined + } +} + +async function makeExecutable(binaryPath: string): Promise { + await fs.access(binaryPath) + if (process.platform !== "win32") { + await fs.chmod(binaryPath, 0o755) + } +} + +async function cleanupStaleArchives(options: ManagedBinaryInstallOptions, currentArchivePath: string): Promise { + try { + const entries = await fs.readdir(options.storageDir) + const suffix = `-${options.archiveName}` + await Promise.all( + entries + .filter( + (name) => + (name === options.archiveName || name.endsWith(suffix)) && + path.join(options.storageDir, name) !== currentArchivePath, + ) + .map((name) => fs.rm(path.join(options.storageDir, name), { force: true }).catch(() => {})), + ) + } catch { + // Archive cleanup is cosmetic and must not invalidate a successful installation. + } +} + +async function installManagedBinary(options: ManagedBinaryInstallOptions): Promise { + const paths = getManagedBinaryPaths(options) + await fs.mkdir(options.storageDir, { recursive: true }) + const installedVersion = await readInstalledVersion(paths.versionPath) + if (installedVersion === options.version) { + try { + await makeExecutable(paths.binaryPath) + return paths.binaryPath + } catch { + // The installation is absent or incomplete, so rebuild it below. + } + } + + await fs.rm(paths.archivePath, { force: true }).catch(() => {}) + await fs.rm(paths.stagingDir, { recursive: true, force: true }).catch(() => {}) + await fs.mkdir(paths.stagingDir, { recursive: true }) + + try { + await options.download(paths.archivePath) + await options.verifyArchive(paths.archivePath) + await options.extractArchive(paths.archivePath, paths.stagingDir) + await makeExecutable(paths.stagedBinaryPath) + await options.validateBinary?.(paths.stagedBinaryPath) + await fs.writeFile(path.join(paths.stagingDir, options.versionFile), options.version, "utf-8") + await fs.rm(paths.installRoot, { recursive: true, force: true }) + await fs.rename(paths.stagingDir, paths.installRoot) + await cleanupStaleArchives(options, paths.archivePath) + return paths.binaryPath + } catch (error) { + if (!options.errorPrefix) { + throw error + } + const message = error instanceof Error ? error.message : String(error) + throw new Error(`${options.errorPrefix}: ${message}`) + } finally { + await fs.rm(paths.archivePath, { force: true }).catch(() => {}) + await fs.rm(paths.stagingDir, { recursive: true, force: true }).catch(() => {}) + } +} + +export function ensureManagedBinaryInstalled(options: ManagedBinaryInstallOptions): Promise { + const key = path.join(options.storageDir, options.id) + const existing = installationPromises.get(key) + if (existing) { + return existing + } + + const installation = installManagedBinary(options).finally(() => installationPromises.delete(key)) + installationPromises.set(key, installation) + return installation +}