diff --git a/src/bin/cmd-mcp.ts b/src/bin/cmd-mcp.ts index f9dbbdc..0cdc91e 100644 --- a/src/bin/cmd-mcp.ts +++ b/src/bin/cmd-mcp.ts @@ -7,6 +7,7 @@ import { FilesystemStore } from "../stores/filesystem.js"; import { runMCPServer } from "../clients/mcp-server.js"; import { parseIndexSpecs } from "../stores/index-spec.js"; import { CompositeStoreReader } from "../stores/composite.js"; +import { ReadOnlyLayeredStore } from "../stores/read-only-layered-store.js"; // stdio subcommand (stdio-based MCP server for local clients like Claude Desktop) const stdioCommand = new Command("stdio") @@ -15,29 +16,43 @@ const stdioCommand = new Command("stdio") "-i, --index ", "Index spec(s): name, path:/path, or s3://bucket/key" ) + .option("--discovery", "Enable discovery mode (read-only, manage indexes via CLI)") .option("--search-only", "Disable list_files/read_file tools (search only)") .action(async (options) => { try { const indexSpecs: string[] | undefined = options.index; + const discoveryFlag = options.discovery; let store; - let indexNames: string[]; + let indexNames: string[] | undefined; + let discovery: boolean; - if (indexSpecs && indexSpecs.length > 0) { - // Parse index specs and create composite store + if (discoveryFlag && indexSpecs && indexSpecs.length > 0) { + // Discovery mode WITH remote indexes: merge local + remote + const specs = parseIndexSpecs(indexSpecs); + const remoteStore = await CompositeStoreReader.fromSpecs(specs); + const localStore = new FilesystemStore(); + store = new ReadOnlyLayeredStore(localStore, remoteStore); + indexNames = undefined; // Discovery mode: no fixed list + discovery = true; + } else if (indexSpecs && indexSpecs.length > 0) { + // Fixed mode: use read-only CompositeStoreReader const specs = parseIndexSpecs(indexSpecs); - store = await CompositeStoreReader.fromSpecs(specs); indexNames = specs.map((s) => s.displayName); + store = await CompositeStoreReader.fromSpecs(specs); + discovery = false; } else { - // No --index: use default store, list all indexes + // No flags: restore original behavior - fail fast if no indexes store = new FilesystemStore(); - indexNames = await store.list(); - if (indexNames.length === 0) { + const availableIndexes = await store.list(); + if (availableIndexes.length === 0) { console.error("Error: No indexes found."); console.error("The MCP server requires at least one index to operate."); console.error("Run 'ctxc index --help' to see how to create an index."); process.exit(1); } + indexNames = availableIndexes; + discovery = false; } // Start MCP server (writes to stdout, reads from stdin) @@ -45,6 +60,7 @@ const stdioCommand = new Command("stdio") store, indexNames, searchOnly: options.searchOnly, + discovery, }); } catch (error) { // Write errors to stderr (stdout is for MCP protocol) @@ -60,6 +76,7 @@ const httpCommand = new Command("http") "-i, --index ", "Index spec(s): name, path:/path, or s3://bucket/key" ) + .option("--discovery", "Enable discovery mode (read-only, manage indexes via CLI)") .option("--port ", "Port to listen on", "3000") .option("--host ", "Host to bind to", "localhost") .option("--cors ", "CORS origins (comma-separated, or '*' for any)") @@ -72,17 +89,28 @@ const httpCommand = new Command("http") .action(async (options) => { try { const indexSpecs: string[] | undefined = options.index; + const discoveryFlag = options.discovery; let store; let indexNames: string[] | undefined; + let discovery: boolean; - if (indexSpecs && indexSpecs.length > 0) { - // Parse index specs and create composite store + if (discoveryFlag && indexSpecs && indexSpecs.length > 0) { + // Discovery mode WITH remote indexes: merge local + remote + const specs = parseIndexSpecs(indexSpecs); + const remoteStore = await CompositeStoreReader.fromSpecs(specs); + const localStore = new FilesystemStore(); + store = new ReadOnlyLayeredStore(localStore, remoteStore); + indexNames = undefined; // Discovery mode: no fixed list + discovery = true; + } else if (indexSpecs && indexSpecs.length > 0) { + // Fixed mode: use read-only CompositeStoreReader const specs = parseIndexSpecs(indexSpecs); - store = await CompositeStoreReader.fromSpecs(specs); indexNames = specs.map((s) => s.displayName); + store = await CompositeStoreReader.fromSpecs(specs); + discovery = false; } else { - // No --index: use default store, serve all + // No flags: restore original behavior - fail fast if no indexes store = new FilesystemStore(); const availableIndexes = await store.list(); if (availableIndexes.length === 0) { @@ -91,7 +119,8 @@ const httpCommand = new Command("http") console.error("Run 'ctxc index --help' to see how to create an index."); process.exit(1); } - indexNames = undefined; + indexNames = availableIndexes; + discovery = false; } // Parse CORS option @@ -112,6 +141,7 @@ const httpCommand = new Command("http") store, indexNames, searchOnly: options.searchOnly, + discovery, port: parseInt(options.port, 10), host: options.host, cors, diff --git a/src/clients/mcp-server.test.ts b/src/clients/mcp-server.test.ts index 96c9e9c..e71152e 100644 --- a/src/clients/mcp-server.test.ts +++ b/src/clients/mcp-server.test.ts @@ -6,6 +6,10 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import type { IndexState } from "../core/types.js"; import type { IndexStoreReader } from "../stores/types.js"; import type { Source } from "../sources/types.js"; +import { + ListToolsRequestSchema, + CallToolRequestSchema, +} from "@modelcontextprotocol/sdk/types.js"; // Try to import SDK-dependent modules let createMCPServer: typeof import("./mcp-server.js").createMCPServer; @@ -134,3 +138,209 @@ describe.skipIf(sdkLoadError !== null)("MCP Server Unit Tests", () => { }); }); +// Tests for list_indexes tool and discovery vs fixed mode +describe.skipIf(sdkLoadError !== null || !hasApiCredentials)( + "list_indexes tool and discovery mode", + () => { + describe("list_indexes tool", () => { + it("returns available indexes with metadata", async () => { + const mockState = createMockState(); + const store = createMockStore(mockState); + + const server = await createMCPServer({ + store, + indexNames: ["test-key"], + }); + + // Get the ListToolsRequestSchema handler + const listToolsHandler = (server as any).requestHandlers.get( + ListToolsRequestSchema + ); + expect(listToolsHandler).toBeDefined(); + + // Call the handler to get tools + const result = await listToolsHandler(); + const listIndexesTool = result.tools.find( + (t: any) => t.name === "list_indexes" + ); + expect(listIndexesTool).toBeDefined(); + expect(listIndexesTool.description).toContain("available indexes"); + }); + + it("returns 'No indexes available' message when empty in discovery mode", async () => { + const store = createMockStore(null); + // Mock store.list() to return empty array for discovery mode + store.list = vi.fn().mockResolvedValue([]); + + const server = await createMCPServer({ + store, + indexNames: [], + discovery: true, + }); + + // Get the CallToolRequestSchema handler + const callToolHandler = (server as any).requestHandlers.get( + CallToolRequestSchema + ); + expect(callToolHandler).toBeDefined(); + + // Call list_indexes + const result = await callToolHandler({ + params: { + name: "list_indexes", + arguments: {}, + }, + }); + + expect(result.content[0].text).toContain("No indexes available"); + }); + + it("handles errors gracefully and returns isError: true", async () => { + const store = createMockStore(createMockState()); + // Mock store.list() to throw an error + store.list = vi.fn().mockRejectedValue(new Error("Store error")); + + const server = await createMCPServer({ + store, + indexNames: ["test-key"], + discovery: true, + }); + + // Get the CallToolRequestSchema handler + const callToolHandler = (server as any).requestHandlers.get( + CallToolRequestSchema + ); + + // Call list_indexes + const result = await callToolHandler({ + params: { + name: "list_indexes", + arguments: {}, + }, + }); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("Error listing indexes"); + }); + }); + + describe("discovery vs fixed mode", () => { + it("fixed mode includes enum in tool schemas for index_name", async () => { + const mockState = createMockState(); + const store = createMockStore(mockState); + + const server = await createMCPServer({ + store, + indexNames: ["test-key"], + discovery: false, // Fixed mode + }); + + // Get the ListToolsRequestSchema handler + const listToolsHandler = (server as any).requestHandlers.get( + ListToolsRequestSchema + ); + const result = await listToolsHandler(); + + // Check search tool has enum + const searchTool = result.tools.find((t: any) => t.name === "search"); + expect(searchTool.inputSchema.properties.index_name.enum).toBeDefined(); + expect(searchTool.inputSchema.properties.index_name.enum).toContain( + "test-key" + ); + + // Check list_files tool has enum (if present) + const listFilesTool = result.tools.find( + (t: any) => t.name === "list_files" + ); + if (listFilesTool) { + expect(listFilesTool.inputSchema.properties.index_name.enum).toBeDefined(); + expect(listFilesTool.inputSchema.properties.index_name.enum).toContain( + "test-key" + ); + } + + // Check read_file tool has enum (if present) + const readFileTool = result.tools.find((t: any) => t.name === "read_file"); + if (readFileTool) { + expect(readFileTool.inputSchema.properties.index_name.enum).toBeDefined(); + expect(readFileTool.inputSchema.properties.index_name.enum).toContain( + "test-key" + ); + } + }); + + it("discovery mode does NOT include enum in tool schemas", async () => { + const mockState = createMockState(); + const store = createMockStore(mockState); + + const server = await createMCPServer({ + store, + indexNames: ["test-key"], + discovery: true, // Discovery mode + }); + + // Get the ListToolsRequestSchema handler + const listToolsHandler = (server as any).requestHandlers.get( + ListToolsRequestSchema + ); + const result = await listToolsHandler(); + + // Check search tool does NOT have enum + const searchTool = result.tools.find((t: any) => t.name === "search"); + expect(searchTool.inputSchema.properties.index_name.enum).toBeUndefined(); + + // Check list_files tool does NOT have enum (if present) + const listFilesTool = result.tools.find( + (t: any) => t.name === "list_files" + ); + if (listFilesTool) { + expect(listFilesTool.inputSchema.properties.index_name.enum).toBeUndefined(); + } + + // Check read_file tool does NOT have enum (if present) + const readFileTool = result.tools.find((t: any) => t.name === "read_file"); + if (readFileTool) { + expect(readFileTool.inputSchema.properties.index_name.enum).toBeUndefined(); + } + }); + + it("list_indexes tool is available in both fixed and discovery modes", async () => { + const mockState = createMockState(); + const store = createMockStore(mockState); + + // Test fixed mode + const fixedServer = await createMCPServer({ + store, + indexNames: ["test-key"], + discovery: false, + }); + + const fixedListToolsHandler = (fixedServer as any).requestHandlers.get( + ListToolsRequestSchema + ); + const fixedResult = await fixedListToolsHandler(); + const fixedListIndexesTool = fixedResult.tools.find( + (t: any) => t.name === "list_indexes" + ); + expect(fixedListIndexesTool).toBeDefined(); + + // Test discovery mode + const discoveryServer = await createMCPServer({ + store, + indexNames: ["test-key"], + discovery: true, + }); + + const discoveryListToolsHandler = (discoveryServer as any).requestHandlers.get( + ListToolsRequestSchema + ); + const discoveryResult = await discoveryListToolsHandler(); + const discoveryListIndexesTool = discoveryResult.tools.find( + (t: any) => t.name === "list_indexes" + ); + expect(discoveryListIndexesTool).toBeDefined(); + }); + }); + } +); + diff --git a/src/clients/mcp-server.ts b/src/clients/mcp-server.ts index cd5151e..ac9d32a 100644 --- a/src/clients/mcp-server.ts +++ b/src/clients/mcp-server.ts @@ -37,21 +37,23 @@ import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js"; -import type { IndexStoreReader } from "../stores/types.js"; +import type { IndexStoreReader, IndexStore } from "../stores/types.js"; +import type { Source } from "../sources/types.js"; import { MultiIndexRunner } from "./multi-index-runner.js"; import { buildClientUserAgent, type MCPClientInfo } from "../core/utils.js"; import { SEARCH_DESCRIPTION, LIST_FILES_DESCRIPTION, READ_FILE_DESCRIPTION, + withListIndexesReference, withIndexList, } from "./tool-descriptions.js"; /** * Configuration for the MCP server. */ export interface MCPServerConfig { - /** Store to load indexes from */ - store: IndexStoreReader; + /** Store to load indexes from (accepts both reader-only and full store) */ + store: IndexStoreReader | IndexStore; /** * Index names to expose. If undefined, all indexes in the store are exposed. */ @@ -71,6 +73,13 @@ export interface MCPServerConfig { * @default "0.1.0" */ version?: string; + /** + * Discovery mode flag. + * When true: use withListIndexesReference (no enum in schemas), dynamic index list + * When false/undefined: use withIndexList (include enum in schemas), static index list + * @default false + */ + discovery?: boolean; } /** * Create an MCP server instance. @@ -97,13 +106,15 @@ export async function createMCPServer( // Create shared runner for multi-index operations // Build User-Agent for analytics tracking const clientUserAgent = buildClientUserAgent("mcp"); - + const runner = await MultiIndexRunner.create({ store: config.store, indexNames: config.indexNames, searchOnly: config.searchOnly, clientUserAgent, + discovery: config.discovery, }); + const { indexNames, indexes } = runner; const searchOnly = !runner.hasFileOperations(); // Format index list for tool descriptions @@ -146,13 +157,35 @@ export async function createMCPServer( required?: string[]; }; }; - // Tool descriptions with available indexes (from shared module) - const searchDescription = withIndexList(SEARCH_DESCRIPTION, indexListStr); - const listFilesDescription = withIndexList(LIST_FILES_DESCRIPTION, indexListStr); - const readFileDescription = withIndexList(READ_FILE_DESCRIPTION, indexListStr); + + // Tool descriptions: use enum in fixed mode, reference in discovery mode + let searchDescription: string; + let listFilesDescription: string; + let readFileDescription: string; + + if (config.discovery) { + // Discovery mode: use reference to list_indexes (no enum) + searchDescription = withListIndexesReference(SEARCH_DESCRIPTION); + listFilesDescription = withListIndexesReference(LIST_FILES_DESCRIPTION); + readFileDescription = withListIndexesReference(READ_FILE_DESCRIPTION); + } else { + // Fixed mode: include enum with index list + searchDescription = withIndexList(SEARCH_DESCRIPTION, indexListStr); + listFilesDescription = withIndexList(LIST_FILES_DESCRIPTION, indexListStr); + readFileDescription = withIndexList(READ_FILE_DESCRIPTION, indexListStr); + } // List available tools server.setRequestHandler(ListToolsRequestSchema, async () => { const tools: Tool[] = [ + { + name: "list_indexes", + description: "List all available indexes with their metadata. Call this to discover what indexes are available before using search, list_files, or read_file tools.", + inputSchema: { + type: "object", + properties: {}, + required: [], + }, + }, { name: "search", description: searchDescription, @@ -162,7 +195,7 @@ export async function createMCPServer( index_name: { type: "string", description: "Name of the index to search.", - enum: indexNames, + ...(config.discovery ? {} : { enum: runner.indexes.map(i => i.name) }), }, query: { type: "string", @@ -189,7 +222,7 @@ export async function createMCPServer( index_name: { type: "string", description: "Name of the index.", - enum: indexNames, + ...(config.discovery ? {} : { enum: runner.indexes.map(i => i.name) }), }, directory: { type: "string", @@ -220,7 +253,7 @@ export async function createMCPServer( index_name: { type: "string", description: "Name of the index.", - enum: indexNames, + ...(config.discovery ? {} : { enum: runner.indexes.map(i => i.name) }), }, path: { type: "string", @@ -261,6 +294,31 @@ export async function createMCPServer( // Handle tool calls server.setRequestHandler(CallToolRequestSchema, async (request) => { const { name, arguments: args } = request.params; + + // Handle list_indexes separately (no index_name required) + if (name === "list_indexes") { + try { + await runner.refreshIndexList(); + const { indexes } = runner; + if (indexes.length === 0) { + return { + content: [{ type: "text", text: "No indexes available. Use `ctxc index` CLI to create one." }], + }; + } + const lines = indexes.map((i) => + `- ${i.name} (${i.type}://${i.identifier}) - synced ${i.syncedAt}` + ); + return { + content: [{ type: "text", text: `Available indexes:\n${lines.join("\n")}` }], + }; + } catch (error) { + return { + content: [{ type: "text", text: `Error listing indexes: ${error}` }], + isError: true, + }; + } + } + try { const indexName = args?.index_name as string; const client = await runner.getClient(indexName); diff --git a/src/clients/multi-index-runner.test.ts b/src/clients/multi-index-runner.test.ts index 8bea9a0..c4f8340 100644 --- a/src/clients/multi-index-runner.test.ts +++ b/src/clients/multi-index-runner.test.ts @@ -1,18 +1,17 @@ /** - * Tests for createSourceFromState + * Tests for MultiIndexRunner * - * These tests verify that createSourceFromState correctly uses resolvedRef + * Tests for createSourceFromState verify that it correctly uses resolvedRef * from state metadata when creating source instances. * - * We mock GitHub and Website sources to capture what config gets passed - * to the constructors, without needing API credentials. - * - * Since all VCS sources (GitHub, GitLab, BitBucket) use the same getRef() logic, - * we only test GitHub as the representative case. + * Tests for MultiIndexRunner.refreshIndexList verify that it respects + * the fixed mode allowlist when refreshing the index list. */ import { describe, it, expect, vi, beforeEach } from "vitest"; -import type { IndexStateSearchOnly, SourceMetadata } from "../core/types.js"; +import type { IndexStateSearchOnly, SourceMetadata, IndexState } from "../core/types.js"; +import type { IndexStoreReader } from "../stores/types.js"; +import { MultiIndexRunner } from "./multi-index-runner.js"; // Mock only the sources we actually test vi.mock("../sources/github.js", () => ({ @@ -113,3 +112,176 @@ describe("createSourceFromState", () => { ); }); }); + +describe("MultiIndexRunner.refreshIndexList", () => { + // Helper to create a mock store with multiple indexes + const createMockStoreWithIndexes = (indexNames: string[]): IndexStoreReader => { + const mockState = (name: string): IndexState => ({ + version: 1, + contextState: { version: 1 } as any, + source: { + type: "github", + config: { owner: "test", repo: name }, + syncedAt: new Date().toISOString(), + }, + }); + + return { + loadState: vi.fn(), + loadSearch: vi.fn().mockImplementation((name: string) => { + if (indexNames.includes(name)) { + return Promise.resolve(mockState(name)); + } + return Promise.resolve(null); + }), + list: vi.fn().mockResolvedValue(indexNames), + }; + }; + + it("in discovery mode, refreshIndexList includes all indexes from store", async () => { + const store = createMockStoreWithIndexes(["pytorch", "react", "docs"]); + + const runner = await MultiIndexRunner.create({ + store, + // No indexNames = discovery mode + }); + + expect(runner.indexNames).toEqual(["pytorch", "react", "docs"]); + + // Simulate store gaining a new index + (store.list as any).mockResolvedValue(["pytorch", "react", "docs", "vue"]); + (store.loadSearch as any).mockImplementation((name: string) => { + if (["pytorch", "react", "docs", "vue"].includes(name)) { + return Promise.resolve({ + version: 1, + contextState: { version: 1 } as any, + source: { + type: "github", + config: { owner: "test", repo: name }, + syncedAt: new Date().toISOString(), + }, + }); + } + return Promise.resolve(null); + }); + + await runner.refreshIndexList(); + expect(runner.indexNames).toEqual(["pytorch", "react", "docs", "vue"]); + }); + + it("in fixed mode, refreshIndexList respects the original allowlist", async () => { + const store = createMockStoreWithIndexes(["pytorch", "react", "docs"]); + + // Create runner in fixed mode with only pytorch and react + const runner = await MultiIndexRunner.create({ + store, + indexNames: ["pytorch", "react"], + }); + + expect(runner.indexNames).toEqual(["pytorch", "react"]); + + // Simulate store gaining a new index (docs is already there, vue is new) + (store.list as any).mockResolvedValue(["pytorch", "react", "docs", "vue"]); + (store.loadSearch as any).mockImplementation((name: string) => { + if (["pytorch", "react", "docs", "vue"].includes(name)) { + return Promise.resolve({ + version: 1, + contextState: { version: 1 } as any, + source: { + type: "github", + config: { owner: "test", repo: name }, + syncedAt: new Date().toISOString(), + }, + }); + } + return Promise.resolve(null); + }); + + await runner.refreshIndexList(); + + // Should still only include pytorch and react, not docs or vue + expect(runner.indexNames).toEqual(["pytorch", "react"]); + }); + + it("in fixed mode, refreshIndexList is a no-op even when indexes are deleted", async () => { + const store = createMockStoreWithIndexes(["pytorch", "react"]); + + // Create runner in fixed mode with pytorch and react + const runner = await MultiIndexRunner.create({ + store, + indexNames: ["pytorch", "react"], + }); + + expect(runner.indexNames).toEqual(["pytorch", "react"]); + + // Simulate pytorch being deleted from store + (store.list as any).mockResolvedValue(["react"]); + (store.loadSearch as any).mockImplementation((name: string) => { + if (name === "react") { + return Promise.resolve({ + version: 1, + contextState: { version: 1 } as any, + source: { + type: "github", + config: { owner: "test", repo: name }, + syncedAt: new Date().toISOString(), + }, + }); + } + return Promise.resolve(null); + }); + + await runner.refreshIndexList(); + + // In fixed mode, the list should remain unchanged even though pytorch was deleted + expect(runner.indexNames).toEqual(["pytorch", "react"]); + }); + + it("in discovery mode, refreshIndexList prunes stale cache entries", async () => { + const store = createMockStoreWithIndexes(["pytorch", "react", "docs"]); + + const runner = await MultiIndexRunner.create({ + store, + // No indexNames = discovery mode + }); + + expect(runner.indexNames).toEqual(["pytorch", "react", "docs"]); + + // Manually populate cache with mock clients (avoid initialization) + const mockClient = { initialized: true } as any; + (runner as any).clientCache.set("pytorch", mockClient); + (runner as any).clientCache.set("react", mockClient); + (runner as any).clientCache.set("docs", mockClient); + + // Verify cache has all three entries + expect((runner as any).clientCache.size).toBe(3); + + // Simulate store losing the "docs" index + (store.list as any).mockResolvedValue(["pytorch", "react"]); + (store.loadSearch as any).mockImplementation((name: string) => { + if (["pytorch", "react"].includes(name)) { + return Promise.resolve({ + version: 1, + contextState: { version: 1 } as any, + source: { + type: "github", + config: { owner: "test", repo: name }, + syncedAt: new Date().toISOString(), + }, + }); + } + return Promise.resolve(null); + }); + + await runner.refreshIndexList(); + + // Index list should be updated + expect(runner.indexNames).toEqual(["pytorch", "react"]); + + // Cache should be pruned - only pytorch and react remain + expect((runner as any).clientCache.size).toBe(2); + expect((runner as any).clientCache.has("pytorch")).toBe(true); + expect((runner as any).clientCache.has("react")).toBe(true); + expect((runner as any).clientCache.has("docs")).toBe(false); + }); +}); diff --git a/src/clients/multi-index-runner.ts b/src/clients/multi-index-runner.ts index 204462c..c367421 100644 --- a/src/clients/multi-index-runner.ts +++ b/src/clients/multi-index-runner.ts @@ -7,7 +7,7 @@ * @module clients/multi-index-runner */ -import type { IndexStoreReader } from "../stores/types.js"; +import type { IndexStoreReader, IndexStore } from "../stores/types.js"; import type { Source } from "../sources/types.js"; import type { IndexStateSearchOnly } from "../core/types.js"; import { getSourceIdentifier, getResolvedRef } from "../core/types.js"; @@ -26,7 +26,7 @@ export interface IndexInfo { /** Configuration for MultiIndexRunner */ export interface MultiIndexRunnerConfig { /** Store to load indexes from */ - store: IndexStoreReader; + store: IndexStoreReader | IndexStore; /** * Index names to expose. If undefined, all indexes in the store are exposed. */ @@ -41,6 +41,13 @@ export interface MultiIndexRunnerConfig { * When provided, this is passed to SearchClient instances for API requests. */ clientUserAgent?: string; + /** + * Discovery mode flag. + * When true: allow empty index list (user can add indexes later via CLI) + * When false: require at least one valid index to be loaded + * @default false + */ + discovery?: boolean; } /** @@ -88,29 +95,32 @@ export async function createSourceFromState(state: IndexStateSearchOnly): Promis * Lazily initializes SearchClient instances as needed and caches them. */ export class MultiIndexRunner { - private readonly store: IndexStoreReader; + private readonly store: IndexStoreReader | IndexStore; private readonly searchOnly: boolean; private clientUserAgent?: string; private readonly clientCache = new Map(); + private readonly originalIndexNames: string[] | undefined; /** Available index names */ - readonly indexNames: string[]; + indexNames: string[]; /** Metadata about available indexes */ - readonly indexes: IndexInfo[]; + indexes: IndexInfo[]; private constructor( - store: IndexStoreReader, + store: IndexStoreReader | IndexStore, indexNames: string[], indexes: IndexInfo[], searchOnly: boolean, - clientUserAgent?: string + clientUserAgent?: string, + originalIndexNames?: string[] ) { this.store = store; this.indexNames = indexNames; this.indexes = indexes; this.searchOnly = searchOnly; this.clientUserAgent = clientUserAgent; + this.originalIndexNames = originalIndexNames; } /** @@ -119,21 +129,21 @@ export class MultiIndexRunner { static async create(config: MultiIndexRunnerConfig): Promise { const store = config.store; const searchOnly = config.searchOnly ?? false; + const discovery = config.discovery ?? false; // Discover available indexes const allIndexNames = await store.list(); const indexNames = config.indexNames ?? allIndexNames; + // In fixed mode, save the original allowlist for later filtering + const originalIndexNames = config.indexNames ? [...config.indexNames] : undefined; + // Validate requested indexes exist const missingIndexes = indexNames.filter((n) => !allIndexNames.includes(n)); if (missingIndexes.length > 0) { throw new Error(`Indexes not found: ${missingIndexes.join(", ")}`); } - if (indexNames.length === 0) { - throw new Error("No indexes available in store"); - } - // Load metadata for available indexes, filtering out any that fail to load const indexes: IndexInfo[] = []; const validIndexNames: string[] = []; @@ -156,17 +166,17 @@ export class MultiIndexRunner { } } - if (validIndexNames.length === 0) { - throw new Error("No valid indexes available (all indexes failed to load)"); + // In fixed mode (non-discovery), require at least one valid index + if (!discovery && validIndexNames.length === 0) { + throw new Error("No valid indexes loaded. Fixed mode requires at least one index."); } - return new MultiIndexRunner(store, validIndexNames, indexes, searchOnly, config.clientUserAgent); + return new MultiIndexRunner(store, validIndexNames, indexes, searchOnly, config.clientUserAgent, originalIndexNames); } - /** * Update the User-Agent string. - * + * * Call this after receiving MCP client info to include the client name/version. * Note: Only affects future client creations, not existing cached clients. */ @@ -205,6 +215,64 @@ export class MultiIndexRunner { return client; } + /** + * Refresh the list of available indexes from the store. + * Call after adding or removing indexes. + * + * In fixed mode (when originalIndexNames is set), this is a no-op. + * The list is completely static and never changes. + * + * In discovery mode, refreshes from the store to pick up new/deleted indexes. + */ + async refreshIndexList(): Promise { + // In fixed mode, the list is static - don't refresh + if (this.originalIndexNames) { + return; + } + + // Discovery mode: refresh from store + const allIndexNames = await this.store.list(); + + const newIndexes: IndexInfo[] = []; + const newIndexNames: string[] = []; + + for (const name of allIndexNames) { + try { + const state = await this.store.loadSearch(name); + if (state) { + newIndexNames.push(name); + newIndexes.push({ + name, + type: state.source.type, + identifier: getSourceIdentifier(state.source), + ref: getResolvedRef(state.source), + syncedAt: state.source.syncedAt, + }); + } + } catch { + // Skip indexes that fail to load + } + } + + this.indexNames = newIndexNames; + this.indexes = newIndexes; + + // Prune stale cache entries for removed indexes + for (const cachedName of this.clientCache.keys()) { + if (!newIndexNames.includes(cachedName)) { + this.clientCache.delete(cachedName); + } + } + } + + /** + * Invalidate cached SearchClient for an index. + * Call after updating an index to ensure fresh data on next access. + */ + invalidateClient(indexName: string): void { + this.clientCache.delete(indexName); + } + /** Check if file operations are enabled */ hasFileOperations(): boolean { return !this.searchOnly; diff --git a/src/clients/tool-descriptions.ts b/src/clients/tool-descriptions.ts index 1a74c85..7f08c90 100644 --- a/src/clients/tool-descriptions.ts +++ b/src/clients/tool-descriptions.ts @@ -65,6 +65,7 @@ NOT supported: \\d, \\s, \\w (use [0-9], [ \\t], [a-zA-Z_] instead)`; /** * Format a tool description with available indexes for multi-index mode. + * Used by CLI agent which cannot call list_indexes dynamically. */ export function withIndexList(baseDescription: string, indexListStr: string): string { return `${baseDescription} @@ -73,3 +74,13 @@ Available indexes: ${indexListStr}`; } +/** + * Format a tool description with a reference to list_indexes. + * Used by MCP server where agents can call list_indexes dynamically. + */ +export function withListIndexesReference(baseDescription: string): string { + return `${baseDescription} + +Use list_indexes to see available indexes.`; +} + diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 61b2606..89632f4 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -198,7 +198,6 @@ export class Indexer { const context = await DirectContext.create({ apiKey: this.apiKey, apiUrl: this.apiUrl, - clientUserAgent: this.clientUserAgent, }); // Fetch all files from source @@ -262,7 +261,6 @@ export class Indexer { const context = await DirectContext.import(previousState.contextState, { apiKey: this.apiKey, apiUrl: this.apiUrl, - clientUserAgent: this.clientUserAgent, }); // Remove deleted files diff --git a/src/stores/index.ts b/src/stores/index.ts index 85b992b..1879b59 100644 --- a/src/stores/index.ts +++ b/src/stores/index.ts @@ -10,6 +10,7 @@ export type { MemoryStoreConfig } from "./memory.js"; export { S3Store } from "./s3.js"; export type { S3StoreConfig } from "./s3.js"; export { CompositeStoreReader } from "./composite.js"; +export { ReadOnlyLayeredStore } from "./read-only-layered-store.js"; export { parseIndexSpec, parseIndexSpecs } from "./index-spec.js"; export type { IndexSpec } from "./index-spec.js"; diff --git a/src/stores/read-only-layered-store.test.ts b/src/stores/read-only-layered-store.test.ts new file mode 100644 index 0000000..4ebec61 --- /dev/null +++ b/src/stores/read-only-layered-store.test.ts @@ -0,0 +1,188 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { ReadOnlyLayeredStore } from "./read-only-layered-store.js"; +import { MemoryStore } from "./memory.js"; +import type { IndexState, IndexStateSearchOnly } from "../core/types.js"; + +function createMockState(): { full: IndexState; search: IndexStateSearchOnly } { + const source = { + type: "github" as const, + config: { owner: "test-owner", repo: "test-repo" }, + syncedAt: new Date().toISOString(), + }; + return { + full: { + version: 1, + contextState: { + mode: "full" as const, + checkpointId: "test-checkpoint-123", + addedBlobs: ["blob-1"], + deletedBlobs: [], + blobs: [["blob-1", "src/file1.ts"]], + }, + source, + }, + search: { + version: 1, + contextState: { + mode: "search-only" as const, + checkpointId: "test-checkpoint-123", + addedBlobs: ["blob-1"], + deletedBlobs: [], + }, + source, + }, + }; +} + +describe("ReadOnlyLayeredStore", () => { + let primary: MemoryStore; + let remote: MemoryStore; + let layered: ReadOnlyLayeredStore; + + beforeEach(() => { + primary = new MemoryStore(); + remote = new MemoryStore(); + layered = new ReadOnlyLayeredStore(primary, remote); + }); + + describe("list", () => { + it("returns merged and deduplicated list from both stores", async () => { + const { full, search } = createMockState(); + + // Add to primary + await primary.save("local-a", full, search); + await primary.save("local-b", full, search); + + // Add to remote + await remote.save("remote-a", full, search); + await remote.save("remote-b", full, search); + + const list = await layered.list(); + + expect(list).toContain("local-a"); + expect(list).toContain("local-b"); + expect(list).toContain("remote-a"); + expect(list).toContain("remote-b"); + expect(list.length).toBe(4); + }); + + it("deduplicates when same index exists in both stores", async () => { + const { full, search } = createMockState(); + + // Add same index to both + await primary.save("shared", full, search); + await remote.save("shared", full, search); + + const list = await layered.list(); + + expect(list).toContain("shared"); + expect(list.length).toBe(1); + }); + + it("returns sorted list", async () => { + const { full, search } = createMockState(); + + await primary.save("zebra", full, search); + await primary.save("apple", full, search); + await remote.save("mango", full, search); + + const list = await layered.list(); + + expect(list).toEqual(["apple", "mango", "zebra"]); + }); + + it("returns empty list when both stores are empty", async () => { + const list = await layered.list(); + expect(list).toEqual([]); + }); + }); + + describe("loadSearch", () => { + it("returns from primary if exists", async () => { + const { full, search } = createMockState(); + await primary.save("test", full, search); + + const result = await layered.loadSearch("test"); + + expect(result).toEqual(search); + }); + + it("falls back to remote if not in primary", async () => { + const { full, search } = createMockState(); + await remote.save("test", full, search); + + const result = await layered.loadSearch("test"); + + expect(result).toEqual(search); + }); + + it("prefers primary over remote when both exist", async () => { + const { full, search } = createMockState(); + const primarySearch = { + ...search, + contextState: { ...search.contextState, checkpointId: "primary" }, + }; + const remoteSearch = { + ...search, + contextState: { ...search.contextState, checkpointId: "remote" }, + }; + + await primary.save("test", full, primarySearch); + await remote.save("test", full, remoteSearch); + + const result = await layered.loadSearch("test"); + + expect(result?.contextState.checkpointId).toBe("primary"); + }); + + it("returns null if not found in either store", async () => { + const result = await layered.loadSearch("nonexistent"); + expect(result).toBeNull(); + }); + }); + + describe("loadState", () => { + it("returns from primary if exists", async () => { + const { full, search } = createMockState(); + await primary.save("test", full, search); + + const result = await layered.loadState("test"); + + expect(result).toEqual(full); + }); + + it("falls back to remote if not in primary", async () => { + const { full, search } = createMockState(); + await remote.save("test", full, search); + + const result = await layered.loadState("test"); + + expect(result).toEqual(full); + }); + + it("prefers primary over remote when both exist", async () => { + const { full, search } = createMockState(); + const primaryFull = { + ...full, + contextState: { ...full.contextState, checkpointId: "primary" }, + }; + const remoteFull = { + ...full, + contextState: { ...full.contextState, checkpointId: "remote" }, + }; + + await primary.save("test", primaryFull, search); + await remote.save("test", remoteFull, search); + + const result = await layered.loadState("test"); + + expect(result?.contextState.checkpointId).toBe("primary"); + }); + + it("returns null if not found in either store", async () => { + const result = await layered.loadState("nonexistent"); + expect(result).toBeNull(); + }); + }); +}); + diff --git a/src/stores/read-only-layered-store.ts b/src/stores/read-only-layered-store.ts new file mode 100644 index 0000000..8bbf162 --- /dev/null +++ b/src/stores/read-only-layered-store.ts @@ -0,0 +1,82 @@ +/** + * Read-only layered store that merges a primary store with a remote store. + * + * Used in discovery mode with remote indexes: + * - Primary store: FilesystemStore (local indexes) + * - Remote store: CompositeStoreReader (remote indexes) + * + * Behavior: + * - list(): Merge both lists, deduplicated, sorted + * - loadState(name): Try primary first, then remote + * - loadSearch(name): Try primary first, then remote + * - No save/delete methods (read-only) + * + * @module stores/read-only-layered-store + */ + +import type { IndexStoreReader } from "./types.js"; +import type { IndexState, IndexStateSearchOnly } from "../core/types.js"; + +/** + * Read-only layered store that combines a primary and remote store. + * + * Useful for discovery mode where users can: + * - Manage local indexes via CLI (stored in FilesystemStore) + * - Reference remote indexes via -i flags (stored in CompositeStoreReader) + * + * @example + * ```typescript + * const primary = new FilesystemStore(); + * const remote = await CompositeStoreReader.fromSpecs([ + * { type: "s3", value: "s3://bucket/shared-index", displayName: "shared" } + * ]); + * const layered = new ReadOnlyLayeredStore(primary, remote); + * + * // Lists both local and remote indexes + * const allIndexes = await layered.list(); + * + * // Tries primary first, then remote + * const state = await layered.loadSearch("my-index"); + * ``` + */ +export class ReadOnlyLayeredStore implements IndexStoreReader { + constructor( + private primary: IndexStoreReader, + private remote: IndexStoreReader + ) {} + + async loadState(key: string): Promise { + // Try primary first + const primaryState = await this.primary.loadState(key); + if (primaryState !== null) { + return primaryState; + } + // Fall back to remote + return this.remote.loadState(key); + } + + async loadSearch(key: string): Promise { + // Try primary first + const primarySearch = await this.primary.loadSearch(key); + if (primarySearch !== null) { + return primarySearch; + } + // Fall back to remote + return this.remote.loadSearch(key); + } + + async list(): Promise { + // Get both lists + const [primaryList, remoteList] = await Promise.all([ + this.primary.list(), + this.remote.list(), + ]); + + // Merge and deduplicate + const merged = new Set([...primaryList, ...remoteList]); + + // Return sorted array + return Array.from(merged).sort(); + } +} +