diff --git a/.env b/.env new file mode 100644 index 00000000..2f5c4837 --- /dev/null +++ b/.env @@ -0,0 +1,244 @@ +# Environment for local Qdrant + MCP server (SSE) +# Qdrant (vector DB) runs as docker service "qdrant". The MCP server connects to it internally. +QDRANT_URL=http://qdrant:6333 +# QDRANT_API_KEY= # not needed for local + +# Repository mode: 0=single-repo (default), 1=multi-repo +# Single-repo: All files go into one collection (COLLECTION_NAME) +# Multi-repo: Each subdirectory gets its own collection +MULTI_REPO_MODE=0 + +# Single unified collection for seamless cross-repo search +# Default: "codebase" - all your code in one collection for unified search +# This enables searching across multiple repos/workspaces without fragmentation +COLLECTION_NAME=codebase + +# Embedding settings (FastEmbed model) +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +EMBEDDING_PROVIDER=fastembed + +# FastMCP server settings (SSE transport) +FASTMCP_HOST=0.0.0.0 +FASTMCP_PORT=8000 +FASTMCP_INDEXER_PORT=8001 + +# Optional: customize tool descriptions (uncomment to use) +TOOL_STORE_DESCRIPTION="Store reusable code snippets for later retrieval. The 'information' is a clear NL description; include the actual code in 'metadata.code' and add 'metadata.language' (e.g., python, typescript) and 'metadata.path' when known. Use this whenever you generate or refine a code snippet." +TOOL_FIND_DESCRIPTION="Search for relevant code snippets using multiple phrasings of the query (multi-query). Prefer results where metadata.language matches the target file and metadata.path is relevant. You may pass optional filters (language, path_prefix, kind) which the server applies server-side. Include 'metadata.code', 'metadata.path', and 'metadata.language' in responses." + + + +# Optional: local cross-encoder reranker (ONNX) +# Baked into Docker image at /app/models/ - no host mount needed +RERANKER_ONNX_PATH=/app/models/reranker.onnx +RERANKER_TOKENIZER_PATH=/app/models/tokenizer.json + +# Reranker toggles and tuning +# Increased TOPN and RETURN_M for better final ranking +RERANKER_ENABLED=1 +RERANKER_TOPN=100 +RERANKER_RETURN_M=20 +RERANKER_TIMEOUT_MS=3000 + +# Safety: minimum rerank timeout floor (ms) to avoid cold-start timeouts +RERANK_TIMEOUT_FLOOR_MS=1000 + +# Optional warmups (disabled by default) +EMBEDDING_WARMUP=0 +RERANK_WARMUP=0 + + +# In-process execution (faster; falls back to subprocess on failure) +HYBRID_IN_PROCESS=1 +RERANK_IN_PROCESS=1 + + +# Query Optimization (adaptive HNSW_EF tuning for 2x faster simple queries) +QUERY_OPTIMIZER_ADAPTIVE=1 +QUERY_OPTIMIZER_MIN_EF=64 +QUERY_OPTIMIZER_MAX_EF=512 +QUERY_OPTIMIZER_SIMPLE_THRESHOLD=0.3 +QUERY_OPTIMIZER_COMPLEX_THRESHOLD=0.7 +QUERY_OPTIMIZER_SIMPLE_FACTOR=0.5 +QUERY_OPTIMIZER_SEMANTIC_FACTOR=1.0 +QUERY_OPTIMIZER_COMPLEX_FACTOR=2.0 +QUERY_OPTIMIZER_DENSE_THRESHOLD=0.2 +QUERY_OPTIMIZER_COLLECTION_SIZE=10000 +QDRANT_EF_SEARCH=128 + +# AST-based code understanding (semantic chunking for 20-30% better precision) +USE_TREE_SITTER=1 +INDEX_USE_ENHANCED_AST=1 +INDEX_SEMANTIC_CHUNKS=1 + +# Hybrid/rerank defaults +HYBRID_EXPAND=0 +HYBRID_PER_PATH=1 +HYBRID_SYMBOL_BOOST=0.35 +HYBRID_RECENCY_WEIGHT=0.1 +RERANK_EXPAND=1 + + +# Memory integration (SSE + Qdrant) +MEMORY_SSE_ENABLED=true +MEMORY_MCP_URL=http://mcp:8000/sse +MEMORY_MCP_TIMEOUT=6 + + +# Local LLM expansion via Ollama (mini model) +LLM_PROVIDER=ollama +OLLAMA_HOST=http://host.docker.internal:11434 +LLM_EXPAND_MODEL=phi3:mini +LLM_EXPAND_MAX=0 +# PRF defaults (enabled by default) +PRF_ENABLED=1 + + +# ReFRAG mode: compact gating + micro-chunking +REFRAG_MODE=1 +MINI_VECTOR_NAME=mini +MINI_VEC_DIM=64 +MINI_VEC_SEED=1337 +HYBRID_MINI_WEIGHT=1.0 +# Micro-chunking controls (token-based) +# Smaller chunks = more precise retrieval (8 tokens ~= 2-3 lines of code) +INDEX_MICRO_CHUNKS=0 +MICRO_CHUNK_TOKENS=24 +MICRO_CHUNK_STRIDE=48 +# Optional: gate-first using mini vectors to prefilter dense search +REFRAG_GATE_FIRST=1 +REFRAG_CANDIDATES=200 + +# Lexical vector settings +LEX_VECTOR_NAME=lex +LEX_VECTOR_DIM=4096 +# Multi-hash: each token hashes to N buckets (reduces collision impact) +LEX_MULTI_HASH=5 +# Bigrams: off (original settings) +LEX_BIGRAMS=0 +LEX_BIGRAM_WEIGHT=0.7 +# Sparse lexical vectors: disabled +LEX_SPARSE_MODE=0 +LEX_SPARSE_NAME=lex_sparse + +# Output shaping for micro spans +# Conservative settings +MICRO_OUT_MAX_SPANS=3 +MICRO_MERGE_LINES=4 +MICRO_BUDGET_TOKENS=1500 +MICRO_TOKENS_PER_LINE=32 + + +# Answer shaping (enforce concise responses from context_answer) +CTX_SUMMARY_CHARS=0 +CTX_SNIPPET_CHARS=400 + + +# Decoder-path ReFRAG +REFRAG_DECODER=1 +REFRAG_RUNTIME=llamacpp +REFRAG_ENCODER_MODEL=BAAI/bge-base-en-v1.5 +REFRAG_PHI_PATH=/work/models/refrag_phi_768_to_dmodel.bin +REFRAG_SENSE=heuristic +REFRAG_PSEUDO_DESCRIBE=1 + +# Llama.cpp sidecar (optional) +# Use docker network hostname from containers; localhost remains ok for host-side runs if LLAMACPP_URL not exported +LLAMACPP_URL=http://host.docker.internal:8081 +LLAMACPP_TIMEOUT_SEC=300 +DECODER_MAX_TOKENS=4000 +REFRAG_DECODER_MODE=prompt # prompt|soft +REFRAG_COMMIT_DESCRIBE=1 + +REFRAG_SOFT_SCALE=1.0 +LLAMACPP_USE_GPU=1 +LLAMACPP_GPU_LAYERS=32 +LLAMACPP_THREADS=6 +LLAMACPP_GPU_SPLIT= +LLAMACPP_EXTRA_ARGS= + + +# Operational safeguards and timeouts +MAX_MICRO_CHUNKS_PER_FILE=200 +QDRANT_TIMEOUT=20 +MEMORY_AUTODETECT=1 +MEMORY_COLLECTION_TTL_SECS=300 +SMART_SYMBOL_REINDEXING=1 +MAX_CHANGED_SYMBOLS_RATIO=0.6 + + +# Watcher-safe defaults (recommended) +# These are applied to the watcher service via docker-compose.yml. +# Uncomment to apply globally if you are not using compose overrides. +# QDRANT_TIMEOUT=60 +# MAX_MICRO_CHUNKS_PER_FILE=200 +# INDEX_UPSERT_BATCH=128 +# INDEX_UPSERT_RETRIES=5 +# INDEX_UPSERT_BACKOFF=0.5 +WATCH_DEBOUNCE_SECS=4 + + +# Duplicate Streamable HTTP MCP instances (run alongside SSE) +FASTMCP_HTTP_TRANSPORT=http +FASTMCP_HTTP_PORT=8002 +FASTMCP_HTTP_HEALTH_PORT=18002 +FASTMCP_INDEXER_HTTP_PORT=8003 +FASTMCP_INDEXER_HTTP_HEALTH_PORT=18003 + + +MAX_EMBED_CACHE=16384 +HYBRID_RESULTS_CACHE=128 +HYBRID_RESULTS_CACHE_ENABLED=1 +INDEX_CHUNK_LINES=60 +INDEX_CHUNK_OVERLAP=10 +USE_GPU_DECODER=0 + +# Development Remote Upload Configuration +HOST_INDEX_PATH=./dev-workspace + +# Cross-Codebase Isolation (multi-repo search scoping) +# When enabled, search results are automatically filtered to the current repo +REPO_AUTO_FILTER=1 +# Explicitly set current repo (overrides auto-detection from git/directory) +# CURRENT_REPO= + +# Post-rerank symbol boost: ensures exact symbol matches rank highest +POST_RERANK_SYMBOL_BOOST=1.0 +# Rerank blend weight: ratio of rerank score vs fusion score (0.0-1.0) +RERANK_BLEND_WEIGHT=0.6 + +# info_request() tool settings (simplified codebase retrieval) +INFO_REQUEST_LIMIT=10 +INFO_REQUEST_CONTEXT_LINES=5 +# INFO_REQUEST_EXPLAIN_DEFAULT=0 +# INFO_REQUEST_RELATIONSHIPS=0 + +# TOON output format (Token-Oriented Object Notation) +# When enabled, search results use compact TOON encoding to reduce token usage +TOON_ENABLED=0 + +# GLM (ZhipuAI) decoder backend +GLM_API_KEY= +GLM_API_BASE=https://api.z.ai/api/coding/paas/v4/ +GLM_MODEL=glm-4.6 +GLM_MODEL_FAST=glm-4.5 + +# Parallel pseudo-tag generation (indexing speedup) +# Set to 4 for 4x parallel GLM calls during indexing; 1 = sequential (default) +PSEUDO_BATCH_CONCURRENCY=4 + +# MiniMax M2 decoder backend +MINIMAX_API_KEY= +MINIMAX_API_BASE=https://api.minimax.io/v1 +MINIMAX_MODEL=MiniMax-M2 + +COMMIT_VECTOR_SEARCH=0 +STRICT_MEMORY_RESTORE=1 +CTXCE_AUTH_ENABLED=0 +CTXCE_AUTH_ADMIN_TOKEN=change-me-admin-token + +# MCP_INDEXER_URL=http://127.0.0.1:30810/mcp +# Multi-hash: each token hashes to N buckets (reduces collision impact) + +REMOTE_UPLOAD_GIT_MAX_COMMITS=500 + diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..36fd1792 --- /dev/null +++ b/.env.example @@ -0,0 +1,496 @@ +# Qdrant connection +QDRANT_URL=http://localhost:6333 +# QDRANT_API_KEY= + +# Multi-repo mode: 0=single-repo (default), 1=multi-repo +# Single-repo: All files go into one collection (COLLECTION_NAME) +# Multi-repo: Each subdirectory gets its own collection +MULTI_REPO_MODE=0 + +# Logical repo reuse (experimental): 0=disabled (default), 1=enable logical_repo_id-based +# collection reuse across git worktrees / clones. When enabled, indexer, watcher, and +# upload service will try to reuse a canonical collection per logical repository and +# use (repo_id + repo_rel_path) for skip-unchanged across worktrees. +#LOGICAL_REPO_REUSE=0 + +# Single unified collection for seamless cross-repo search (default: "codebase") +# Leave unset or use "codebase" for unified search across all your code +COLLECTION_NAME=codebase + + +# Embeddings +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +EMBEDDING_PROVIDER=fastembed +# Optional repo tag attached to each payload +REPO_NAME=workspace + +# Qwen3-Embedding Feature Flag (optional, experimental) +# Enable to use Qwen3-Embedding-0.6B instead of BGE-base (requires reindex) +# QWEN3_EMBEDDING_ENABLED=0 +# When enabled, set EMBEDDING_MODEL to the Qwen3 model: +# EMBEDDING_MODEL=electroglyph/Qwen3-Embedding-0.6B-onnx-uint8 +# Add instruction prefix to search queries (recommended for Qwen3) +# QWEN3_QUERY_INSTRUCTION=1 +# Custom instruction text (default: code search instruction) +# QWEN3_INSTRUCTION_TEXT=Instruct: Given a code search query, retrieve relevant code snippets\nQuery: + +# Cross-Codebase Isolation (multi-repo search scoping) +# When enabled, search results are automatically filtered to the current repo +# Disable (=0) to search all repos by default (legacy behavior) +REPO_AUTO_FILTER=1 +# Explicitly set current repo (overrides auto-detection from git/directory) +# CURRENT_REPO=my-project + +# MCP servers (SSE) +FASTMCP_HOST=0.0.0.0 +FASTMCP_PORT=8000 # search/store MCP (mcp-server-qdrant) +FASTMCP_INDEXER_PORT=8001 # companion indexer MCP (index/prune/list) +FASTMCP_SERVER_NAME=qdrant-mcp +# MCP_MAX_LOG_TAIL=4000 # Max chars for subprocess stdout/stderr tail (default: 4000) + +# Logging configuration +# LOG_LEVEL controls verbosity: DEBUG, INFO, WARNING, ERROR, CRITICAL +LOG_LEVEL=INFO + +# Transport: sse (default), http (streamable), or stdio +FASTMCP_TRANSPORT=sse + + +# Optional duplicate Streamable HTTP instances (run alongside SSE) +# Use these in docker-compose overrides to expose /mcp/ on separate ports. +FASTMCP_HTTP_TRANSPORT=http +FASTMCP_HTTP_PORT=8002 +FASTMCP_HTTP_HEALTH_PORT=18002 +FASTMCP_INDEXER_HTTP_PORT=8003 +FASTMCP_INDEXER_HTTP_HEALTH_PORT=18003 +# MCP_INDEXER_URL=http://localhost:8003/mcp + + +# Reranker settings (FastEmbed model - recommended) +# Set RERANKER_MODEL to enable cross-encoder reranking (auto-downloads ONNX model) +# Popular models: jinaai/jina-reranker-v2-base-multilingual, BAAI/bge-reranker-base +# RERANKER_MODEL=jinaai/jina-reranker-v2-base-multilingual + +# Legacy: manual ONNX paths (alternative to RERANKER_MODEL) +# Use these if you have a custom ONNX model or want explicit control +# RERANKER_ONNX_PATH=/work/models/model_qint8_avx512_vnni.onnx +# RERANKER_TOKENIZER_PATH=/work/models/tokenizer.json + +# Enable reranking in the indexer MCP search path +RERANKER_ENABLED=0 +# Tuning knobs (effective when enabled) +RERANKER_TOPN=50 +RERANKER_RETURN_M=12 +RERANKER_TIMEOUT_MS=2000 + +# Post-rerank symbol boost: ensures exact symbol matches rank highest even when +# the neural reranker disagrees. Applied after rerank blending as a direct score addition. +# Set to 0 to disable, >1.0 to boost symbol matches more aggressively. +POST_RERANK_SYMBOL_BOOST=1.0 +# Rerank blend weight: ratio of rerank score vs fusion score (0.0-1.0) +# Higher = more weight on neural reranker, lower = more weight on lexical/symbol boosts +RERANK_BLEND_WEIGHT=0.6 + +# Safety: minimum rerank timeout floor (ms) to avoid cold-start timeouts +RERANK_TIMEOUT_FLOOR_MS=1000 + +# Optional warmups (disabled by default) +EMBEDDING_WARMUP=0 +RERANK_WARMUP=0 + + +# In-process execution (faster; falls back to subprocess on failure) +HYBRID_IN_PROCESS=1 +RERANK_IN_PROCESS=1 + +# Parallel dense query execution (for multi-query searches) +# Queries >= threshold are executed in parallel using thread pool +# Set higher to avoid thread overhead on small query counts; set 1 to always parallelize +PARALLEL_DENSE_QUERIES=1 +PARALLEL_DENSE_THRESHOLD=4 + +# LLM query expansion (uses existing decoder infrastructure) +# Set REFRAG_RUNTIME=glm|minimax|llamacpp to choose runtime (defaults to llamacpp) +# Requires REFRAG_DECODER=1 for llama.cpp path +# EXPAND_MAX_TOKENS=512 # Max tokens for expansion response (default: 512) + +# PRF (Pseudo-Relevance Feedback) - enabled by default +# Mines top terms from search results for a refined second pass +# PRF_ENABLED=1 +# PRF_TOP_DOCS=8 +# PRF_MAX_TERMS=6 + +# Semantic expansion settings (synonym/related term expansion for better recall) +SEMANTIC_EXPANSION_ENABLED=1 +SEMANTIC_EXPANSION_TOP_K=5 +SEMANTIC_EXPANSION_SIMILARITY_THRESHOLD=0.7 +SEMANTIC_EXPANSION_MAX_TERMS=3 +SEMANTIC_EXPANSION_CACHE_SIZE=1000 +SEMANTIC_EXPANSION_CACHE_TTL=3600 +# Hybrid search tuning +# HYBRID_EXPAND=0 +# HYBRID_PER_PATH=2 +# HYBRID_SYMBOL_BOOST=0.35 +# HYBRID_RECENCY_WEIGHT=0.1 +# RERANK_EXPAND=1 + +# Caching (embeddings and search results) +# MAX_EMBED_CACHE=16384 +# HYBRID_RESULTS_CACHE=128 +# HYBRID_RESULTS_CACHE_ENABLED=1 + +# Query Optimization (adaptive HNSW_EF tuning for 2x faster simple queries) +QUERY_OPTIMIZER_ADAPTIVE=1 +QUERY_OPTIMIZER_MIN_EF=64 +QUERY_OPTIMIZER_MAX_EF=512 +QUERY_OPTIMIZER_SIMPLE_THRESHOLD=0.3 +QUERY_OPTIMIZER_COMPLEX_THRESHOLD=0.7 +QUERY_OPTIMIZER_SIMPLE_FACTOR=0.5 +QUERY_OPTIMIZER_SEMANTIC_FACTOR=1.0 +QUERY_OPTIMIZER_COMPLEX_FACTOR=2.0 +QUERY_OPTIMIZER_DENSE_THRESHOLD=0.2 +QUERY_OPTIMIZER_COLLECTION_SIZE=10000 +QDRANT_EF_SEARCH=128 + +# AST-based code understanding (semantic chunking for 20-30% better precision) +USE_TREE_SITTER=1 +INDEX_USE_ENHANCED_AST=1 +INDEX_SEMANTIC_CHUNKS=1 + +# Pattern Search - structural code similarity across languages +# Enables pattern_search MCP tool and indexes 64-dim pattern vectors +# Uses WL graph kernel, CFG fingerprints, SimHash, spectral features +# PATTERN_VECTORS=1 + +# Indexer scaling and exclusions +# QDRANT_DEFAULT_EXCLUDES=0 +# QDRANT_IGNORE_FILE=.qdrantignore +# QDRANT_EXCLUDES=tokenizer.json,*.onnx,/vendor +# INDEX_CHUNK_LINES=120 +# INDEX_CHUNK_OVERLAP=20 +# INDEX_BATCH_SIZE=64 +# INDEX_PROGRESS_EVERY=200 + + +# ReFRAG mode (optional): compact gating + micro-chunking +# Enable to add a 64-dim mini vector for fast gating and use token-based micro-chunks +REFRAG_MODE=0 +MINI_VECTOR_NAME=mini +MINI_VEC_DIM=64 +MINI_VEC_SEED=1337 +HYBRID_MINI_WEIGHT=0.5 +# Micro-chunking controls (token-based) +INDEX_MICRO_CHUNKS=0 +MICRO_CHUNK_TOKENS=16 +MICRO_CHUNK_STRIDE=8 +# Optional: gate-first using mini vectors to prefilter dense search +REFRAG_GATE_FIRST=0 +REFRAG_CANDIDATES=200 + +# Lexical vector settings (v2: multi-hash + bigrams for better keyword matching) +LEX_VECTOR_NAME=lex +LEX_VECTOR_DIM=2048 +# Multi-hash: each token hashes to N buckets (more = better collision resistance, default 3) +LEX_MULTI_HASH=3 +# Bigrams: enable bigram hashing for phrase matching (default on) +LEX_BIGRAMS=1 +# Weight for bigram entries relative to unigrams +LEX_BIGRAM_WEIGHT=0.7 +# Sparse lexical vectors: lossless exact matching (no hash collisions) +# When enabled, stores sparse vectors alongside dense for exact term matching +LEX_SPARSE_MODE=0 +LEX_SPARSE_NAME=lex_sparse + +# Output shaping for micro spans +# Defaults are runtime-aware: GLM uses higher values automatically +# GLM + micro chunks: 24 spans, 8192 tokens | GLM + semantic: 12 spans, 4096 tokens +# Granite/llamacpp: 3 spans, 512 tokens +# Set these to override the computed defaults: +# MICRO_OUT_MAX_SPANS=24 +# MICRO_MERGE_LINES=6 +# MICRO_BUDGET_TOKENS=8192 +# MICRO_TOKENS_PER_LINE=32 + + +# Decoder-path ReFRAG (feature-flagged; off by default) +REFRAG_DECODER=0 +# Runtime selection: openai, glm, minimax, llamacpp (default) +# Set explicitly - no auto-detection to avoid surprise API calls +REFRAG_RUNTIME=llamacpp +REFRAG_ENCODER_MODEL=BAAI/bge-base-en-v1.5 +REFRAG_PHI_PATH=/work/models/refrag_phi_768_to_dmodel.json +REFRAG_SENSE=heuristic + +# Enable index-time pseudo descriptions for micro-chunks (requires REFRAG_DECODER) +REFRAG_PSEUDO_DESCRIBE=1 + +# Llama.cpp sidecar (optional) +# Docker CPU-only (stable): http://llamacpp:8080 +# Native GPU-accelerated (fast): http://localhost:8081 +LLM_PROVIDER=ollama +OLLAMA_HOST=http://host.docker.internal:11434 +LLAMACPP_URL=http://llamacpp:8080 +REFRAG_DECODER_MODE=prompt # prompt|soft + +# OpenAI decoder backend (GPT-4.1, GPT-4.1-mini, o1, etc.) +# OPENAI_API_KEY=sk-your-openai-api-key +# OPENAI_MODEL=gpt-4.1-mini +# OPENAI_API_BASE=https://api.openai.com/v1 # or Azure/compatible endpoint + +# GLM (ZhipuAI) decoder backend +# Supports GLM-4.5, GLM-4.6, GLM-4.7 with version-specific defaults +# GLM-4.7: temp=1.0, top_p=0.95, max_output=128K, context=200K, tool_stream support +# GLM-4.6: temp=1.0, top_p=0.95, thinking support +# GLM-4.5: temp=1.0, top_p=0.95, fast model (thinking can be disabled) +# GLM_API_BASE=https://api.z.ai/api/coding/paas/v4/ +# GLM_MODEL=glm-4.6 +# GLM_MODEL_FAST=glm-4.5 +# GLM_API_KEY=your_glm_api_key_here + +# Answer shaping (enforce concise responses from context_answer) +# CTX_SUMMARY_CHARS=0 +# CTX_SNIPPET_CHARS=400 +# DECODER_MAX_TOKENS=4000 + +# Parallel pseudo-tag generation (indexing speedup) +# Set to 4 for 4x parallel GLM calls during indexing; 1 = sequential (default) +# PSEUDO_BATCH_CONCURRENCY=4 + +# MiniMax M2 decoder backend (OpenAI-compatible API) +# MINIMAX_API_BASE=https://api.minimax.io/v1 +# MINIMAX_MODEL=MiniMax-M2 +# MINIMAX_API_KEY=your_minimax_api_key_here + +# GPU Performance Toggle +# Set to 1 to use native GPU-accelerated server on localhost:8081 +# Set to 0 to use Docker CPU-only server (default, stable) +USE_GPU_DECODER=0 + +REFRAG_SOFT_SCALE=1.0 + +# Llama.cpp runtime tuning +LLAMACPP_USE_GPU=0 # Set to 1 to enable Metal/CLBlast acceleration +# LLAMACPP_GPU_LAYERS=-1 # Override number of layers to offload (defaults to -1 when USE_GPU=1) +# LLAMACPP_GPU_SPLIT= # Optional tensor split for multi-GPU setups +# LLAMACPP_THREADS= # Override number of CPU threads +# LLAMACPP_CTX_SIZE=8192 # Context tokens; higher values need more VRAM +# LLAMACPP_EXTRA_ARGS= # Additional flags passed verbatim to llama.cpp + +# Operational safeguards and timeouts +# Limit explosion of micro-chunks on huge files (0 to disable) +MAX_MICRO_CHUNKS_PER_FILE=2000 +# Qdrant request timeout (seconds) +QDRANT_TIMEOUT=20 +# Memory collection auto-detection cache +MEMORY_AUTODETECT=1 +MEMORY_COLLECTION_TTL_SECS=300 + +# Smarter re-indexing for symbol cache, reuse embeddings and reduce decoder/pseudo tags to re-index +SMART_SYMBOL_REINDEXING=1 +MAX_CHANGED_SYMBOLS_RATIO=0.6 + +# Watcher-safe defaults (recommended) +# Applied to watcher via compose; uncomment to apply globally. +# Qdrant write/read timeout (seconds) +# QDRANT_TIMEOUT=60 +# Cap per-file micro-chunks (keeps upserts small) +# MAX_MICRO_CHUNKS_PER_FILE=200 +# Upsert batching + retries for large payloads +# INDEX_UPSERT_BATCH=128 +# INDEX_UPSERT_RETRIES=5 +# INDEX_UPSERT_BACKOFF=0.5 +# Debounce file events to coalesce bursts +# WATCH_DEBOUNCE_SECS=1.5 +# Optional fs metadata fast-path for unchanged files (skips re-reading files +# when size/mtime match cache.json in the same workspace). +# INDEX_FS_FASTPATH=0 +# Optional 2-phase pseudo/tag mode (disabled by default). +# When enabled, indexer/watcher skip inline pseudo/tag generation and a background +# backfill worker adds them asynchronously via Qdrant. +# PSEUDO_DEFER_TO_WORKER=1 +# Backfill worker settings (only used when PSEUDO_DEFER_TO_WORKER=1): +# PSEUDO_BACKFILL_DEBUG=0 +# PSEUDO_BACKFILL_TICK_SECS=60 +# PSEUDO_BACKFILL_MAX_POINTS=256 + +# Development Remote Upload Configuration +# HOST_INDEX_PATH=./dev-workspace + +# Remote upload git history (used by upload clients) +# Max number of commits to include per bundle (0 disables git history) +REMOTE_UPLOAD_GIT_MAX_COMMITS=500 +# Optional git log since filter, e.g. '6 months ago' or '2024-01-01' +# REMOTE_UPLOAD_GIT_SINCE= +# Force full snapshot manifests instead of incremental deltas +# REMOTE_UPLOAD_GIT_FORCE=0 + +# Git history ingestion cleanup +# When ingesting a snapshot git history manifest, prune any older git_message points +# for the same repo that are not part of the latest snapshot run. +GIT_HISTORY_PRUNE=1 +# Delete git_history_*.json manifest files under .remote-git after successful ingest. +# This prevents unbounded accumulation of manifests on the server. +GIT_HISTORY_DELETE_MANIFEST=1 +# If keeping manifests, cap how many git_history_*.json files to keep per .remote-git dir +# (oldest beyond this count are deleted). Set 0 to disable. +GIT_HISTORY_MANIFEST_MAX_FILES=50 + +# Enable commit lineage goals for indexing +REFRAG_COMMIT_DESCRIBE=1 +COMMIT_VECTOR_SEARCH=0 + +STRICT_MEMORY_RESTORE=1 + +# info_request() tool settings (simplified codebase retrieval) +# Default result limit for info_request queries +INFO_REQUEST_LIMIT=10 +# Default context lines in snippets (richer than repo_search default) +INFO_REQUEST_CONTEXT_LINES=5 +# Enable explanation mode by default (summary, primary_locations, related_concepts) +# INFO_REQUEST_EXPLAIN_DEFAULT=0 +# Enable relationship mapping by default (imports_from, calls, related_paths) +# INFO_REQUEST_RELATIONSHIPS=0 + +# TOON output format (Token-Oriented Object Notation) +# When enabled, search results use compact TOON encoding to reduce token usage +# TOON_ENABLED=0 + + +# --------------------------------------------------------------------------- +# OpenLit Observability (OFF by default) +# --------------------------------------------------------------------------- +# Self-hosted LLM/vector-DB tracing via OpenTelemetry. +# +# Dashboard: http://localhost:3000 +# Email: user@openlit.io +# Password: openlituser +# +# To enable: set OPENLIT_ENABLED=1 and ensure openlit SDK is installed. +# Traces LLM calls (GLM, Ollama, llama.cpp) and Qdrant operations automatically. + +# Master toggle for OpenLit instrumentation +OPENLIT_ENABLED=0 + +# OTLP endpoint for trace export (OpenLit's built-in collector) +# Port 4317 = gRPC, Port 4318 = HTTP (default) +OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 + +# Application name shown in traces +OPENLIT_APP_NAME=context-engine + +# Environment tag for filtering traces (development, staging, production) +OPENLIT_ENVIRONMENT=development + +# ClickHouse backend (used by OpenLit for storing traces) +# OPENLIT_CLICKHOUSE_HOST=clickhouse +# OPENLIT_CLICKHOUSE_PORT=8123 +# OPENLIT_CLICKHOUSE_USER=default +# OPENLIT_CLICKHOUSE_PASSWORD=OPENLIT +# OPENLIT_CLICKHOUSE_DATABASE=openlit + + + +# --------------------------------------------------------------------------- +# Optional Auth & Bridge Configuration (OFF by default) +# --------------------------------------------------------------------------- + +# Global auth toggle for backend services (upload_service, MCP indexer/memory). +# When set to 1, services will require a valid session for protected operations. +# When unset or 0, auth is disabled and behavior matches previous versions. +# CTXCE_AUTH_ENABLED=0 + +# Shared token used by /auth/login for the bridge and other service clients. +# When CTXCE_AUTH_ENABLED=1 and this is set, /auth/login will only issue +# sessions when the provided token matches this value. +# CTXCE_AUTH_SHARED_TOKEN=change-me-dev-token + +# Create "open-dev" tokens when a shared token is unset, basically allows requesting a session with no real auth +# CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=0 + +# Optional admin token for creating additional users via /auth/users once the +# first user has been bootstrapped. If unset, only the initial user can be +# created (no additional users). +# CTXCE_AUTH_ADMIN_TOKEN=change-me-admin-token + +# Auth database location (default: sqlite file under WORK_DIR/.codebase). +# Use a SQLite URL for local/dev, or point to a different path. +# CTXCE_AUTH_DB_URL=sqlite:////work/.codebase/ctxce_auth.sqlite + +# Session TTL (seconds) for issued auth sessions. +# 0 or negative values disable expiry (sessions do not expire). When >0, +# active sessions are extended with a sliding window whenever they are used. +# CTXCE_AUTH_SESSION_TTL_SECONDS=0 + +# Collection registry & ACL (only used when CTXCE_AUTH_ENABLED=1). +# When enabled, infrastructure/health checks may populate an internal SQLite +# registry of known Qdrant collections, and ACL rules can be applied by services. + +# Admin collection deletion (dangerous). Set to 1 to allow the Admin UI to delete +# Qdrant collections and related server-managed workspaces. Leave 0/empty to +# completely disable collection deletion endpoints. +# CTXCE_ADMIN_COLLECTION_DELETE_ENABLED=0 + +# Collection registry sync behavior: when set to 1, a collection that is marked +# deleted in the auth DB (is_deleted=1) will be automatically un-deleted +# (is_deleted=0) if it reappears in Qdrant during registry discovery (e.g. via +# scripts/health_check.py). Default is OFF. +# CTXCE_COLLECTION_REGISTRY_UNDELETE_ON_DISCOVERY=0 + +# ACL bypass (dev/early deployments): allow all users to access all collections. +# CTXCE_ACL_ALLOW_ALL=0 + +# MCP boundary enforcement (OFF by default). +# When enabled, MCP servers will enforce collection ACLs using the auth DB. +# Requires CTXCE_AUTH_ENABLED=1. If CTXCE_ACL_ALLOW_ALL=1, enforcement is bypassed. +# This is intended for gradually rolling out collection-level permissions without +# changing existing client auth/session mechanisms. +# CTXCE_MCP_ACL_ENFORCE=0 + +# Bridge-side configuration (ctx-mcp-bridge): +# The bridge will POST to this URL for auth login and store the returned +# session id, then inject it into all MCP tool calls as the `session` field. +# CTXCE_AUTH_BACKEND_URL=http://localhost:8004 + +# Optional defaults for bridge CLI (env) auth: +# CTXCE_AUTH_TOKEN=dev-shared-token +# CTXCE_AUTH_USERNAME=you@example.com +# CTXCE_AUTH_PASSWORD=your-password + +# Controls cloning original collection into _old collection, recreating (full reindex) of a collection +# so that search is not blocked while reindexing. Bridge can route queries to _old. Uploads go to both +# original and _old collection +# CTXCE_STAGING_ENABLED=0 + +# Collection copy timeout (seconds) - falls back to QDRANT_TIMEOUT if not set +# Note: Qdrant enforces its own request deadline. Bump QDRANT_TIMEOUT (and matching timeout in your Qdrant config) +# to something large (e.g. 600 seconds) so the server doesn't drop long-running scrolls. +# CTXCE_COPY_COLLECTION_TIMEOUT=1800 # 30 mins, should be compatible with QDRANT timeout value + +# Collection copy batching and verification +# CTXCE_COPY_COLLECTION_BATCH=128 +# CTXCE_COPY_VERIFY_RETRIES=3 +# CTXCE_COPY_VERIFY_DELAY=2 # Delay in seconds between verification retries + +# Admin UI refresh interval for staging status (milliseconds) +# CTXCE_ADMIN_COLLECTION_REFRESH_MS=30000 # 30s + +# Snapshot refresh dry-run mode - preview snapshot refresh operations without executing +# CTXCE_SNAPSHOT_REFRESH_DRY_RUN=0 + +# Force collection name - ingest_code subprocess flag for admin-driven runs (staging rebuild). +# When set to 1, forces ingest_code to honor explicit COLLECTION_NAME even in multi-repo mode. +# This is primarily used by admin subprocess runs that supply per-repo indexing_env overrides. +# Normal watcher and indexer flows do not set this. +# CTXCE_FORCE_COLLECTION_NAME=0 + +# Collection mapping cache TTL - seconds to cache collection mapping index lookups +# CTXCE_COLLECTION_MAPPING_INDEX_TTL_SECS=5 + +# Bridge state token - optional shared token for bridge state validation +# CTXCE_BRIDGE_STATE_TOKEN= + +# --------------------------------------------------------------------------- +# End of Auth & Bridge Configuration +# --------------------------------------------------------------------------- + + diff --git a/.github/workflows/claude.yaml b/.github/workflows/claude.yaml new file mode 100644 index 00000000..732de78c --- /dev/null +++ b/.github/workflows/claude.yaml @@ -0,0 +1,68 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened] + pull_request_review: + types: [submitted] + pull_request_target: + types: [opened, synchronize] + +jobs: + claude: + # This simplified condition is more robust and correctly checks permissions. + if: > + (contains(github.event.comment.body, '@claude') || + contains(github.event.review.body, '@claude') || + contains(github.event.issue.body, '@claude') || + contains(github.event.pull_request.body, '@claude')) && + (github.event.sender.type == 'User' && ( + github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'COLLABORATOR' + )) + runs-on: ubuntu-latest + permissions: + # CRITICAL: Write permissions are required for the action to push branches and update issues/PRs. + contents: write + pull-requests: write + issues: write + id-token: write # Required for OIDC token exchange + actions: read # Required for Claude to read CI results on PRs + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # This correctly checks out the PR's head commit for pull_request_target events. + ref: ${{ github.event.pull_request.head.sha }} + + - name: Create Claude settings file + run: | + mkdir -p /home/runner/.claude + cat > /home/runner/.claude/settings.json << 'EOF' + { + "env": { + "ANTHROPIC_BASE_URL": "https://api.z.ai/api/anthropic", + "ANTHROPIC_AUTH_TOKEN": "${{ secrets.CUSTOM_ENDPOINT_API_KEY }}" + } + } + EOF + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + # Still need this to satisfy the action's validation + anthropic_api_key: ${{ secrets.CUSTOM_ENDPOINT_API_KEY }} + + # Use the same variable names as your local setup + settings: '{"env": {"ANTHROPIC_BASE_URL": "https://api.z.ai/api/anthropic", "ANTHROPIC_AUTH_TOKEN": "${{ secrets.CUSTOM_ENDPOINT_API_KEY }}"}}' + + track_progress: true + claude_args: | + --allowedTools "Bash,Edit,Read,Write,Glob,Grep" diff --git a/.github/workflows/publish-vscode-extension.yml b/.github/workflows/publish-vscode-extension.yml deleted file mode 100644 index f2e79a8a..00000000 --- a/.github/workflows/publish-vscode-extension.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Publish VS Code Extension - -on: - workflow_dispatch: - push: - branches: - - test - paths: - - "vscode-extension/context-engine-uploader/package.json" - -concurrency: - group: vscode-marketplace-publish - cancel-in-progress: false - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Use Node.js 20 - uses: actions/setup-node@v4 - with: - node-version: "20" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Check extension version change - id: version_check - shell: bash - run: | - set -euo pipefail - - FILE="vscode-extension/context-engine-uploader/package.json" - CURRENT_VERSION="$(node -p "require('./' + process.env.FILE).version" )" - - PREVIOUS_VERSION="" - if git cat-file -e "${{ github.event.before }}:${FILE}" 2>/dev/null; then - PREVIOUS_VERSION="$(git show "${{ github.event.before }}:${FILE}" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>{try{console.log(JSON.parse(d).version||"")}catch(e){console.log("")}})')" - fi - - echo "current_version=${CURRENT_VERSION}" >> "$GITHUB_OUTPUT" - echo "previous_version=${PREVIOUS_VERSION}" >> "$GITHUB_OUTPUT" - - if [[ -n "$CURRENT_VERSION" && "$CURRENT_VERSION" != "$PREVIOUS_VERSION" ]]; then - echo "changed=true" >> "$GITHUB_OUTPUT" - else - echo "changed=false" >> "$GITHUB_OUTPUT" - fi - env: - FILE: "vscode-extension/context-engine-uploader/package.json" - - - name: Publish to VS Code Marketplace - if: steps.version_check.outputs.changed == 'true' - shell: bash - env: - VSCE_PAT: ${{ secrets.VSCE_PAT }} - run: | - chmod +x vscode-extension/build/publish-vscode-extension.sh - vscode-extension/build/publish-vscode-extension.sh --bundle-deps diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c9635c20 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# Unified Context-Engine image for Kubernetes deployment +# Supports multiple roles: memory, indexer, watcher, llamacpp +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + WORK_ROOTS="/work,/app" + +# Install OS dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Python deps: reuse shared requirements file for consistency across services +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt + +# Copy scripts for all services +COPY scripts /app/scripts + +# Create directories +WORKDIR /work + +# Expose all necessary ports +EXPOSE 8000 8001 8002 8003 18000 18001 18002 18003 + +# Default to memory server +CMD ["python", "/app/scripts/mcp_memory_server.py"] \ No newline at end of file diff --git a/Dockerfile.indexer b/Dockerfile.indexer new file mode 100644 index 00000000..18ce4586 --- /dev/null +++ b/Dockerfile.indexer @@ -0,0 +1,37 @@ +# Dockerized code indexer for Qdrant +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + WORK_ROOTS="/work,/app" \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 \ + LOG_LEVEL=INFO + +# OS packages needed: git for history ingestion, curl for model downloads +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates curl && rm -rf /var/lib/apt/lists/* + +# Python deps: reuse shared requirements file +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir --upgrade --timeout=${PIP_DEFAULT_TIMEOUT} --retries=${PIP_RETRIES} -r /tmp/requirements.txt + +# Download reranker model and tokenizer during build +# Cross-encoder for reranking (ms-marco-MiniLM) + BGE tokenizer for micro-chunking +ARG RERANKER_ONNX_URL=https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2/resolve/main/onnx/model.onnx +ARG TOKENIZER_URL=https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json +RUN mkdir -p /app/models && \ + curl -L --fail --retry 3 -o /app/models/reranker.onnx "${RERANKER_ONNX_URL}" && \ + curl -L --fail --retry 3 -o /app/models/tokenizer.json "${TOKENIZER_URL}" + +# Set default paths for reranker (can be overridden via env) +ENV RERANKER_ONNX_PATH=/app/models/reranker.onnx \ + RERANKER_TOKENIZER_PATH=/app/models/tokenizer.json + +# Bake scripts into the image so we can mount arbitrary code at /work +COPY scripts /app/scripts + + +WORKDIR /work + +# Default command shows help; Makefile/compose will override entrypoint +CMD ["python", "/app/scripts/ingest_code.py", "--help"] diff --git a/Dockerfile.llamacpp b/Dockerfile.llamacpp new file mode 100644 index 00000000..42dc378b --- /dev/null +++ b/Dockerfile.llamacpp @@ -0,0 +1,21 @@ +# Optional: build llama.cpp server with default settings +# Production note: choose a tiny GGUF model (e.g., tinyllama) in ./models/model.gguf +FROM debian:bookworm-slim AS build +RUN apt-get update && apt-get install -y --no-install-recommends \ + git build-essential cmake curl ca-certificates libcurl4-openssl-dev && rm -rf /var/lib/apt/lists/* +WORKDIR /src +RUN git clone --depth=1 https://github.com/ggerganov/llama.cpp.git +WORKDIR /src/llama.cpp +RUN cmake -S . -B build -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=OFF && cmake --build build -j + +FROM debian:bookworm-slim +ARG MODEL_URL="" +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl libgomp1 libcurl4 && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=build /src/llama.cpp/build/bin/llama-server /app/llama-server +# Optionally bake a model into the image if MODEL_URL is provided +RUN mkdir -p /models \ + && if [ -n "$MODEL_URL" ]; then echo "Fetching model: $MODEL_URL" && curl -L --fail --retry 3 -C - "$MODEL_URL" -o /models/model.gguf; else echo "No MODEL_URL provided; expecting host volume /models"; fi +EXPOSE 8080 +ENTRYPOINT ["/app/llama-server"] diff --git a/Dockerfile.mcp b/Dockerfile.mcp new file mode 100644 index 00000000..a97142ed --- /dev/null +++ b/Dockerfile.mcp @@ -0,0 +1,25 @@ +# Minimal image to run the Qdrant MCP server with SSE transport +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + WORK_ROOTS="/work,/app" \ + HF_HOME=/tmp/cache \ + TRANSFORMERS_CACHE=/tmp/cache + +# Python deps: reuse shared requirements file for consistency across services +# Create cache/rerank directories in same layer +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt \ + && mkdir -p /tmp/cache && chmod 755 /tmp/cache \ + && mkdir -p /tmp/rerank_events /tmp/rerank_weights \ + && chmod 777 /tmp/rerank_events /tmp/rerank_weights + +# Bake scripts into image so server can run even when /work points elsewhere +COPY scripts /app/scripts + +# Expose SSE port +EXPOSE 8000 + +# Default command: run the server with SSE transport (env provides host/port) +CMD ["python", "/app/scripts/mcp_memory_server.py"] diff --git a/Dockerfile.mcp-indexer b/Dockerfile.mcp-indexer new file mode 100644 index 00000000..064a9188 --- /dev/null +++ b/Dockerfile.mcp-indexer @@ -0,0 +1,44 @@ +# Companion MCP server that exposes indexing/pruning/list tools over SSE +# Reuse the same deps as the indexer image so we can call the scripts directly. +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + WORK_ROOTS="/work,/app" \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 \ + LOG_LEVEL=INFO + +# OS deps (git for history if we extend later, curl for model downloads, build-essential for compiling native extensions) +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates curl build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Python deps: reuse shared requirements (includes FastMCP + OpenAI SDK) +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir --upgrade --timeout=${PIP_DEFAULT_TIMEOUT} --retries=${PIP_RETRIES} -r /tmp/requirements.txt + +# Create rerank dirs + download reranker model/tokenizer in single layer +ARG RERANKER_ONNX_URL=https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2/resolve/main/onnx/model.onnx +ARG TOKENIZER_URL=https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json +RUN mkdir -p /tmp/rerank_events /tmp/rerank_weights \ + && chmod 777 /tmp/rerank_events /tmp/rerank_weights \ + && mkdir -p /app/models \ + && curl -L --fail --retry 3 -o /app/models/reranker.onnx "${RERANKER_ONNX_URL}" \ + && curl -L --fail --retry 3 -o /app/models/tokenizer.json "${TOKENIZER_URL}" + +# Set default paths for reranker (can be overridden via env) +ENV RERANKER_ONNX_PATH=/app/models/reranker.onnx \ + RERANKER_TOKENIZER_PATH=/app/models/tokenizer.json + +# Bake scripts into the image so entrypoints don't rely on /work +COPY scripts /app/scripts + +COPY bench /app/bench + +# Expose SSE port for this companion server +EXPOSE 8001 + +WORKDIR /work + +# Default command runs the companion MCP server +CMD ["python", "/app/scripts/mcp_indexer_server.py"] diff --git a/Dockerfile.upload-service b/Dockerfile.upload-service new file mode 100644 index 00000000..8c3f47c9 --- /dev/null +++ b/Dockerfile.upload-service @@ -0,0 +1,53 @@ +# Dockerfile for Context-Engine Delta Upload Service +FROM python:3.11-slim + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONPATH=/app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Create app directory +WORKDIR /app + +# Copy requirements first for better caching +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY scripts/ ./scripts/ +COPY . . + +# Create work dir, non-root user, and set ownership in single layer +RUN mkdir -p /work && chmod 755 /work \ + && useradd --create-home --shell /bin/bash app \ + && chown -R app:app /app /work +USER app + +# Expose port +EXPOSE 8002 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8002/health || exit 1 + +# Default environment variables +ENV UPLOAD_SERVICE_HOST=0.0.0.0 \ + UPLOAD_SERVICE_PORT=8002 \ + QDRANT_URL=http://qdrant:6333 \ + WORK_DIR=/work \ + MAX_BUNDLE_SIZE_MB=100 \ + UPLOAD_TIMEOUT_SECS=300 + +# Run the upload service +CMD ["python", "scripts/upload_service.py"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..068f765f --- /dev/null +++ b/Makefile @@ -0,0 +1,371 @@ +SHELL := /bin/bash + +# Avoid inheriting Docker context from shells/venvs (e.g., DOCKER_HOST=unix:///Users/...) +# An empty export forces docker to use its default context/socket. +export DOCKER_HOST = + +.PHONY: help up down logs ps restart rebuild index reindex watch watch-remote env hybrid bootstrap history rerank-local setup-reranker prune warm health test-e2e +.PHONY: venv venv-install dev-remote-up dev-remote-down dev-remote-logs dev-remote-restart dev-remote-bootstrap dev-remote-test dev-remote-client dev-remote-clean +.PHONY: rerank-eval rerank-eval-ablations rerank-benchmark + +.PHONY: qdrant-status qdrant-list qdrant-prune qdrant-index-root + +venv: ## create local virtualenv .venv + python3 -m venv .venv && . .venv/bin/activate && pip install -U pip + +venv-install: ## install project dependencies into .venv + [ -d .venv ] || $(MAKE) venv + . .venv/bin/activate && pip install -r requirements.txt + + + +# Show available targets +help: ## show targets and their descriptions + @grep -E '^[a-zA-Z0-9_-]+:.*?## ' Makefile | sed 's/:.*##/: /' | column -t + +# Guard for required env vars: usage `make guard-VAR` +guard-%: + @if [ -z "${${*}}" ]; then echo "Missing env: $*"; exit 1; fi + +up: ## docker compose up (build if needed) + docker compose up -d --build + +down: ## docker compose down + docker compose down + +logs: ## follow logs + docker compose logs -f --tail=100 + +ps: ## show container status + docker compose ps + +restart: ## restart stack (rebuild) + docker compose down && docker compose up -d --build + +rebuild: ## rebuild images without cache + docker compose build --no-cache + +index: ## index code into Qdrant without dropping the collection + docker compose run --rm indexer --root /work + +reindex: ## recreate collection then index from scratch (will remove existing points!) + docker compose run --rm indexer --root /work --recreate + +reindex-hard: ## clear all caches (local + container) then recreate collection and index from scratch + @echo "Clearing local caches..." + @rm -f .codebase/cache.json || true + @rm -rf .codebase/symbols || true + @find dev-workspace -path "*/.codebase/cache.json" -delete 2>/dev/null || true + @find dev-workspace -path "*/.codebase/symbols" -type d -exec rm -rf {} + 2>/dev/null || true + @echo "Clearing container caches..." + @for c in indexer watcher mcp_indexer; do \ + docker compose exec -T $$c sh -c 'find /work -path "*/.codebase/cache.json" -delete 2>/dev/null; find /work -path "*/.codebase/symbols" -type d -exec rm -rf {} + 2>/dev/null' 2>/dev/null || true; \ + done + @echo "Reindexing..." + docker compose run --rm indexer --root /work --recreate + + +# Index an arbitrary local path without cloning into this repo +index-path: ## index an arbitrary repo: make index-path REPO_PATH=/abs/path [RECREATE=1] [REPO_NAME=name] [COLLECTION=name] + @if [ -z "$(REPO_PATH)" ]; then \ + echo "Usage: make index-path REPO_PATH=/abs/path [RECREATE=1] [REPO_NAME=name] [COLLECTION=name]"; exit 1; \ + fi + @NAME=$${REPO_NAME:-$$(basename "$(REPO_PATH)")}; \ + COLL=$${COLLECTION:-$$NAME}; \ + HOST_INDEX_PATH="$(REPO_PATH)" COLLECTION_NAME="$$COLL" REPO_NAME="$$NAME" \ + docker compose run --rm -v "$$PWD":/app:ro --entrypoint python indexer /app/scripts/ingest_code.py --root /work $${RECREATE:+--recreate} + +# Index the current working directory quickly +index-here: ## index the current directory: make index-here [RECREATE=1] [REPO_NAME=name] [COLLECTION=name] + @RP=$$(pwd); \ + NAME=$${REPO_NAME:-$$(basename "$$RP")}; \ + COLL=$${COLLECTION:-$$NAME}; \ + HOST_INDEX_PATH="$$RP" COLLECTION_NAME="$$COLL" REPO_NAME="$$NAME" \ + docker compose run --rm indexer --root /work $${RECREATE:+--recreate} + + +watch: ## watch mode: reindex changed files on save (Ctrl+C to stop) + docker compose run --rm --entrypoint python indexer /work/scripts/watch_index.py + +watch-remote: ## remote watch mode: upload delta bundles to remote server (Ctrl+C to stop) + @echo "Starting remote watch mode..." + @if [ -z "$(REMOTE_UPLOAD_ENDPOINT)" ]; then \ + echo "Error: REMOTE_UPLOAD_ENDPOINT is required"; \ + echo "Usage: make watch-remote REMOTE_UPLOAD_ENDPOINT=http://your-server:8080 [REMOTE_UPLOAD_MAX_RETRIES=3] [REMOTE_UPLOAD_TIMEOUT=30]"; \ + exit 1; \ + fi + @echo "Remote upload endpoint: $(REMOTE_UPLOAD_ENDPOINT)" + @echo "Max retries: $${REMOTE_UPLOAD_MAX_RETRIES:-3}" + @echo "Timeout: $${REMOTE_UPLOAD_TIMEOUT:-30} seconds" + docker compose run --rm --entrypoint python \ + -e REMOTE_UPLOAD_ENABLED=1 \ + -e REMOTE_UPLOAD_ENDPOINT=$(REMOTE_UPLOAD_ENDPOINT) \ + -e REMOTE_UPLOAD_MAX_RETRIES=$${REMOTE_UPLOAD_MAX_RETRIES:-3} \ + -e REMOTE_UPLOAD_TIMEOUT=$${REMOTE_UPLOAD_TIMEOUT:-30} \ + indexer /work/scripts/watch_index.py + +rerank: ## multi-query re-ranker helper example + docker compose run --rm --entrypoint python indexer /work/scripts/rerank_query.py \ + --query "chunk code by lines with overlap for indexing" \ + --query "function to split code into overlapping line chunks" \ + --language python --under /work/scripts --limit 5 + +warm: ## prime ANN/search caches with a few queries + docker compose run --rm --entrypoint python indexer /work/scripts/warm_start.py --ef 256 --limit 3 + +health: ## run health checks for collection/model settings + docker compose run --rm --entrypoint python indexer /work/scripts/health_check.py + + +# Check llama.cpp decoder health on localhost:8080 (200 OK expected) +decoder-health: ## ping llama.cpp server + @URL=$${LLAMACPP_HEALTH_URL:-http://localhost:8080}; \ + CODE=$$(curl -s -o /dev/null -w "%{http_code}" $$URL); \ + echo "llamacpp @ $$URL -> $$CODE"; \ + [ "$$CODE" = "200" ] && echo "OK" || echo "WARN: non-200" + +env: ## create .env from example if missing + [ -f .env ] || cp .env.example .env + +hybrid: ## hybrid search: dense + lexical RRF fuse (respects --language/--under/--kind) + docker compose run --rm --entrypoint python indexer /work/scripts/hybrid_search.py \ + --query "chunk code by lines" --query "overlapping line chunks" --limit 8 + +bootstrap: env up ## one-shot: up -> wait -> index -> warm -> health + ./scripts/wait-for-qdrant.sh + $(MAKE) reindex + $(MAKE) warm || true + $(MAKE) health + +history: ## ingest Git history (messages + file lists) + docker compose run --rm --entrypoint python indexer /work/scripts/ingest_history.py --max-commits 200 + +prune-path: ## prune a repo by path: make prune-path REPO_PATH=/abs/path + @if [ -z "$(REPO_PATH)" ]; then \ + echo "Usage: make prune-path REPO_PATH=/abs/path"; exit 1; \ + fi + HOST_INDEX_PATH="$(REPO_PATH)" PRUNE_ROOT=/work \ + docker compose run --rm --entrypoint python indexer /work/scripts/prune.py + +rerank-local: ## local cross-encoder reranker (requires RERANKER_ONNX_PATH, RERANKER_TOKENIZER_PATH) + @if [ -z "$(RERANKER_ONNX_PATH)" ] || [ -z "$(RERANKER_TOKENIZER_PATH)" ]; then \ + echo "RERANKER_ONNX_PATH and RERANKER_TOKENIZER_PATH must be set in .env"; exit 1; \ + fi + docker compose run --rm --entrypoint python indexer /work/scripts/rerank_local.py --query "search symbols" --topk 50 --limit 12 + +setup-reranker: ## download ONNX reranker + tokenizer, update .env, then smoke-test + @if [ -z "$(ONNX_URL)" ] || [ -z "$(TOKENIZER_URL)" ]; then \ + echo "Provide ONNX_URL and TOKENIZER_URL, e.g."; \ + echo " make setup-reranker ONNX_URL=https://.../model.onnx TOKENIZER_URL=https://.../tokenizer.json"; \ + exit 1; \ + fi + python3 scripts/setup_reranker.py --onnx-url "$(ONNX_URL)" --tokenizer-url "$(TOKENIZER_URL)" --dest "$(or $(DEST),models)" && \ + $(MAKE) rerank-local + +prune: ## remove points for missing files or mismatched file_hash + docker compose run --rm --entrypoint python indexer /work/scripts/prune.py + + + +# Convenience: full no-cache rebuild and bring-up sequences +up-nc: ## up with full no-cache rebuild + docker compose build --no-cache && docker compose up -d + +restart-nc: ## down, no-cache rebuild, up + docker compose down && docker compose build --no-cache && docker compose up -d + +reset-dev: ## full dev reset: qdrant -> wait -> init payload -> reindex -> bring up services (incl. decoder) + docker compose down || true + docker compose build --no-cache indexer mcp mcp_indexer mcp_http mcp_indexer_http watcher llamacpp + docker compose up -d qdrant + ./scripts/wait-for-qdrant.sh + docker compose run --rm init_payload || true + $(MAKE) tokenizer + + docker compose run --rm -e INDEX_MICRO_CHUNKS -e MAX_MICRO_CHUNKS_PER_FILE -e TOKENIZER_PATH -e TOKENIZER_URL indexer --root /work --recreate + $(MAKE) llama-model + docker compose up -d mcp mcp_indexer watcher llamacpp + # Ensure watcher is up even if a prior step or manual bring-up omitted it + docker compose up -d watcher + docker compose ps + + +reset-dev-codex: ## bring up Qdrant + Streamable HTTP MCPs only (for OpenAI Codex RMCP) + docker compose down || true + docker compose build --no-cache indexer mcp_http mcp_indexer_http watcher llamacpp + docker compose up -d qdrant + ./scripts/wait-for-qdrant.sh + # Seed Qdrant and create a fresh index for Codex + docker compose run --rm init_payload || true + $(MAKE) tokenizer + docker compose run --rm -e INDEX_MICRO_CHUNKS -e MAX_MICRO_CHUNKS_PER_FILE -e TOKENIZER_PATH -e TOKENIZER_URL indexer --root /work --recreate + $(MAKE) llama-model + + docker compose up -d mcp_http mcp_indexer_http watcher llamacpp + docker compose ps + + +quantize-reranker: ## Quantize reranker ONNX to INT8 (set RERANKER_ONNX_PATH, optional OUTPUT_ONNX_PATH) + @[ -n "$(RERANKER_ONNX_PATH)" ] || { echo "Set RERANKER_ONNX_PATH to your ONNX file"; exit 1; } + python3 scripts/quantize_reranker.py + + + +reset-dev-dual: ## bring up BOTH legacy SSE and Streamable HTTP MCPs (dual-compat mode) + docker compose down || true + docker compose build --no-cache indexer mcp mcp_indexer mcp_http mcp_indexer_http watcher llamacpp upload_service + docker compose up -d qdrant + ./scripts/wait-for-qdrant.sh + $(MAKE) tokenizer + # Index first (creates collection), then init_payload (creates indexes on existing collection) + docker compose run --rm -e INDEX_MICRO_CHUNKS -e MAX_MICRO_CHUNKS_PER_FILE -e TOKENIZER_PATH -e TOKENIZER_URL indexer --root /work --recreate + docker compose run --rm init_payload || true + $(MAKE) llama-model + docker compose up -d mcp mcp_indexer mcp_http mcp_indexer_http watcher llamacpp upload_service + docker compose up -d watcher + docker compose ps + +dev-core: ## core dev stack including uploader (alias for reset-dev-dual) + $(MAKE) reset-dev-dual + +# --- llama.cpp tiny model provisioning --- +LLAMACPP_MODEL_URL ?= https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF/resolve/main/granite-4.0-micro-Q4_K_M.gguf +LLAMACPP_MODEL_PATH ?= models/model.gguf + +llama-model: ## download tiny GGUF model into ./models/model.gguf (override with LLAMACPP_MODEL_URL/LLAMACPP_MODEL_PATH) + @mkdir -p $(dir $(LLAMACPP_MODEL_PATH)) + @echo "Downloading: $(LLAMACPP_MODEL_URL) -> $(LLAMACPP_MODEL_PATH)" && \ + curl -L --fail --retry 3 -C - "$(LLAMACPP_MODEL_URL)" -o "$(LLAMACPP_MODEL_PATH)" + +llamacpp-up: llama-model ## fetch tiny model (if missing) and start llama.cpp sidecar + docker compose up -d llamacpp && sleep 2 && curl -sI http://localhost:8080 | head -n1 || true + +# Optional: build a custom image that bakes the model into the image (no host volume needed) +llamacpp-build-image: ## build custom llama.cpp image with baked model (override LLAMACPP_MODEL_URL) + docker build -f Dockerfile.llamacpp --build-arg MODEL_URL="$(LLAMACPP_MODEL_URL)" -t context-llamacpp:tiny . + +# Download a tokenizer.json for micro-chunking (default: BAAI/bge-base-en-v1.5) +TOKENIZER_URL ?= https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json +TOKENIZER_PATH ?= models/tokenizer.json +tokenizer: ## download tokenizer.json to models/tokenizer.json (override with TOKENIZER_URL/TOKENIZER_PATH) + @mkdir -p $(dir $(TOKENIZER_PATH)) + @echo "Downloading: $(TOKENIZER_URL) -> $(TOKENIZER_PATH)" && \ + curl -L --fail --retry 3 -C - "$(TOKENIZER_URL)" -o "$(TOKENIZER_PATH)" + +# --- Development Remote Upload System Targets --- + +dev-remote-up: ## start dev-remote stack with upload service + @echo "Starting development remote upload system..." + @mkdir -p dev-workspace/.codebase + docker compose -f docker-compose.yml up -d --build + +dev-remote-down: ## stop dev-remote stack + @echo "Stopping development remote upload system..." + docker compose -f docker-compose.yml down + +dev-remote-logs: ## follow logs for dev-remote stack + docker compose -f docker-compose.yml logs -f --tail=100 + +dev-remote-restart: ## restart dev-remote stack (rebuild) + docker compose -f docker-compose.yml down && docker compose -f docker-compose.yml up -d --build + +dev-remote-bootstrap: env dev-remote-up ## bootstrap dev-remote: up -> wait -> init -> index -> warm + @echo "Bootstrapping development remote upload system..." + ./scripts/wait-for-qdrant.sh + docker compose -f docker-compose.yml run --rm init_payload || true + $(MAKE) tokenizer + docker compose -f docker-compose.yml run --rm indexer --root /work --recreate + $(MAKE) warm || true + $(MAKE) health + +dev-remote-test: ## test remote upload workflow + @echo "Testing remote upload workflow..." + @echo "Upload service should be accessible at http://localhost:8004" + @echo "Health check: curl http://localhost:8004/health" + @echo "Status check: curl 'http://localhost:8004/api/v1/delta/status?workspace_path=/work/test-repo'" + @echo "Test upload: curl -X POST -F 'bundle=@test-bundle.tar.gz' -F 'workspace_path=/work/test-repo' http://localhost:8004/api/v1/delta/upload" + +dev-remote-client: ## start remote upload client for testing + @echo "Starting remote upload client..." + docker compose -f docker-compose.yml --profile client up -d remote_upload_client + +dev-remote-clean: ## clean up dev-remote volumes and containers + @echo "Cleaning up development remote upload system..." + docker compose -f docker-compose.yml down -v + docker volume rm context-engine_shared_workspace context-engine_shared_codebase context-engine_upload_temp context-engine_qdrant_storage_dev_remote 2>/dev/null || true + rm -rf dev-workspace + + +# Router helpers +Q ?= what is hybrid search? +route-plan: ## plan-only route for a query: make route-plan Q="your question" + python3 scripts/mcp_router.py --plan "$(Q)" + +route-run: ## execute routed tool(s) over HTTP: make route-run Q="your question" + python3 scripts/mcp_router.py --run "$(Q)" +router-eval: ## run the mock-based router eval harness + python3 scripts/router_eval.py + + +# Live orchestration smoke test (no CI): bring up stack, reindex, run router +router-smoke: ## spin up compose, reindex, store a memory via router, then answer; exits nonzero on failure + set -e; \ + docker compose down || true; \ + docker compose up -d qdrant; \ + ./scripts/wait-for-qdrant.sh; \ + $(MAKE) llama-model; \ + docker compose up -d mcp_http mcp_indexer_http llamacpp; \ + echo "Waiting for MCP HTTP health..."; \ + for i in $$(seq 1 30); do \ + code1=$$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$${FASTMCP_HTTP_HEALTH_PORT:-18002}/readyz || true); \ + code2=$$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$${FASTMCP_INDEXER_HTTP_HEALTH_PORT:-18003}/readyz || true); \ + if [ "$$code1" = "200" ] && [ "$$code2" = "200" ]; then echo "MCP HTTP ready"; break; fi; \ + sleep 1; \ + if [ $$i -eq 30 ]; then echo "MCP HTTP health timeout"; exit 1; fi; \ + done; \ + $(MAKE) reindex; \ + echo "Storing a smoke memory via router..."; \ + python3 scripts/mcp_router.py --run "remember this: router smoke memory"; \ + echo "Running a router answer..."; \ + python3 scripts/mcp_router.py --run "recap our architecture decisions for the indexer"; \ + echo "router-smoke: PASS" + + + +# Qdrant via MCP router convenience targets +qdrant-status: + python3 scripts/mcp_router.py --run "status" + +qdrant-list: + python3 scripts/mcp_router.py --run "list collections" + +qdrant-prune: + python3 scripts/mcp_router.py --run "prune" + +qdrant-index-root: + python3 scripts/mcp_router.py --run "reindex repo" + + +# --- ctx CLI helper --- +# Usage examples (default prints ONLY the improved prompt): +# make ctx Q="how does hybrid search work?" +# make ctx Q="explain caching" ARGS="--language python --under scripts/" +# To include Supporting Context: +# make ctx Q="explain caching" ARGS="--with-context --limit 2" +ctx: ## enhance a prompt with repo context: make ctx Q="your question" [ARGS='--language python --under scripts/ --with-context'] + @if [ -z "$(Q)" ]; then \ + echo 'Usage: make ctx Q="your question" [ARGS="--language python --under scripts/ --with-context"]'; \ + exit 1; \ + fi; \ + python3 scripts/ctx.py "$(Q)" $(ARGS) + + +# --- Reranker Evaluation --- +rerank-eval: ## run offline reranker evaluation (fixed queries, MRR/Recall/latency) + python3 scripts/rerank_eval.py --output rerank_eval_results.json + +rerank-eval-ablations: ## run full ablation study (baseline, recursive, learning, onnx) + python3 scripts/rerank_eval.py --ablations --output rerank_eval_ablations.json + +rerank-benchmark: ## run production benchmark on real codebase + python3 scripts/rerank_real_benchmark.py diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..824c629a --- /dev/null +++ b/bench/README.md @@ -0,0 +1,97 @@ +# Context-Engine Benchmark Suite + +Reproducible retrieval quality benchmarks with Hit@k, MRR, and latency metrics. + +## Quick Start + +### 1. Clone the public snapshot + +```bash +python bench/clone_snapshot.py --manifest bench/datasets/public_v1.json +``` + +This clones kubernetes@v1.28.0, vscode@1.86.0, and transformers@v4.37.0 to `bench/data/`. + +### 2. Index into Qdrant + +```bash +# Index each repo (adjust paths as needed) +python scripts/ingest_code.py --root bench/data/kubernetes_kubernetes --collection ctx-bench-k8s +python scripts/ingest_code.py --root bench/data/microsoft_vscode --collection ctx-bench-vscode +python scripts/ingest_code.py --root bench/data/huggingface_transformers --collection ctx-bench-hf + +# Or create a combined collection +python scripts/ingest_code.py --root bench/data --collection ctx-bench-public-v1 +``` + +### 3. Run quality evaluation + +```bash +# Single model, single config +python bench/eval_quality.py \ + --collection ctx-bench-public-v1 \ + --model "BAAI/bge-base-en-v1.5" \ + --gold-file bench/gold/public_v1.queries.jsonl +``` + +**Output:** +``` +MRR: 0.5800 +Hit@1: 0.4500 +Hit@5: 0.7200 +Hit@10: 0.8000 +``` + +### 4. Run full matrix comparison + +```bash +# Compare models +python bench/run_matrix.py \ + --src ctx-bench-public-v1 \ + --gold-file bench/gold/public_v1.queries.jsonl \ + --models "BAAI/bge-base-en-v1.5,sentence-transformers/all-MiniLM-L6-v2" + +# Compare models × configs (A/B testing) +python bench/run_matrix.py \ + --src ctx-bench-public-v1 \ + --gold-file bench/gold/public_v1.queries.jsonl \ + --config-file bench/configs/ctx_ab_configs.json \ + --models "BAAI/bge-base-en-v1.5" \ + --output results/bench_matrix.json +``` + +## Files + +| File | Purpose | +|------|---------| +| `datasets/public_v1.json` | Snapshot manifest (repos + pinned refs) | +| `gold/public_v1.queries.jsonl` | 30 gold queries with expected file paths | +| `configs/ctx_ab_configs.json` | A/B config variants (rerank on/off, RRF weights, etc.) | +| `clone_snapshot.py` | Clone repos at pinned commits | +| `eval_quality.py` | Compute Hit@k, MRR against gold set | +| `eval.py` | Latency harness (p50/p95, Jaccard overlap) | +| `run_matrix.py` | Orchestrate (model × config) comparison | +| `copy-coll.py` | Clone/re-embed collections for model comparison | + +## Gold Query Format + +```jsonl +{"id": "k8s-eviction-001", "query": "where is pod eviction logic", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/kubelet/eviction/eviction_manager.go"}], "task_type": "navigation"} +``` + +## Metrics + +- **Hit@k**: Did any of the top-k results match expected files? +- **MRR**: Mean Reciprocal Rank (1/rank of first correct result) +- **Latency p50/p95**: Response time percentiles + +## Adding Your Own Queries + +Edit `bench/gold/public_v1.queries.jsonl` or create a new file: + +```bash +python bench/eval_quality.py \ + --collection my-collection \ + --model "BAAI/bge-base-en-v1.5" \ + --gold-file bench/gold/my_queries.jsonl +``` diff --git a/bench/clone_snapshot.py b/bench/clone_snapshot.py new file mode 100644 index 00000000..2e6d351b --- /dev/null +++ b/bench/clone_snapshot.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +""" +Clone public repos at pinned refs for reproducible benchmarks. + +Usage: + python bench/clone_snapshot.py --manifest bench/datasets/public_v1.json + python bench/clone_snapshot.py --manifest bench/datasets/public_v1.json --repo kubernetes/kubernetes +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + +BENCH_DIR = Path(__file__).resolve().parent +DATA_DIR = BENCH_DIR / "data" + + +def clone_repo( + name: str, + ref: str, + dest: Path, + shallow: bool = True, +) -> Optional[str]: + """Clone a repo at a specific ref, return resolved SHA.""" + if dest.exists(): + print(f"[clone] {name} already exists at {dest}, fetching ref...") + # Fetch and checkout the ref + subprocess.run( + ["git", "fetch", "--depth=1", "origin", f"refs/tags/{ref}:refs/tags/{ref}"], + cwd=dest, + capture_output=True, + ) + proc = subprocess.run( + ["git", "checkout", ref], + cwd=dest, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + print(f"[clone] WARNING: checkout {ref} failed: {proc.stderr[:200]}") + else: + print(f"[clone] Cloning {name}@{ref} to {dest}...") + dest.parent.mkdir(parents=True, exist_ok=True) + + clone_cmd = [ + "git", "clone", + "--branch", ref, + f"https://github.com/{name}.git", + str(dest), + ] + if shallow: + clone_cmd.insert(2, "--depth=1") + + proc = subprocess.run(clone_cmd, capture_output=True, text=True) + if proc.returncode != 0: + print(f"[clone] ERROR cloning {name}: {proc.stderr[:500]}") + return None + + # Get resolved SHA + proc = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=dest, + capture_output=True, + text=True, + ) + if proc.returncode == 0: + sha = proc.stdout.strip() + print(f"[clone] {name}@{ref} → {sha[:12]}") + return sha + return None + + +def count_files(path: Path, extensions: List[str]) -> int: + """Count files with given extensions.""" + count = 0 + for ext in extensions: + count += len(list(path.rglob(f"*{ext}"))) + return count + + +def main(): + parser = argparse.ArgumentParser(description="Clone benchmark snapshot repos") + parser.add_argument( + "--manifest", + type=str, + default=str(BENCH_DIR / "datasets" / "public_v1.json"), + help="Path to dataset manifest JSON", + ) + parser.add_argument( + "--repo", + type=str, + default=None, + help="Clone only this repo (owner/name format)", + ) + parser.add_argument( + "--data-dir", + type=str, + default=str(DATA_DIR), + help="Directory to clone repos into", + ) + parser.add_argument( + "--full-clone", + action="store_true", + help="Do full clone instead of shallow (for history analysis)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print what would be done without cloning", + ) + + args = parser.parse_args() + + # Load manifest + manifest_path = Path(args.manifest) + if not manifest_path.exists(): + print(f"ERROR: Manifest not found: {manifest_path}") + sys.exit(1) + + with open(manifest_path) as f: + manifest = json.load(f) + + data_dir = Path(args.data_dir) + repos = manifest.get("repos", []) + + # Filter to specific repo if requested + if args.repo: + repos = [r for r in repos if r["name"] == args.repo] + if not repos: + print(f"ERROR: Repo '{args.repo}' not found in manifest") + sys.exit(1) + + print(f"[clone] Dataset: {manifest.get('id', 'unknown')}") + print(f"[clone] Repos: {len(repos)}") + print(f"[clone] Data dir: {data_dir}") + + if args.dry_run: + for repo in repos: + name = repo["name"] + ref = repo["ref"] + dest = data_dir / name.replace("/", "_") + print(f" Would clone: {name}@{ref} → {dest}") + return + + # Clone each repo and update manifest with SHAs + resolved: List[Dict[str, Any]] = [] + + for repo in repos: + name = repo["name"] + ref = repo["ref"] + dest = data_dir / name.replace("/", "_") + + sha = clone_repo(name, ref, dest, shallow=not args.full_clone) + + repo_info = dict(repo) + repo_info["sha"] = sha + repo_info["local_path"] = str(dest) + + # Count files + ext_map = { + "go": [".go"], + "python": [".py"], + "typescript": [".ts", ".tsx"], + "javascript": [".js", ".jsx"], + "yaml": [".yaml", ".yml"], + } + file_counts = {} + for lang in repo.get("language", []): + exts = ext_map.get(lang, []) + if exts: + file_counts[lang] = count_files(dest, exts) + repo_info["file_counts"] = file_counts + + resolved.append(repo_info) + + # Write resolved manifest + resolved_manifest = { + "id": manifest.get("id", "unknown"), + "version": manifest.get("version", "1.0.0"), + "cloned_at": datetime.now().isoformat(), + "data_dir": str(data_dir), + "repos": resolved, + } + + resolved_path = manifest_path.with_suffix(".resolved.json") + with open(resolved_path, "w") as f: + json.dump(resolved_manifest, f, indent=2) + + print(f"\n[clone] Resolved manifest written to: {resolved_path}") + + # Summary + print("\n" + "=" * 60) + print("SNAPSHOT SUMMARY") + print("=" * 60) + for repo in resolved: + name = repo["name"] + sha = repo.get("sha", "?")[:12] + counts = repo.get("file_counts", {}) + count_str = ", ".join(f"{k}:{v}" for k, v in counts.items()) + print(f" {name}@{sha} → {count_str or 'no counts'}") + + +if __name__ == "__main__": + main() diff --git a/bench/configs/ctx_ab_configs.json b/bench/configs/ctx_ab_configs.json new file mode 100644 index 00000000..21ece5ca --- /dev/null +++ b/bench/configs/ctx_ab_configs.json @@ -0,0 +1,54 @@ +{ + "description": "A/B config variants for Context-Engine retrieval benchmarks", + "version": "1.0.0", + "configs": [ + { + "id": "default", + "name": "Default (rerank + RRF balanced)", + "args": [], + "description": "Production defaults" + }, + { + "id": "no_rerank", + "name": "No Reranker", + "args": ["--no-rerank"], + "description": "Disable cross-encoder reranking" + }, + { + "id": "no_lex", + "name": "Dense Only", + "args": ["--no-lex"], + "description": "Disable lexical/BM25 component" + }, + { + "id": "no_dense", + "name": "Lexical Only", + "args": ["--no-dense"], + "description": "Disable dense vector component" + }, + { + "id": "rrf_lex_heavy", + "name": "RRF Lex Heavy (0.7/0.3)", + "args": ["--rrf-lex-weight", "0.7", "--rrf-dense-weight", "0.3"], + "description": "Weight lexical higher in RRF fusion" + }, + { + "id": "rrf_dense_heavy", + "name": "RRF Dense Heavy (0.3/0.7)", + "args": ["--rrf-lex-weight", "0.3", "--rrf-dense-weight", "0.7"], + "description": "Weight dense higher in RRF fusion" + }, + { + "id": "expand_query", + "name": "Query Expansion", + "args": ["--expand"], + "description": "Enable LLM query expansion" + }, + { + "id": "high_limit", + "name": "High Limit (50)", + "args": ["--limit", "50"], + "description": "Retrieve more candidates before reranking" + } + ] +} diff --git a/bench/copy-coll.py b/bench/copy-coll.py new file mode 100644 index 00000000..540069fb --- /dev/null +++ b/bench/copy-coll.py @@ -0,0 +1,533 @@ + +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import os +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +from qdrant_client import QdrantClient +from qdrant_client import models + +from fastembed import TextEmbedding + +from scripts.utils import sanitize_vector_name + + +def _env_int(name: str, default: int) -> int: + try: + v = os.environ.get(name) + if v is None: + return default + v = v.strip() + if not v: + return default + return int(v) + except Exception: + return default + + +def _env_str(name: str, default: str) -> str: + v = os.environ.get(name) + if v is None: + return default + v = v.strip() + return v or default + + +LEX_VECTOR_NAME = _env_str("LEX_VECTOR_NAME", "lex") +LEX_VECTOR_DIM = _env_int("LEX_VECTOR_DIM", 4096) +MINI_VECTOR_NAME = _env_str("MINI_VECTOR_NAME", "mini") +MINI_VEC_DIM = _env_int("MINI_VEC_DIM", 64) + + +try: + from scripts.utils import lex_hash_vector_text +except Exception: + lex_hash_vector_text = None + +try: + from scripts.ingest_code import project_mini +except Exception: + project_mini = None + + +@dataclass(frozen=True) +class CopyStats: + points_written: int + missing_dense: int + missing_lex: int + missing_mini: int + filled_lex: int + filled_mini: int + + +def _payload_text_for_embedding(payload: Dict[str, Any]) -> str: + info = payload.get("information") or payload.get("document") + if isinstance(info, str) and info.strip(): + return info + md = payload.get("metadata") or {} + if isinstance(md, dict): + code = md.get("code") + if isinstance(code, str) and code.strip(): + return code + return "" + + +def _payload_text_for_lex(payload: Dict[str, Any]) -> str: + md = payload.get("metadata") or {} + code = "" + if isinstance(md, dict): + v = md.get("code") + if isinstance(v, str): + code = v + pseudo = payload.get("pseudo") if isinstance(payload.get("pseudo"), str) else "" + tags = payload.get("tags") if isinstance(payload.get("tags"), list) else [] + tags_s = " ".join([str(t) for t in tags if t is not None]) if tags else "" + out = code + if pseudo: + out = (out + " " + pseudo) if out else pseudo + if tags_s: + out = (out + " " + tags_s) if out else tags_s + if out.strip(): + return out + return _payload_text_for_embedding(payload) + + +def _as_floats(v: Any) -> List[float]: + try: + if hasattr(v, "tolist"): + return list(v.tolist()) + except Exception: + pass + return list(v) + + +def _scroll_all( + client: QdrantClient, + collection: str, + *, + limit: int, +) -> Iterable[List[Any]]: + offset = None + while True: + points, offset = client.scroll( + collection_name=collection, + limit=limit, + offset=offset, + with_payload=True, + with_vectors=True, + ) + if not points: + break + yield points + if offset is None: + break + + +def _recreate_collection( + client: QdrantClient, + *, + name: str, + dense_vector_name: str, + dense_dim: int, + include_lex: bool, + include_mini: bool, + timeout_hint_s: int, +) -> None: + try: + client.delete_collection(name) + except Exception: + pass + + vectors_cfg: Dict[str, models.VectorParams] = { + dense_vector_name: models.VectorParams( + size=int(dense_dim), + distance=models.Distance.COSINE, + ) + } + if include_lex: + vectors_cfg[LEX_VECTOR_NAME] = models.VectorParams( + size=int(LEX_VECTOR_DIM), + distance=models.Distance.COSINE, + ) + if include_mini: + vectors_cfg[MINI_VECTOR_NAME] = models.VectorParams( + size=int(MINI_VEC_DIM), + distance=models.Distance.COSINE, + ) + + client.create_collection( + collection_name=name, + vectors_config=vectors_cfg, + hnsw_config=models.HnswConfigDiff(m=16, ef_construct=256), + ) + + print( + f"[created] {name} vectors={list(vectors_cfg.keys())} " + f"(timeout_hint_s={timeout_hint_s})", + flush=True, + ) + + +def clone_collection( + client: QdrantClient, + *, + src: str, + dest: str, + src_dense_vector_name: str, + dest_dense_vector_name: str, + scroll_batch: int, + upsert_batch: int, + upsert_wait: bool, + copy_lex: bool, + copy_mini: bool, + fill_missing_lex: bool, + fill_missing_mini: bool, +) -> CopyStats: + missing_dense = 0 + missing_lex = 0 + missing_mini = 0 + filled_lex = 0 + filled_mini = 0 + written = 0 + + if fill_missing_lex and lex_hash_vector_text is None: + raise RuntimeError("fill_missing_lex requested but scripts.utils.lex_hash_vector_text import failed") + if fill_missing_mini and project_mini is None: + raise RuntimeError("fill_missing_mini requested but scripts.ingest_code.project_mini import failed") + + t0 = time.time() + buf: List[models.PointStruct] = [] + + def flush_buf() -> None: + nonlocal written + if not buf: + return + client.upsert(collection_name=dest, points=buf, wait=upsert_wait) + written += len(buf) + buf.clear() + + for batch in _scroll_all(client, src, limit=scroll_batch): + for p in batch: + v = p.vector or {} + if not isinstance(v, dict): + missing_dense += 1 + continue + + dense = v.get(src_dense_vector_name) + if dense is None: + missing_dense += 1 + continue + + vecs: Dict[str, Any] = {dest_dense_vector_name: dense} + + lex = v.get(LEX_VECTOR_NAME) + mini = v.get(MINI_VECTOR_NAME) + + if copy_lex: + if lex is None: + missing_lex += 1 + if fill_missing_lex: + payload = p.payload or {} + lex_text = _payload_text_for_lex(payload) + lex = lex_hash_vector_text(lex_text) + filled_lex += 1 + if lex is not None: + vecs[LEX_VECTOR_NAME] = lex + + if copy_mini: + if mini is None: + missing_mini += 1 + if fill_missing_mini: + mini = project_mini(_as_floats(dense), MINI_VEC_DIM) + filled_mini += 1 + if mini is not None: + vecs[MINI_VECTOR_NAME] = mini + + buf.append(models.PointStruct(id=p.id, vector=vecs, payload=p.payload)) + if len(buf) >= upsert_batch: + flush_buf() + + flush_buf() + if written and written % 2048 == 0: + print(f"[clone] {dest}: written={written} elapsed={time.time()-t0:.1f}s", flush=True) + + flush_buf() + return CopyStats( + points_written=written, + missing_dense=missing_dense, + missing_lex=missing_lex, + missing_mini=missing_mini, + filled_lex=filled_lex, + filled_mini=filled_mini, + ) + + +def reembed_dense( + client: QdrantClient, + *, + src: str, + dest: str, + src_lex_vector_name: str, + src_mini_vector_name: str, + dest_dense_vector_name: str, + model_name: str, + scroll_batch: int, + embed_batch: int, + upsert_batch: int, + upsert_wait: bool, + copy_lex: bool, + copy_mini: bool, + fill_missing_lex: bool, + reembed_mini: bool, + fill_missing_mini: bool, +) -> CopyStats: + missing_dense = 0 + missing_lex = 0 + missing_mini = 0 + filled_lex = 0 + filled_mini = 0 + written = 0 + + if fill_missing_lex and lex_hash_vector_text is None: + raise RuntimeError("fill_missing_lex requested but scripts.utils.lex_hash_vector_text import failed") + if (reembed_mini or fill_missing_mini) and project_mini is None: + raise RuntimeError("mini generation requested but scripts.ingest_code.project_mini import failed") + + model = TextEmbedding(model_name=model_name) + t0 = time.time() + + pts_buf: List[models.PointStruct] = [] + + def flush_pts() -> None: + nonlocal written + if not pts_buf: + return + client.upsert(collection_name=dest, points=pts_buf, wait=upsert_wait) + written += len(pts_buf) + pts_buf.clear() + + for batch in _scroll_all(client, src, limit=scroll_batch): + items: List[Any] = [] + texts: List[str] = [] + for p in batch: + payload = p.payload or {} + texts.append(_payload_text_for_embedding(payload)) + items.append(p) + + for i in range(0, len(items), embed_batch): + sub_items = items[i : i + embed_batch] + sub_texts = texts[i : i + embed_batch] + dense_vecs = list(model.embed(sub_texts)) + + for p, dv in zip(sub_items, dense_vecs): + v = p.vector or {} + if not isinstance(v, dict): + missing_dense += 1 + continue + + vecs: Dict[str, Any] = {dest_dense_vector_name: _as_floats(dv)} + + lex = v.get(src_lex_vector_name) + mini = v.get(src_mini_vector_name) + + if copy_lex: + if lex is None: + missing_lex += 1 + if fill_missing_lex: + payload = p.payload or {} + lex_text = _payload_text_for_lex(payload) + lex = lex_hash_vector_text(lex_text) + filled_lex += 1 + if lex is not None: + vecs[LEX_VECTOR_NAME] = lex + + if copy_mini: + if reembed_mini: + mini = project_mini(vecs[dest_dense_vector_name], MINI_VEC_DIM) + filled_mini += 1 + else: + if mini is None: + missing_mini += 1 + if fill_missing_mini: + mini = project_mini(vecs[dest_dense_vector_name], MINI_VEC_DIM) + filled_mini += 1 + if mini is not None: + vecs[MINI_VECTOR_NAME] = mini + + pts_buf.append(models.PointStruct(id=p.id, vector=vecs, payload=p.payload)) + if len(pts_buf) >= upsert_batch: + flush_pts() + + flush_pts() + + if written and written % 1024 == 0: + print(f"[reembed] {dest}: written={written} elapsed={time.time()-t0:.1f}s", flush=True) + + flush_pts() + return CopyStats( + points_written=written, + missing_dense=missing_dense, + missing_lex=missing_lex, + missing_mini=missing_mini, + filled_lex=filled_lex, + filled_mini=filled_mini, + ) + + +def _probe_dim(model_name: str) -> int: + model = TextEmbedding(model_name=model_name) + return len(next(model.embed(["dimension probe"]))) + + +def main() -> None: + ap = argparse.ArgumentParser( + description="Clone a source Qdrant collection into benchmark collections (clone BGE; recreate + re-embed dense for MiniLM)." + ) + ap.add_argument("--qdrant-url", default=os.environ.get("QDRANT_URL", "http://qdrant:6333")) + ap.add_argument("--qdrant-timeout", type=int, default=int(os.environ.get("QDRANT_TIMEOUT", "180") or 180)) + ap.add_argument("--src", required=True, help="Source Qdrant collection name") + + ap.add_argument("--bge-dest", default="bench-bge") + ap.add_argument("--bge-model", default="BAAI/bge-base-en-v1.5") + ap.add_argument("--skip-bge", action="store_true") + + ap.add_argument("--minilm-dest", default="bench-minilm") + ap.add_argument("--minilm-model", default="sentence-transformers/all-MiniLM-L6-v2") + ap.add_argument("--skip-minilm", action="store_true") + + ap.add_argument("--scroll-batch", type=int, default=256) + ap.add_argument("--embed-batch", type=int, default=128) + ap.add_argument("--upsert-batch", type=int, default=256) + ap.add_argument("--upsert-wait", action="store_true", help="Wait for each upsert to be committed") + + ap.add_argument("--no-lex", action="store_true", help="Do not write lexical vectors") + ap.add_argument("--no-mini", action="store_true", help="Do not write mini vectors") + ap.add_argument("--fill-missing-lex", action="store_true", help="Compute lex vectors when missing in source") + ap.add_argument( + "--fill-missing-mini", + action="store_true", + help="Compute mini vectors from dense when missing in source", + ) + ap.add_argument( + "--reembed-mini", + action="store_true", + help="When re-embedding MiniLM dense, also recompute mini vector from the new dense vector", + ) + + args = ap.parse_args() + + include_lex = not args.no_lex + include_mini = not args.no_mini + + client = QdrantClient(url=args.qdrant_url, timeout=args.qdrant_timeout) + src_info = client.get_collection(args.src) + src_vectors = src_info.config.params.vectors + if not isinstance(src_vectors, dict): + raise RuntimeError(f"Source collection {args.src} does not use named vectors") + + bge_dense_name = sanitize_vector_name(args.bge_model) + minilm_dense_name = sanitize_vector_name(args.minilm_model) + + src_dense_candidates = [bge_dense_name] + [k for k in src_vectors.keys() if k not in {LEX_VECTOR_NAME, MINI_VECTOR_NAME}] + src_dense_name = None + for k in src_dense_candidates: + if k in src_vectors: + src_dense_name = k + break + if not src_dense_name: + raise RuntimeError(f"Could not determine source dense vector name in {args.src}. vectors={list(src_vectors.keys())}") + + print( + f"[src] {args.src} points={src_info.points_count} vectors={list(src_vectors.keys())} src_dense={src_dense_name}", + flush=True, + ) + + if not args.skip_bge: + bge_dim = getattr(src_vectors.get(src_dense_name), "size", None) or _probe_dim(args.bge_model) + _recreate_collection( + client, + name=args.bge_dest, + dense_vector_name=bge_dense_name, + dense_dim=int(bge_dim), + include_lex=include_lex, + include_mini=include_mini, + timeout_hint_s=args.qdrant_timeout, + ) + t0 = time.time() + stats = clone_collection( + client, + src=args.src, + dest=args.bge_dest, + src_dense_vector_name=src_dense_name, + dest_dense_vector_name=bge_dense_name, + scroll_batch=args.scroll_batch, + upsert_batch=args.upsert_batch, + upsert_wait=args.upsert_wait, + copy_lex=include_lex, + copy_mini=include_mini, + fill_missing_lex=args.fill_missing_lex, + fill_missing_mini=args.fill_missing_mini, + ) + print( + f"[done] clone -> {args.bge_dest}: points={stats.points_written} " + f"elapsed={time.time()-t0:.1f}s missing=(dense={stats.missing_dense},lex={stats.missing_lex},mini={stats.missing_mini}) " + f"filled=(lex={stats.filled_lex},mini={stats.filled_mini})", + flush=True, + ) + + if not args.skip_minilm: + minilm_dim = _probe_dim(args.minilm_model) + _recreate_collection( + client, + name=args.minilm_dest, + dense_vector_name=minilm_dense_name, + dense_dim=int(minilm_dim), + include_lex=include_lex, + include_mini=include_mini, + timeout_hint_s=args.qdrant_timeout, + ) + t0 = time.time() + stats = reembed_dense( + client, + src=args.src, + dest=args.minilm_dest, + src_lex_vector_name=LEX_VECTOR_NAME, + src_mini_vector_name=MINI_VECTOR_NAME, + dest_dense_vector_name=minilm_dense_name, + model_name=args.minilm_model, + scroll_batch=min(args.scroll_batch, 256), + embed_batch=args.embed_batch, + upsert_batch=args.upsert_batch, + upsert_wait=args.upsert_wait, + copy_lex=include_lex, + copy_mini=include_mini, + fill_missing_lex=args.fill_missing_lex, + reembed_mini=args.reembed_mini, + fill_missing_mini=args.fill_missing_mini, + ) + print( + f"[done] reembed -> {args.minilm_dest}: points={stats.points_written} " + f"elapsed={time.time()-t0:.1f}s missing=(dense={stats.missing_dense},lex={stats.missing_lex},mini={stats.missing_mini}) " + f"filled=(lex={stats.filled_lex},mini={stats.filled_mini})", + flush=True, + ) + + print("[ok]", flush=True) + + +if __name__ == "__main__": + main() + diff --git a/bench/datasets/public_v1.json b/bench/datasets/public_v1.json new file mode 100644 index 00000000..454c8035 --- /dev/null +++ b/bench/datasets/public_v1.json @@ -0,0 +1,33 @@ +{ + "id": "ctx-bench-v1", + "version": "1.0.0", + "description": "Public benchmark dataset for Context-Engine retrieval quality evaluation", + "created": "2024-12-29", + "repos": [ + { + "name": "kubernetes/kubernetes", + "ref": "v1.28.0", + "sha": null, + "language": ["go", "yaml"], + "domain": "infrastructure", + "size_hint": "large" + }, + { + "name": "microsoft/vscode", + "ref": "1.86.0", + "sha": null, + "language": ["typescript", "javascript"], + "domain": "editor", + "size_hint": "large" + }, + { + "name": "huggingface/transformers", + "ref": "v4.37.0", + "sha": null, + "language": ["python"], + "domain": "ml", + "size_hint": "large" + } + ], + "notes": "SHA fields populated by clone_snapshot.py after cloning" +} diff --git a/bench/doc.md b/bench/doc.md new file mode 100644 index 00000000..3afac52c --- /dev/null +++ b/bench/doc.md @@ -0,0 +1,319 @@ + +# Bench: Copy/Re-embed Collections + +This folder contains small utilities to help benchmark different embedding models against the same underlying code/payload. + +## What we did (repeatable) + +We took an existing Qdrant collection (example: `Context-Engine-41e67959`) and produced two benchmark collections: + +- `bench-bge` + - Recreated with the **BGE** dense vector params + - Populated by **cloning** points (payload + dense vectors) from the source collection + - Lexical (`lex`) + mini (`mini`) vectors are copied when present + +- `bench-minilm` + - Recreated with the **MiniLM** dense vector params (384-d) + - Populated by **re-embedding** dense vectors from the source payload text using `sentence-transformers/all-MiniLM-L6-v2` + - Lexical (`lex`) + mini (`mini`) vectors are copied when present (or optionally regenerated) + +This produces apples-to-apples retrieval comparisons where only the **dense embedding model** differs. + +## Script + +- `bench/copy-coll.py` + - Recreates destination collections with correct vector schemas + - Clones or re-embeds data in batches with progress output + - Uses the same vector-name conventions as the core code (`scripts.utils.sanitize_vector_name`) + +## Prereqs + +- Qdrant running (the dev-remote compose stack is fine) +- The `mcp_indexer` container has Python deps (`qdrant-client`, `fastembed`, `onnxruntime`) +- Source collection exists in Qdrant + +## How to run (dev-remote docker compose) + +### Rebuild (required after updating `bench/`) + +If you changed anything under `bench/` (including `copy-coll.py`), rebuild the indexer image so `/app/bench` exists in the container: + +```bash +docker compose -f docker-compose.dev-remote.yml up -d --build mcp_indexer +``` + +Run inside the `mcp_indexer` container so model caches and deps match indexing/search: + +```bash +docker compose -f docker-compose.dev-remote.yml exec -T mcp_indexer sh -lc ' + export QDRANT_URL=http://qdrant:6333 + export QDRANT_TIMEOUT=180 + python /app/bench/copy-coll.py \ + --src Context-Engine-41e67959 \ + --bge-dest bench-bge \ + --minilm-dest bench-minilm +' +``` + +If your source collection has missing `lex`/`mini` vectors and you want to fill them: + +```bash +docker compose -f docker-compose.dev-remote.yml exec -T mcp_indexer sh -lc ' + export QDRANT_URL=http://qdrant:6333 + export QDRANT_TIMEOUT=180 + python /app/bench/copy-coll.py \ + --src Context-Engine-41e67959 \ + --fill-missing-lex \ + --fill-missing-mini +' +``` + +### Useful knobs + +- `--scroll-batch N` + - Qdrant scroll page size +- `--embed-batch N` + - MiniLM embed batch size +- `--upsert-batch N` + - Upsert batch size +- `--upsert-wait` + - Wait for each upsert; slower but easier to reason about during debugging + +## Why MiniLM can take longer here + +Even though MiniLM is a smaller model (384-d vs 768-d), in this workflow MiniLM has to: + +- Scroll every point +- Extract payload text +- Run the embedding model for every point +- Upsert new dense vectors + +Whereas BGE cloning is mostly: + +- Scroll points +- Upsert vectors/payload as-is + +So **copying** will usually beat **re-embedding**, regardless of which model you re-embed with. + +If you want a fair “model speed” comparison, measure: + +- Embedding throughput alone (texts/sec) +- Query latency on already-built collections + +## Notes on `lex` and `mini` + +- `lex` is a fixed-size lexical hashing vector (default dim `4096`). +- `mini` is a compact vector used for gating in ReFRAG-style mode (default dim `64`). +- Some source points may not have `lex`/`mini` populated (we saw this in `Context-Engine-41e67959`). + - The script safely skips `None` vectors. + - Optionally you can fill missing vectors via `--fill-missing-lex` / `--fill-missing-mini`. + +## Evaluation loop (BGE vs MiniLM) + +The simplest way to compare the two collections is to run the same query against each and measure latency. + +Example (run in `mcp_indexer`): + +```bash +docker compose -f docker-compose.dev-remote.yml exec -T mcp_indexer sh -lc ' + export QDRANT_URL=http://qdrant:6333 + export QDRANT_TIMEOUT=180 + + for C in bench-bge bench-minilm; do + echo "\n== $C ==" + /usr/bin/time -f "elapsed=%Es cpu=%P maxrss=%MKB" \ + python /app/scripts/hybrid_search.py \ + --collection "$C" \ + --limit 8 \ + -q "how does repo_search combine dense and lexical" \ + -q "where is sanitize_vector_name defined" \ + -q "how is multi repo collection name derived" \ + --json \ + >/tmp/${C}.json + echo "saved /tmp/${C}.json" + done +' +``` + +Notes: + +- This uses the existing hybrid search path (dense + lexical + optional boosts). +- For more stable numbers, run multiple times and take p50/p95. + +### Preferred: `bench/eval.py` (warmup + repeats + overlap) + +`hybrid_search.py` uses `EMBEDDING_MODEL` for **query embedding**, so for a fair comparison we run it twice: + +- `bench-bge` with `EMBEDDING_MODEL=BAAI/bge-base-en-v1.5` +- `bench-minilm` with `EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2` + +The harness below does: + +- warmup runs (discarded) +- N measured runs per target +- p50/p95 latency +- top-k overlap (Jaccard on result paths) between collections + +```bash +docker compose -f docker-compose.dev-remote.yml exec -T mcp_indexer sh -lc ' + export QDRANT_URL=http://qdrant:6333 + export QDRANT_TIMEOUT=180 + python /app/bench/eval.py \ + --target bench-bge BAAI/bge-base-en-v1.5 \ + --target bench-minilm sentence-transformers/all-MiniLM-L6-v2 \ + --query "how does repo_search combine dense and lexical" \ + --query "where is sanitize_vector_name defined" \ + --query "how is multi repo collection name derived" \ + --repeats 10 \ + --warmup 1 \ + --limit 8 \ + --topk 8 \ + --per-path 1 \ + --json-out /tmp/bench_eval.json +' +``` + +To load queries from a file (one query per line): + +```bash +python /app/bench/eval.py ... --query-file /work/.codebase/bench_queries.txt +``` + +To pass extra flags through to `hybrid_search.py`, you must append them after `--` (otherwise `eval.py` errors): + +```bash +python /app/bench/eval.py ... -- --expand +``` + +## How to decide “ship MiniLM for low-end devices” + +Treat this as a multi-metric decision with explicit gates. + +### Performance gates + +- **Query p50/p95** + - Compare `bench-bge` vs `bench-minilm` on a warmed system (warmup then repeats) + - Goal for “low-end”: meaningfully lower p95 (tail latency matters most) + +- **Embedding throughput (queries/sec)** + - Measure standalone query embedding speed for BGE vs MiniLM (no Qdrant) + - Goal: MiniLM should be faster per query on CPU + +- **Memory footprint** + - Measure RSS of the process serving search (and the embedding model) on a constrained machine + - Goal: MiniLM should reduce RSS enough to matter + +### Quality gates + +- **Hit@k against a small gold set** + - Write ~30–100 representative queries and expected file/symbol hints + - Score “did an expected path appear in top-k?” + - Gate: MiniLM should not regress more than an agreed threshold (e.g., <= 5% absolute hit@k loss) + +- **Overlap sanity check** + - The harness reports top-k path overlap. Overlap being high is good, but it’s not a substitute for a gold set. + +### Recommendation for next iteration + +- Create a stable `bench_queries.txt` and run `bench/eval.py --query-file ...` with `--repeats 20 --warmup 2`. +- Add a separate embedding-only microbench (we can add `bench/embed_bench.py`) to isolate model speed from Qdrant/search overhead. + +## Future work (not doing now) + +The work so far was **plumbing / dirty validation**: + +- Can we build separate collections per embedding model? +- Can we run the same query set against both and see stable-ish timings? + +To make a defensible call like “MiniLM is good/fast enough to ship for low-end devices”, add a proper evaluation workflow: + +### 1) Separate benchmarks by component + +- **Embedding-only throughput** + - Measure texts/sec and p50/p95 embed latency for each model (no Qdrant). + - This is the true model CPU cost; it should be the main win for MiniLM. + +- **Query-time latency (warm)** + - Use `bench/eval.py` with warmups and higher repeats (e.g. `--warmup 2 --repeats 50`). + - Report p50/p95, not just p50. + +- **Index-time throughput** + - Measure indexing wall time, chunks/sec, and peak memory. + +### 2) Memory + footprint gates + +- **Process RSS** + - Measure RSS for the embedding process (and MCP server) with each model. + - Low-end devices are often memory-bound. + +- **Model size / cache impact** + - Track on-disk cache size and first-run download time. + +### 3) Quality gates + +- **Gold-set hit@k** + - Maintain a query file plus expected paths/symbols. + - Score hit@k (e.g. hit@8) for each model. + - Define an acceptable regression threshold. + +- **Manual review slice** + - Pick a smaller set of representative “hard” queries and manually compare top-k. + +### 4) Decide with explicit thresholds + +- **Latency** + - “Ship for low-end” target might be: p95 improves by X% and doesn’t exceed Y seconds. +- **Quality** + - hit@k within threshold; no major regressions on critical queries. +- **Footprint** + - RSS reduction is meaningful on the target hardware. + +### Exact rebuild+eval command used (copy/paste) + +(Legacy; the preferred approach is `bench/eval.py` above.) + +This is the exact command that rebuilt `mcp_indexer`, verified `/app/bench`, and ran a 3-repeat eval against both `bench-bge` and `bench-minilm`: + +```bash +docker compose -f docker-compose.dev-remote.yml up -d --build mcp_indexer && \ +docker compose -f docker-compose.dev-remote.yml exec -T mcp_indexer sh -lc ' +set -e +ls -al /app/bench +export QDRANT_URL=http://qdrant:6333 +export QDRANT_TIMEOUT=180 +export BENCH_REPEATS=${BENCH_REPEATS:-3} +python - <<"PY" +import os, subprocess, sys, time, statistics + +collections = ["bench-bge", "bench-minilm"] +queries = [ + "how does repo_search combine dense and lexical", + "where is sanitize_vector_name defined", + "how is multi repo collection name derived", +] +repeats = int(os.environ.get("BENCH_REPEATS", "3") or 3) + +print("QDRANT_URL", os.environ.get("QDRANT_URL")) +print("repeats", repeats) + +for coll in collections: + times = [] + for r in range(repeats): + cmd = [sys.executable, "/app/scripts/hybrid_search.py", "--collection", coll, "--limit", "8", "--json"] + for q in queries: + cmd.extend(["-q", q]) + out_path = f"/tmp/{coll}.run{r}.json" + t0 = time.time() + with open(out_path, "w", encoding="utf-8") as f: + subprocess.run(cmd, check=True, stdout=f, env=os.environ.copy()) + dt = time.time() - t0 + times.append(dt) + print(f"{coll} run{r}: {dt:.3f}s -> {out_path}") + + p50 = statistics.median(times) + print(f"{coll} summary: n={len(times)} p50={p50:.3f}s min={min(times):.3f}s max={max(times):.3f}s") +PY +' +``` + + diff --git a/bench/eval.py b/bench/eval.py new file mode 100644 index 00000000..73214431 --- /dev/null +++ b/bench/eval.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import subprocess +import sys +import time +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence, Tuple + + +@dataclass(frozen=True) +class Target: + collection: str + model_name: str + + +@dataclass(frozen=True) +class RunResult: + seconds: float + results: List[Dict[str, Any]] + + +def _parse_json_lines(text: str) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + for raw in (text or "").splitlines(): + line = raw.strip() + if not line: + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict) and "results" in obj and isinstance(obj.get("results"), list): + return list(obj.get("results") or []) + if isinstance(obj, dict): + out.append(obj) + return out + + +def _run_hybrid_search( + *, + collection: str, + model_name: str, + queries: Sequence[str], + limit: int, + per_path: int, + expand: bool, + extra_args: Sequence[str], + env_base: Dict[str, str], +) -> RunResult: + cmd: List[str] = [ + sys.executable, + "/app/scripts/hybrid_search.py", + "--collection", + collection, + "--limit", + str(limit), + "--per-path", + str(per_path), + "--json", + ] + if expand: + cmd.append("--expand") + for q in queries: + cmd.extend(["-q", q]) + cmd.extend(list(extra_args)) + + env = dict(env_base) + env["EMBEDDING_MODEL"] = model_name + + t0 = time.time() + proc = subprocess.run(cmd, capture_output=True, text=True, env=env) + dt = time.time() - t0 + if proc.returncode != 0: + raise RuntimeError(proc.stderr.strip() or proc.stdout.strip() or f"hybrid_search failed: rc={proc.returncode}") + return RunResult(seconds=dt, results=_parse_json_lines(proc.stdout)) + + +def _topk_paths(results: Sequence[Dict[str, Any]], k: int) -> List[str]: + out: List[str] = [] + for r in results: + p = r.get("path") + if isinstance(p, str) and p: + out.append(p) + if len(out) >= k: + break + return out + + +def _percentile(xs: Sequence[float], p: float) -> float: + if not xs: + return 0.0 + s = sorted(xs) + if len(s) == 1: + return s[0] + i = (len(s) - 1) * p + lo = int(i) + hi = min(lo + 1, len(s) - 1) + frac = i - lo + return s[lo] * (1.0 - frac) + s[hi] * frac + + +def _jaccard(a: Sequence[str], b: Sequence[str]) -> float: + sa = set(a) + sb = set(b) + if not sa and not sb: + return 1.0 + inter = len(sa & sb) + union = len(sa | sb) + return (inter / union) if union else 0.0 + + +def main() -> None: + ap = argparse.ArgumentParser(description="Evaluate hybrid_search latency + top-k overlap across collections/models") + ap.add_argument( + "--target", + action="append", + nargs=2, + metavar=("COLLECTION", "MODEL"), + required=True, + help="Pair: collection name and EMBEDDING_MODEL to use for query embedding", + ) + ap.add_argument("--query", action="append", default=[]) + ap.add_argument("--query-file", type=str, default=None) + ap.add_argument("--repeats", type=int, default=int(os.environ.get("BENCH_REPEATS", "10") or 10)) + ap.add_argument("--warmup", type=int, default=1) + ap.add_argument("--limit", type=int, default=8) + ap.add_argument("--topk", type=int, default=8) + ap.add_argument("--per-path", type=int, default=1) + ap.add_argument("--expand", action="store_true", default=False) + ap.add_argument("--json-out", type=str, default=None) + + args, extra = ap.parse_known_args() + if extra: + if extra[:1] != ["--"]: + raise SystemExit( + "Unknown args: " + + " ".join(extra) + + "\nIf you intended to pass flags through to hybrid_search.py, put them after '--'." + ) + extra = extra[1:] + + targets = [Target(collection=c, model_name=m) for c, m in (args.target or [])] + if len(targets) < 2: + raise SystemExit("need at least two --target pairs") + + queries = [q for q in (args.query or []) if str(q).strip()] + if args.query_file: + try: + with open(args.query_file, "r", encoding="utf-8") as f: + for raw in f.read().splitlines(): + line = (raw or "").strip() + if not line: + continue + queries.append(line) + except Exception as e: + raise SystemExit(f"Failed to read --query-file {args.query_file}: {e}") + if not queries: + queries = [ + "how does repo_search combine dense and lexical", + "where is sanitize_vector_name defined", + "how is multi repo collection name derived", + "where is auth and acl, where is it enforced?", + "How does the CTX system work? How does extension integrate with it?" + ] + + print(f"queries={len(queries)}", flush=True) + for i, q in enumerate(queries): + print(f"q{i}: {q}", flush=True) + + env_base = dict(os.environ) + + for t in targets: + for _ in range(max(0, int(args.warmup))): + _run_hybrid_search( + collection=t.collection, + model_name=t.model_name, + queries=queries, + limit=args.limit, + per_path=args.per_path, + expand=args.expand, + extra_args=extra, + env_base=env_base, + ) + + summary: Dict[str, Any] = { + "queries": list(queries), + "repeats": int(args.repeats), + "warmup": int(args.warmup), + "limit": int(args.limit), + "topk": int(args.topk), + "per_path": int(args.per_path), + "expand": bool(args.expand), + "hybrid_search_args": list(extra), + "targets": [{"collection": t.collection, "model": t.model_name} for t in targets], + "results": {}, + "overlap": {}, + } + + per_target_runs: Dict[str, List[RunResult]] = {} + + for t in targets: + runs: List[RunResult] = [] + for i in range(int(args.repeats)): + r = _run_hybrid_search( + collection=t.collection, + model_name=t.model_name, + queries=queries, + limit=args.limit, + per_path=args.per_path, + expand=args.expand, + extra_args=extra, + env_base=env_base, + ) + runs.append(r) + print(f"{t.collection} run{i}: {r.seconds:.3f}s", flush=True) + per_target_runs[t.collection] = runs + + times = [x.seconds for x in runs] + rec = { + "n": len(times), + "min": min(times) if times else 0.0, + "max": max(times) if times else 0.0, + "p50": statistics.median(times) if times else 0.0, + "p95": _percentile(times, 0.95) if times else 0.0, + } + summary["results"][t.collection] = rec + + base = targets[0].collection + for other in [t.collection for t in targets[1:]]: + overlaps: List[float] = [] + for i in range(min(len(per_target_runs[base]), len(per_target_runs[other]))): + a = _topk_paths(per_target_runs[base][i].results, args.topk) + b = _topk_paths(per_target_runs[other][i].results, args.topk) + overlaps.append(_jaccard(a, b)) + summary["overlap"][f"{base}__vs__{other}"] = { + "n": len(overlaps), + "p50": statistics.median(overlaps) if overlaps else 0.0, + "p95": _percentile(overlaps, 0.95) if overlaps else 0.0, + "min": min(overlaps) if overlaps else 0.0, + "max": max(overlaps) if overlaps else 0.0, + } + + print(json.dumps(summary, ensure_ascii=False)) + + if args.json_out: + with open(args.json_out, "w", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False, indent=2)) + f.write("\n") + + +if __name__ == "__main__": + main() diff --git a/bench/eval_quality.py b/bench/eval_quality.py new file mode 100644 index 00000000..fa3a8a56 --- /dev/null +++ b/bench/eval_quality.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +""" +Quality evaluation harness for retrieval benchmarks. + +Computes Hit@k and MRR against a gold query set. + +Usage: + python bench/eval_quality.py \ + --collection ctx-public-v1-bge \ + --model "BAAI/bge-base-en-v1.5" \ + --gold-file bench/gold/public_v1.queries.jsonl \ + --limit 20 + + # With config toggles + python bench/eval_quality.py \ + --collection ctx-public-v1-bge \ + --model "BAAI/bge-base-en-v1.5" \ + --gold-file bench/gold/public_v1.queries.jsonl \ + --config-id no_rerank \ + -- --no-rerank +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Set + +BENCH_DIR = Path(__file__).resolve().parent +REPO_ROOT = BENCH_DIR.parent + + +@dataclass +class GoldQuery: + """A gold-standard query with expected relevant files.""" + id: str + query: str + repo: str + relevant: List[str] # List of relative paths + task_type: str = "navigation" + k: int = 20 + notes: str = "" + + +@dataclass +class QueryResult: + """Result of evaluating a single query.""" + query_id: str + query: str + retrieved: List[str] # Paths returned by search + relevant: Set[str] # Gold relevant paths + hits: Dict[int, bool] # hit@k for various k + rr: float # reciprocal rank + latency_s: float + + +@dataclass +class EvalResult: + """Aggregated evaluation results.""" + dataset_id: str + collection: str + model: str + config_id: str + n_queries: int + metrics: Dict[str, float] + per_query: List[Dict[str, Any]] = field(default_factory=list) + timestamp: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + "dataset_id": self.dataset_id, + "collection": self.collection, + "model": self.model, + "config_id": self.config_id, + "n_queries": self.n_queries, + "metrics": self.metrics, + "per_query": self.per_query, + "timestamp": self.timestamp, + } + + +def load_gold_queries(path: Path, repo_filter: Optional[str] = None) -> List[GoldQuery]: + """Load gold queries from JSONL file.""" + queries = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + obj = json.loads(line) + + if repo_filter and obj.get("repo") != repo_filter: + continue + + relevant_paths = [r["path"] for r in obj.get("relevant", [])] + + queries.append(GoldQuery( + id=obj["id"], + query=obj["query"], + repo=obj["repo"], + relevant=relevant_paths, + task_type=obj.get("task_type", "navigation"), + k=obj.get("k", 20), + notes=obj.get("notes", ""), + )) + return queries + + +def normalize_path(path: str) -> str: + """Normalize path for comparison (remove leading ./ and trailing /).""" + p = path.strip() + while p.startswith("./"): + p = p[2:] + while p.startswith("/"): + p = p[1:] + return p.rstrip("/") + + +def run_search( + query: str, + collection: str, + model: str, + limit: int, + extra_args: Sequence[str], + env_base: Dict[str, str], +) -> tuple[List[str], float]: + """Run hybrid search and return retrieved paths + latency.""" + cmd = [ + sys.executable, + str(REPO_ROOT / "scripts" / "hybrid_search.py"), + "--collection", collection, + "--limit", str(limit), + "--per-path", "1", + "--json", + "-q", query, + ] + cmd.extend(list(extra_args)) + + env = dict(env_base) + env["EMBEDDING_MODEL"] = model + + t0 = time.time() + proc = subprocess.run(cmd, capture_output=True, text=True, env=env) + latency = time.time() - t0 + + if proc.returncode != 0: + print(f"[eval] WARNING: search failed for query: {query[:50]}...") + return [], latency + + # Parse results + paths = [] + for line in proc.stdout.strip().splitlines(): + if not line.strip(): + continue + try: + obj = json.loads(line) + if "results" in obj: + for r in obj["results"]: + if "path" in r: + paths.append(normalize_path(r["path"])) + elif "path" in obj: + paths.append(normalize_path(obj["path"])) + except json.JSONDecodeError: + continue + + return paths, latency + + +def compute_hits(retrieved: List[str], relevant: Set[str], k_values: List[int]) -> Dict[int, bool]: + """Compute hit@k for various k values.""" + hits = {} + for k in k_values: + top_k = set(retrieved[:k]) + hits[k] = bool(top_k & relevant) + return hits + + +def compute_rr(retrieved: List[str], relevant: Set[str]) -> float: + """Compute reciprocal rank (1/rank of first relevant result).""" + for i, path in enumerate(retrieved): + if path in relevant: + return 1.0 / (i + 1) + return 0.0 + + +def evaluate_query( + gold: GoldQuery, + collection: str, + model: str, + limit: int, + extra_args: Sequence[str], + env_base: Dict[str, str], + k_values: List[int], +) -> QueryResult: + """Evaluate a single gold query.""" + retrieved, latency = run_search( + query=gold.query, + collection=collection, + model=model, + limit=limit, + extra_args=extra_args, + env_base=env_base, + ) + + # Normalize relevant paths + relevant = {normalize_path(p) for p in gold.relevant} + + # For matching, we check if retrieved path ends with any relevant path + # (handles cases where collection has full paths but gold has relative) + def matches_relevant(retrieved_path: str, relevant_set: Set[str]) -> bool: + rp = normalize_path(retrieved_path) + for rel in relevant_set: + if rp.endswith(rel) or rel.endswith(rp) or rp == rel: + return True + return False + + # Convert retrieved to matched/unmatched + matched_retrieved = [] + for rp in retrieved: + if matches_relevant(rp, relevant): + matched_retrieved.append(rp) + + hits = compute_hits(retrieved, relevant, k_values) + + # Recompute hits using fuzzy matching + hits_fuzzy = {} + for k in k_values: + top_k = retrieved[:k] + hits_fuzzy[k] = any(matches_relevant(p, relevant) for p in top_k) + + # Compute RR with fuzzy matching + rr = 0.0 + for i, path in enumerate(retrieved): + if matches_relevant(path, relevant): + rr = 1.0 / (i + 1) + break + + return QueryResult( + query_id=gold.id, + query=gold.query, + retrieved=retrieved[:limit], + relevant=relevant, + hits=hits_fuzzy, + rr=rr, + latency_s=latency, + ) + + +def aggregate_results( + results: List[QueryResult], + k_values: List[int], +) -> Dict[str, float]: + """Aggregate per-query results into summary metrics.""" + if not results: + return {} + + n = len(results) + metrics = {} + + # Hit@k (mean over queries) + for k in k_values: + hits = [1.0 if r.hits.get(k, False) else 0.0 for r in results] + metrics[f"hit@{k}"] = sum(hits) / n + + # MRR (mean reciprocal rank) + mrr = sum(r.rr for r in results) / n + metrics["mrr"] = round(mrr, 4) + + # Latency stats + latencies = [r.latency_s for r in results] + metrics["latency_mean"] = round(sum(latencies) / n, 4) + metrics["latency_p50"] = round(sorted(latencies)[n // 2], 4) + if n >= 20: + metrics["latency_p95"] = round(sorted(latencies)[int(n * 0.95)], 4) + + # Round hit@k + for k in k_values: + metrics[f"hit@{k}"] = round(metrics[f"hit@{k}"], 4) + + return metrics + + +def main(): + parser = argparse.ArgumentParser( + description="Evaluate retrieval quality against gold query set" + ) + parser.add_argument( + "--collection", required=True, + help="Qdrant collection to evaluate" + ) + parser.add_argument( + "--model", required=True, + help="Embedding model name (for EMBEDDING_MODEL env var)" + ) + parser.add_argument( + "--gold-file", required=True, + help="Path to gold queries JSONL file" + ) + parser.add_argument( + "--repo", type=str, default=None, + help="Filter queries to specific repo (e.g., kubernetes/kubernetes)" + ) + parser.add_argument( + "--limit", type=int, default=20, + help="Number of results to retrieve per query" + ) + parser.add_argument( + "--k-values", type=str, default="1,3,5,10", + help="Comma-separated k values for hit@k" + ) + parser.add_argument( + "--config-id", type=str, default="default", + help="Config variant identifier for output" + ) + parser.add_argument( + "--dataset-id", type=str, default=None, + help="Dataset ID for output (defaults to gold file stem)" + ) + parser.add_argument( + "--output", type=str, default=None, + help="Output JSON file path" + ) + parser.add_argument( + "--verbose", action="store_true", + help="Print per-query results" + ) + + args, extra = parser.parse_known_args() + + # Handle -- separator for extra args to hybrid_search + if extra and extra[0] == "--": + extra = extra[1:] + + # Parse k values + k_values = [int(k.strip()) for k in args.k_values.split(",")] + + # Load gold queries + gold_path = Path(args.gold_file) + if not gold_path.exists(): + print(f"ERROR: Gold file not found: {gold_path}") + sys.exit(1) + + queries = load_gold_queries(gold_path, repo_filter=args.repo) + if not queries: + print(f"ERROR: No queries loaded from {gold_path}") + sys.exit(1) + + dataset_id = args.dataset_id or gold_path.stem + + print(f"[eval] Collection: {args.collection}") + print(f"[eval] Model: {args.model}") + print(f"[eval] Gold queries: {len(queries)}") + print(f"[eval] Config: {args.config_id}") + print(f"[eval] k values: {k_values}") + if extra: + print(f"[eval] Extra args: {extra}") + + # Run evaluation + env_base = dict(os.environ) + results: List[QueryResult] = [] + + for i, gold in enumerate(queries): + result = evaluate_query( + gold=gold, + collection=args.collection, + model=args.model, + limit=args.limit, + extra_args=extra, + env_base=env_base, + k_values=k_values, + ) + results.append(result) + + if args.verbose: + hit_str = " ".join(f"@{k}:{'✓' if result.hits.get(k) else '✗'}" for k in k_values) + print(f" [{i+1}/{len(queries)}] {gold.id}: RR={result.rr:.2f} {hit_str}") + else: + print(f" [{i+1}/{len(queries)}] {gold.id}: RR={result.rr:.2f}", end="\r") + + print() # Clear line + + # Aggregate + metrics = aggregate_results(results, k_values) + + # Build output + eval_result = EvalResult( + dataset_id=dataset_id, + collection=args.collection, + model=args.model, + config_id=args.config_id, + n_queries=len(queries), + metrics=metrics, + per_query=[ + { + "id": r.query_id, + "rr": r.rr, + "hits": {f"@{k}": v for k, v in r.hits.items()}, + "latency_s": round(r.latency_s, 4), + } + for r in results + ], + timestamp=datetime.now().isoformat(), + ) + + # Print summary + print("\n" + "=" * 60) + print("QUALITY METRICS") + print("=" * 60) + print(f" MRR: {metrics.get('mrr', 0):.4f}") + for k in k_values: + print(f" Hit@{k}: {metrics.get(f'hit@{k}', 0):.4f}") + print(f" Latency (mean): {metrics.get('latency_mean', 0):.3f}s") + print(f" Latency (p50): {metrics.get('latency_p50', 0):.3f}s") + + # Output JSON + output_dict = eval_result.to_dict() + print(json.dumps(output_dict)) + + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(output_dict, f, indent=2) + print(f"\n[eval] Results saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/bench/gold/public_v1.queries.jsonl b/bench/gold/public_v1.queries.jsonl new file mode 100644 index 00000000..38a6fcff --- /dev/null +++ b/bench/gold/public_v1.queries.jsonl @@ -0,0 +1,30 @@ +{"id": "k8s-eviction-001", "query": "where is the pod eviction logic when a node is under memory pressure", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/kubelet/eviction/eviction_manager.go"}, {"path": "pkg/kubelet/eviction/helpers.go"}], "task_type": "navigation", "notes": "Core eviction manager handles memory pressure signals"} +{"id": "k8s-scheduler-001", "query": "how does the scheduler pick which node to place a pod on", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/scheduler/scheduler.go"}, {"path": "pkg/scheduler/framework/runtime/framework.go"}], "task_type": "conceptual", "notes": "Scheduler uses framework plugins for node selection"} +{"id": "k8s-service-001", "query": "where are kubernetes service endpoints managed", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/controller/endpoint/endpoints_controller.go"}, {"path": "pkg/controller/endpointslice/endpointslice_controller.go"}], "task_type": "navigation", "notes": "EndpointSlice is the modern approach"} +{"id": "k8s-admission-001", "query": "how do admission webhooks validate pod creation", "repo": "kubernetes/kubernetes", "relevant": [{"path": "staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/plugin.go"}, {"path": "staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go"}], "task_type": "conceptual", "notes": "Validating admission webhooks"} +{"id": "k8s-configmap-001", "query": "how are configmaps mounted into pods as volumes", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/volume/configmap/configmap.go"}, {"path": "pkg/kubelet/configmap/configmap_manager.go"}], "task_type": "conceptual", "notes": "ConfigMap volume plugin"} +{"id": "k8s-hpa-001", "query": "where is horizontal pod autoscaler scaling logic", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/controller/podautoscaler/horizontal.go"}, {"path": "pkg/controller/podautoscaler/replica_calculator.go"}], "task_type": "navigation", "notes": "HPA controller calculates desired replicas"} +{"id": "k8s-dns-001", "query": "how does kubernetes resolve service DNS names", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/kubelet/network/dns/dns.go"}, {"path": "pkg/apis/core/types.go"}], "task_type": "conceptual", "notes": "Kubelet configures pod DNS"} +{"id": "k8s-pv-001", "query": "where is persistent volume claim binding logic", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/controller/volume/persistentvolume/pv_controller.go"}, {"path": "pkg/controller/volume/persistentvolume/binder.go"}], "task_type": "navigation", "notes": "PV controller handles binding"} +{"id": "k8s-probe-001", "query": "how does kubelet execute liveness probes", "repo": "kubernetes/kubernetes", "relevant": [{"path": "pkg/kubelet/prober/prober.go"}, {"path": "pkg/kubelet/prober/prober_manager.go"}], "task_type": "conceptual", "notes": "Prober runs HTTP/exec/TCP probes"} +{"id": "k8s-rbac-001", "query": "where is RBAC authorization enforced", "repo": "kubernetes/kubernetes", "relevant": [{"path": "plugin/pkg/auth/authorizer/rbac/rbac.go"}, {"path": "staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory/builtin.go"}], "task_type": "navigation", "notes": "RBAC authorizer implementation"} +{"id": "vscode-ext-001", "query": "how does vscode load and activate extensions", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/workbench/services/extensions/browser/extensionService.ts"}, {"path": "src/vs/workbench/services/extensions/common/extensionHostManager.ts"}], "task_type": "conceptual", "notes": "Extension service manages lifecycle"} +{"id": "vscode-cmd-001", "query": "where is the command palette implemented", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/workbench/contrib/quickaccess/browser/commandsQuickAccess.ts"}, {"path": "src/vs/platform/quickinput/browser/quickInput.ts"}], "task_type": "navigation", "notes": "Quick access provides command palette"} +{"id": "vscode-editor-001", "query": "how does the text editor handle syntax highlighting", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/editor/common/languages/supports/tokenization.ts"}, {"path": "src/vs/editor/standalone/browser/standaloneLanguages.ts"}], "task_type": "conceptual", "notes": "Tokenization drives highlighting"} +{"id": "vscode-file-001", "query": "where is file watching implemented for the explorer", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/platform/files/node/watcher/watcherMain.ts"}, {"path": "src/vs/platform/files/node/watcher/parcel/parcelWatcher.ts"}], "task_type": "navigation", "notes": "Parcel watcher for file system events"} +{"id": "vscode-git-001", "query": "how does vscode integrate with git for source control", "repo": "microsoft/vscode", "relevant": [{"path": "extensions/git/src/git.ts"}, {"path": "extensions/git/src/repository.ts"}], "task_type": "conceptual", "notes": "Git extension wraps git CLI"} +{"id": "vscode-debug-001", "query": "where is the debug adapter protocol implemented", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/workbench/contrib/debug/browser/debugService.ts"}, {"path": "src/vs/workbench/contrib/debug/common/debugProtocol.ts"}], "task_type": "navigation", "notes": "Debug service implements DAP"} +{"id": "vscode-autocomplete-001", "query": "how does intellisense autocomplete work", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/editor/contrib/suggest/browser/suggestController.ts"}, {"path": "src/vs/editor/contrib/suggest/browser/suggestModel.ts"}], "task_type": "conceptual", "notes": "Suggest controller manages completions"} +{"id": "vscode-settings-001", "query": "where are user settings stored and loaded", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/platform/configuration/common/configurationService.ts"}, {"path": "src/vs/workbench/services/configuration/browser/configurationService.ts"}], "task_type": "navigation", "notes": "Configuration service handles settings"} +{"id": "vscode-terminal-001", "query": "how is the integrated terminal implemented", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/workbench/contrib/terminal/browser/terminalService.ts"}, {"path": "src/vs/workbench/contrib/terminal/browser/terminalInstance.ts"}], "task_type": "conceptual", "notes": "Terminal service manages xterm instances"} +{"id": "vscode-search-001", "query": "where is workspace text search implemented", "repo": "microsoft/vscode", "relevant": [{"path": "src/vs/workbench/services/search/node/ripgrepTextSearchEngine.ts"}, {"path": "src/vs/workbench/services/search/common/searchService.ts"}], "task_type": "navigation", "notes": "Ripgrep powers text search"} +{"id": "hf-tokenizer-001", "query": "how does the tokenizer encode text into tokens", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/tokenization_utils_base.py"}, {"path": "src/transformers/tokenization_utils.py"}], "task_type": "conceptual", "notes": "Base tokenizer handles encoding"} +{"id": "hf-trainer-001", "query": "where is the training loop implemented", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/trainer.py"}, {"path": "src/transformers/training_args.py"}], "task_type": "navigation", "notes": "Trainer class runs training"} +{"id": "hf-pipeline-001", "query": "how do inference pipelines work for text generation", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/pipelines/text_generation.py"}, {"path": "src/transformers/pipelines/base.py"}], "task_type": "conceptual", "notes": "Pipeline provides easy inference API"} +{"id": "hf-attention-001", "query": "where is multi-head attention implemented", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/models/bert/modeling_bert.py"}, {"path": "src/transformers/models/gpt2/modeling_gpt2.py"}], "task_type": "navigation", "notes": "Each model has attention implementation"} +{"id": "hf-config-001", "query": "how are model configurations loaded from hub", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/configuration_utils.py"}, {"path": "src/transformers/utils/hub.py"}], "task_type": "conceptual", "notes": "PretrainedConfig handles loading"} +{"id": "hf-generate-001", "query": "where is beam search text generation implemented", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/generation/utils.py"}, {"path": "src/transformers/generation/beam_search.py"}], "task_type": "navigation", "notes": "GenerationMixin has generate()"} +{"id": "hf-dataset-001", "query": "how does the data collator batch training examples", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/data/data_collator.py"}, {"path": "src/transformers/trainer_utils.py"}], "task_type": "conceptual", "notes": "DataCollator pads and batches"} +{"id": "hf-quantize-001", "query": "where is model quantization to 8bit implemented", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/integrations/bitsandbytes.py"}, {"path": "src/transformers/utils/quantization_config.py"}], "task_type": "navigation", "notes": "BitsAndBytes integration for quantization"} +{"id": "hf-save-001", "query": "how are pretrained models saved to disk", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/modeling_utils.py"}, {"path": "src/transformers/configuration_utils.py"}], "task_type": "conceptual", "notes": "save_pretrained() in PreTrainedModel"} +{"id": "hf-peft-001", "query": "where is LoRA adapter loading implemented", "repo": "huggingface/transformers", "relevant": [{"path": "src/transformers/integrations/peft.py"}, {"path": "src/transformers/modeling_utils.py"}], "task_type": "navigation", "notes": "PEFT integration for adapters"} diff --git a/bench/queries.txt b/bench/queries.txt new file mode 100644 index 00000000..478a5d1a --- /dev/null +++ b/bench/queries.txt @@ -0,0 +1,29 @@ +# Gold Query Set for Context-Engine Benchmarks +# Format: query text (one per line) +# Comments start with # + +# Core search functionality +hybrid search RRF ranking implementation +memory store implementation details +recursive reranker learning workflow + +# Embeddings and models +embedder model loading and caching +where is the embedding dimension configured + +# MCP and routing +MCP router intent classification +symbol graph callers query + +# Context and grounding +context answer grounded citations +deduplication request fingerprint + +# Infrastructure +workspace state persistence +openlit initialization tracing + +# Advanced features +upload service delta bundle +pattern search structural matching +query expansion pseudo relevance feedback diff --git a/bench/run_matrix.py b/bench/run_matrix.py new file mode 100644 index 00000000..b6a2789c --- /dev/null +++ b/bench/run_matrix.py @@ -0,0 +1,550 @@ +#!/usr/bin/env python3 +""" +Unified Benchmark Matrix Pipeline. + +Orchestrates corpus-based comparison across embedding models and rerankers: +1. Requires source collection (fail-fast) +2. Builds variant collections (copy/re-embed per model) +3. Runs eval harness + benchmarks per variant +4. Outputs comparison matrix + +Usage: + python bench/run_matrix.py \ + --src Context-Engine-41e67959 \ + --models "BAAI/bge-base-en-v1.5,sentence-transformers/all-MiniLM-L6-v2" \ + --benchmarks "eval,router,rrf" \ + --output results/bench_matrix.json +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +@dataclass +class ModelVariant: + """A benchmark variant with a specific embedding model.""" + name: str # e.g., "bench-bge" + model: str # e.g., "BAAI/bge-base-en-v1.5" + collection: str # Qdrant collection name + + +@dataclass +class ConfigVariant: + """A config variant for A/B testing.""" + id: str # e.g., "no_rerank" + name: str # e.g., "No Reranker" + args: List[str] # e.g., ["--no-rerank"] + description: str = "" + + +def load_configs(config_file: Optional[str]) -> List[ConfigVariant]: + """Load config variants from JSON file.""" + if not config_file: + return [ConfigVariant(id="default", name="Default", args=[])] + + path = Path(config_file) + if not path.exists(): + print(f"[matrix] WARNING: Config file not found: {config_file}") + return [ConfigVariant(id="default", name="Default", args=[])] + + with open(path) as f: + data = json.load(f) + + configs = [] + for cfg in data.get("configs", []): + configs.append(ConfigVariant( + id=cfg["id"], + name=cfg.get("name", cfg["id"]), + args=cfg.get("args", []), + description=cfg.get("description", ""), + )) + + return configs if configs else [ConfigVariant(id="default", name="Default", args=[])] + + +@dataclass +class MatrixResult: + """Results from running the benchmark matrix.""" + source: str + timestamp: str + models: List[str] + benchmarks: List[str] + matrix: Dict[str, Dict[str, Any]] = field(default_factory=dict) + comparison: Dict[str, Any] = field(default_factory=dict) + errors: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + return { + "source": self.source, + "timestamp": self.timestamp, + "models": self.models, + "benchmarks": self.benchmarks, + "matrix": self.matrix, + "comparison": self.comparison, + "errors": self.errors, + } + + +def require_source_collection(collection: str, qdrant_url: str, min_points: int = 100) -> int: + """Fail-fast: require source collection to exist with sufficient points.""" + from qdrant_client import QdrantClient + + client = QdrantClient(url=qdrant_url) + try: + info = client.get_collection(collection) + count = int(info.points_count or 0) + except Exception as e: + raise ValueError(f"Source collection '{collection}' not accessible: {e}") + + if count < min_points: + raise ValueError( + f"Source collection '{collection}' has {count} points, " + f"need at least {min_points} for meaningful benchmarks." + ) + + print(f"[matrix] Source collection '{collection}' verified: {count} points") + return count + + +def build_variant_collections( + src: str, + models: List[str], + qdrant_url: str, + skip_existing: bool = True, +) -> List[ModelVariant]: + """Build variant collections for each embedding model.""" + from scripts.utils import sanitize_vector_name + from qdrant_client import QdrantClient + + client = QdrantClient(url=qdrant_url) + variants = [] + + for model in models: + # Generate collection name from model + safe_name = sanitize_vector_name(model) + coll_name = f"bench-{safe_name[:30]}" # Truncate for sanity + + # Check if already exists + try: + info = client.get_collection(coll_name) + count = int(info.points_count or 0) + if skip_existing and count > 0: + print(f"[matrix] Variant '{coll_name}' exists with {count} points, skipping rebuild") + variants.append(ModelVariant(name=safe_name, model=model, collection=coll_name)) + continue + except Exception: + pass # Collection doesn't exist, will create + + # Build variant via copy-coll.py + print(f"[matrix] Building variant '{coll_name}' with model '{model}'...") + + # Determine if this is the first model (clone) or needs re-embed + is_first = (model == models[0]) + + cmd = [ + sys.executable, + str(REPO_ROOT / "bench" / "copy-coll.py"), + "--qdrant-url", qdrant_url, + "--src", src, + ] + + if is_first: + # First model: clone as-is (assume source uses this model) + cmd.extend(["--bge-dest", coll_name, "--bge-model", model, "--skip-minilm"]) + else: + # Other models: re-embed + cmd.extend(["--minilm-dest", coll_name, "--minilm-model", model, "--skip-bge"]) + + t0 = time.time() + proc = subprocess.run(cmd, capture_output=True, text=True) + dt = time.time() - t0 + + if proc.returncode != 0: + print(f"[matrix] ERROR building {coll_name}: {proc.stderr[:500]}") + continue + + print(f"[matrix] Built '{coll_name}' in {dt:.1f}s") + variants.append(ModelVariant(name=safe_name, model=model, collection=coll_name)) + + return variants + + +def run_benchmarks_for_variant( + variant: ModelVariant, + benchmarks: List[str], + qdrant_url: str, + query_file: Optional[str] = None, + gold_file: Optional[str] = None, + repeats: int = 5, + warmup: int = 1, + config_id: str = "default", + extra_args: Optional[List[str]] = None, +) -> Dict[str, Any]: + """Run selected benchmarks against a specific variant collection.""" + results: Dict[str, Any] = { + "model": variant.model, + "collection": variant.collection, + } + + # Set up environment for this variant + env = os.environ.copy() + env["COLLECTION_NAME"] = variant.collection + env["EMBEDDING_MODEL"] = variant.model + env["QDRANT_URL"] = qdrant_url + env["BENCH_STRICT"] = "1" # Fail-fast mode + + # Run benchmarks via run_all.py or directly + if benchmarks: + bench_components = ",".join(benchmarks) + cmd = [ + sys.executable, + str(REPO_ROOT / "scripts" / "benchmarks" / "run_all.py"), + "--components", + ] + benchmarks + + print(f"[matrix] Running benchmarks {benchmarks} for {variant.collection}...") + t0 = time.time() + proc = subprocess.run(cmd, capture_output=True, text=True, env=env) + dt = time.time() - t0 + + results["benchmark_time_s"] = round(dt, 2) + + if proc.returncode == 0: + # Try to parse JSON from output + for line in reversed(proc.stdout.strip().splitlines()): + if line.startswith("{"): + try: + results["benchmarks"] = json.loads(line) + break + except json.JSONDecodeError: + pass + else: + results["error"] = proc.stderr[:500] if proc.stderr else "Unknown error" + + # Run quality eval if gold file provided + if gold_file: + print(f"[matrix] Running quality eval for {variant.collection}...") + quality_cmd = [ + sys.executable, + str(REPO_ROOT / "bench" / "eval_quality.py"), + "--collection", variant.collection, + "--model", variant.model, + "--gold-file", gold_file, + "--config-id", config_id, + "--limit", "20", + ] + if extra_args: + quality_cmd.append("--") + quality_cmd.extend(extra_args) + + proc = subprocess.run(quality_cmd, capture_output=True, text=True, env=env) + if proc.returncode == 0: + for line in reversed(proc.stdout.strip().splitlines()): + if line.startswith("{"): + try: + quality_data = json.loads(line) + results["quality"] = quality_data.get("metrics", {}) + break + except json.JSONDecodeError: + pass + else: + results["quality_error"] = proc.stderr[:300] if proc.stderr else "Unknown error" + + # Run latency eval with repeats + print(f"[matrix] Running latency eval for {variant.collection}...") + eval_cmd = [ + sys.executable, + str(REPO_ROOT / "bench" / "eval.py"), + "--target", variant.collection, variant.model, + "--target", variant.collection, variant.model, # Dummy second target (required) + "--repeats", str(repeats), + "--warmup", str(warmup), + ] + if query_file: + eval_cmd.extend(["--query-file", query_file]) + + proc = subprocess.run(eval_cmd, capture_output=True, text=True, env=env) + if proc.returncode == 0: + for line in reversed(proc.stdout.strip().splitlines()): + if line.startswith("{"): + try: + latency_data = json.loads(line) + results["latency"] = latency_data.get("results", {}).get(variant.collection, {}) + break + except json.JSONDecodeError: + pass + + return results + + +def compute_comparison(matrix: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + """Compute comparison metrics across variants.""" + if len(matrix) < 2: + return {} + + comparison: Dict[str, Any] = {} + + # Extract MRR values (prefer quality metrics from eval_quality.py) + mrr_values = {} + hit_at_5 = {} + latency_p50 = {} + for name, data in matrix.items(): + # First try quality metrics from eval_quality.py + quality = data.get("quality", {}) + if quality.get("mrr"): + mrr_values[name] = quality["mrr"] + if quality.get("hit@5"): + hit_at_5[name] = quality["hit@5"] + else: + # Fallback to legacy benchmarks + benchmarks = data.get("benchmarks", {}).get("components", {}) + if "eval_harness" in benchmarks: + mrr = benchmarks["eval_harness"].get("metrics", {}).get("mrr", 0) + mrr_values[name] = mrr + lat = data.get("latency", {}).get("p50", 0) + if lat: + latency_p50[name] = lat + + if mrr_values: + best_mrr_name = max(mrr_values, key=lambda k: mrr_values[k]) + worst_mrr_name = min(mrr_values, key=lambda k: mrr_values[k]) + comparison["mrr"] = { + "best": {"name": best_mrr_name, "value": mrr_values[best_mrr_name]}, + "worst": {"name": worst_mrr_name, "value": mrr_values[worst_mrr_name]}, + "delta": round(mrr_values[best_mrr_name] - mrr_values[worst_mrr_name], 4), + } + + if latency_p50: + fastest_name = min(latency_p50, key=lambda k: latency_p50[k]) + slowest_name = max(latency_p50, key=lambda k: latency_p50[k]) + if latency_p50[slowest_name] > 0: + improvement_pct = (1 - latency_p50[fastest_name] / latency_p50[slowest_name]) * 100 + else: + improvement_pct = 0 + comparison["latency"] = { + "fastest": {"name": fastest_name, "p50": latency_p50[fastest_name]}, + "slowest": {"name": slowest_name, "p50": latency_p50[slowest_name]}, + "improvement_pct": round(improvement_pct, 1), + } + + # Generate recommendation + if comparison.get("mrr") and comparison.get("latency"): + mrr_best = comparison["mrr"]["best"]["name"] + lat_best = comparison["latency"]["fastest"]["name"] + if mrr_best == lat_best: + comparison["recommendation"] = f"{mrr_best} wins on both quality and latency" + else: + comparison["recommendation"] = ( + f"{mrr_best} wins on quality (MRR); " + f"{lat_best} wins on latency" + ) + + return comparison + + +async def run_matrix( + src: str, + models: List[str], + benchmarks: List[str], + qdrant_url: str, + query_file: Optional[str] = None, + gold_file: Optional[str] = None, + config_file: Optional[str] = None, + repeats: int = 5, + warmup: int = 1, + skip_existing: bool = True, + min_points: int = 100, +) -> MatrixResult: + """Run the full benchmark matrix over (model × config) pairs.""" + # Load config variants + configs = load_configs(config_file) + + result = MatrixResult( + source=src, + timestamp=datetime.now().isoformat(), + models=models, + benchmarks=benchmarks, + ) + + # Step 1: Verify source collection + try: + require_source_collection(src, qdrant_url, min_points) + except ValueError as e: + result.errors.append(str(e)) + return result + + # Step 2: Build variant collections (one per model) + variants = build_variant_collections(src, models, qdrant_url, skip_existing) + if not variants: + result.errors.append("No variant collections could be built") + return result + + # Step 3: Run benchmarks for each (model × config) pair + for variant in variants: + for config in configs: + # Create unique key for this (model, config) combination + if len(configs) > 1: + matrix_key = f"{variant.name}__{config.id}" + else: + matrix_key = variant.name + + print(f"\n{'='*60}") + print(f"[matrix] Evaluating: {variant.name} + {config.name}") + print(f"[matrix] Config args: {config.args}") + print(f"{'='*60}") + + variant_results = run_benchmarks_for_variant( + variant, benchmarks, qdrant_url, query_file, gold_file, + repeats, warmup, config.id, config.args if config.args else None + ) + variant_results["config_id"] = config.id + variant_results["config_name"] = config.name + result.matrix[matrix_key] = variant_results + + # Step 4: Compute comparison + result.comparison = compute_comparison(result.matrix) + + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Run unified benchmark matrix across embedding models" + ) + parser.add_argument( + "--src", required=True, + help="Source collection to use as benchmark corpus" + ) + parser.add_argument( + "--models", default="BAAI/bge-base-en-v1.5,sentence-transformers/all-MiniLM-L6-v2", + help="Comma-separated list of embedding models to compare" + ) + parser.add_argument( + "--benchmarks", default="eval,router", + help="Comma-separated list of benchmarks to run (eval,router,rrf,refrag,grounding)" + ) + parser.add_argument( + "--qdrant-url", default=os.environ.get("QDRANT_URL", "http://localhost:6333"), + help="Qdrant server URL" + ) + parser.add_argument( + "--query-file", type=str, default=None, + help="Path to query file for latency eval (one query per line)" + ) + parser.add_argument( + "--gold-file", type=str, default=None, + help="Path to gold queries JSONL for quality eval (Hit@k, MRR)" + ) + parser.add_argument( + "--config-file", type=str, default=None, + help="Path to A/B config JSON file (e.g., bench/configs/ctx_ab_configs.json)" + ) + parser.add_argument( + "--repeats", type=int, default=5, + help="Number of benchmark repeats for latency stats" + ) + parser.add_argument( + "--warmup", type=int, default=1, + help="Number of warmup runs before measurement" + ) + parser.add_argument( + "--rebuild", action="store_true", + help="Rebuild variant collections even if they exist" + ) + parser.add_argument( + "--output", type=str, default=None, + help="Output JSON file path" + ) + parser.add_argument( + "--min-points", type=int, default=100, + help="Minimum points required in source collection (default: 100)" + ) + + args = parser.parse_args() + + models = [m.strip() for m in args.models.split(",") if m.strip()] + benchmarks = [b.strip() for b in args.benchmarks.split(",") if b.strip()] + + print(f"[matrix] Source: {args.src}") + print(f"[matrix] Models: {models}") + print(f"[matrix] Benchmarks: {benchmarks}") + print(f"[matrix] Qdrant: {args.qdrant_url}") + if args.gold_file: + print(f"[matrix] Gold file: {args.gold_file}") + if args.config_file: + print(f"[matrix] Config file: {args.config_file}") + + result = asyncio.run(run_matrix( + src=args.src, + models=models, + benchmarks=benchmarks, + qdrant_url=args.qdrant_url, + query_file=args.query_file, + gold_file=args.gold_file, + config_file=args.config_file, + repeats=args.repeats, + warmup=args.warmup, + skip_existing=not args.rebuild, + min_points=args.min_points, + )) + + # Print summary + print(f"\n{'='*60}") + print("MATRIX RESULTS") + print(f"{'='*60}") + + for name, data in result.matrix.items(): + print(f"\n{name}:") + if "latency" in data: + lat = data["latency"] + print(f" Latency: p50={lat.get('p50', 0):.3f}s p95={lat.get('p95', 0):.3f}s") + # Print quality metrics from eval_quality.py + if "quality" in data: + q = data["quality"] + print(f" MRR: {q.get('mrr', 0):.4f} Hit@5: {q.get('hit@5', 0):.4f}") + else: + # Fallback to legacy benchmarks + benchmarks_data = data.get("benchmarks", {}).get("components", {}) + if "eval_harness" in benchmarks_data: + metrics = benchmarks_data["eval_harness"].get("metrics", {}) + print(f" MRR: {metrics.get('mrr', 0):.3f}") + + if result.comparison: + print(f"\nCOMPARISON:") + if "recommendation" in result.comparison: + print(f" → {result.comparison['recommendation']}") + + if result.errors: + print(f"\nERRORS:") + for e in result.errors: + print(f" ❌ {e}") + + # Write output + output = result.to_dict() + print(json.dumps(output, indent=2)) + + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(output, f, indent=2) + print(f"\n[matrix] Results saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/build-images.sh b/build-images.sh new file mode 100644 index 00000000..2cc3cd3a --- /dev/null +++ b/build-images.sh @@ -0,0 +1,226 @@ +#!/bin/bash +# Docker Build Script for Context-Engine +# Builds all service images with custom registry tagging + +set -euo pipefail + +# Configuration +REGISTRY="192.168.96.61:30009/library" +PROJECT_NAME="context-engine" +TAG="${TAG:-latest}" + +# Service mapping (service_name:dockerfile:final_image_name) +declare -A SERVICES=( + ["memory"]="Dockerfile.mcp:${PROJECT_NAME}-memory" + ["indexer"]="Dockerfile.mcp-indexer:${PROJECT_NAME}-indexer" + ["indexer-service"]="Dockerfile.indexer:${PROJECT_NAME}-indexer-service" + ["llamacpp"]="Dockerfile.llamacpp:${PROJECT_NAME}-llamacpp" +) + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Build function +build_image() { + local service=$1 + local dockerfile=$2 + local image_name=$3 + local full_image="${REGISTRY}/${image_name}:${TAG}" + + log_info "Building ${service} service..." + log_info "Dockerfile: ${dockerfile}" + log_info "Image: ${full_image}" + + if ! docker build \ + -f "${dockerfile}" \ + -t "${full_image}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + .; then + log_error "Failed to build ${service} image" + return 1 + fi + + log_info "Successfully built ${service} image: ${full_image}" + + # Push if registry is accessible + if [[ "${PUSH_IMAGES:-false}" == "true" ]]; then + log_info "Pushing ${service} image..." + if ! docker push "${full_image}"; then + log_warn "Failed to push ${service} image (registry may be inaccessible)" + return 1 + fi + log_info "Successfully pushed ${service} image" + fi + + echo "${full_image}" +} + +# Main build process +main() { + log_info "Starting Context-Engine Docker build process..." + log_info "Registry: ${REGISTRY}" + log_info "Tag: ${TAG}" + log_info "Push enabled: ${PUSH_IMAGES:-false}" + echo + + # Check if Docker is running + if ! docker info >/dev/null 2>&1; then + log_error "Docker is not running or not accessible" + exit 1 + fi + + # Check if Dockerfiles exist + for service in "${!SERVICES[@]}"; do + IFS=':' read -r dockerfile image_name <<< "${SERVICES[$service]}" + if [[ ! -f "${dockerfile}" ]]; then + log_error "Dockerfile not found: ${dockerfile}" + exit 1 + fi + done + + local built_images=() + local failed_services=() + + # Build each service + for service in "${!SERVICES[@]}"; do + IFS=':' read -r dockerfile image_name <<< "${SERVICES[$service]}" + + if built_image=$(build_image "$service" "$dockerfile" "$image_name"); then + built_images+=("$built_image") + else + failed_services+=("$service") + fi + echo + done + + # Summary + log_info "Build Summary:" + log_info "Successfully built: ${#built_images[@]} images" + for img in "${built_images[@]}"; do + log_info " ✓ ${img}" + done + + if [[ ${#failed_services[@]} -gt 0 ]]; then + log_error "Failed to build: ${#failed_services[@]} services" + for service in "${failed_services[@]}"; do + log_error " ✗ ${service}" + done + exit 1 + fi + + log_info "All images built successfully!" + + # Generate updated kustomization.yaml + cat > "deploy/kubernetes/kustomization-images.yaml" << 'EOF' +# Image overrides for Context-Engine Kubernetes deployment +# Use this with: kustomize build . --load-restrictor=LoadRestrictionsNone | kubectl apply -f - +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - configmap.yaml + - qdrant.yaml + - mcp-memory.yaml + - mcp-indexer.yaml + - mcp-http.yaml + - indexer-services.yaml + - llamacpp.yaml + - ingress.yaml + +images: +EOF + + # Add images to kustomization + for service in "${!SERVICES[@]}"; do + IFS=':' read -r dockerfile image_name <<< "${SERVICES[$service]}" + full_image="${REGISTRY}/${image_name}:${TAG}" + cat >> "deploy/kubernetes/kustomization-images.yaml" << EOF + - name: ${image_name} + newName: ${full_image%:*} # Remove tag + newTag: ${TAG} +EOF + done + + cat >> "deploy/kubernetes/kustomization-images.yaml" << 'EOF' + +# Common labels +commonLabels: + app.kubernetes.io/name: context-engine + app.kubernetes.io/component: kubernetes-deployment + app.kubernetes.io/managed-by: kustomize + +# Namespace override +namespace: context-engine +EOF + + log_info "Generated deploy/kubernetes/kustomization-images.yaml" + log_info "To deploy: kustomize build deploy/kubernetes/ | kubectl apply -f -" +} + +# Help function +show_help() { + cat << EOF +Context-Engine Docker Build Script + +Usage: $0 [OPTIONS] + +Options: + -t, --tag TAG Set image tag (default: latest) + -p, --push Push images to registry after build + -h, --help Show this help message + +Examples: + $0 # Build with default tag + $0 -t v1.0.0 # Build with custom tag + $0 --push # Build and push to registry + TAG=dev-branch $0 # Build using environment variable + +Environment Variables: + TAG Image tag to use + PUSH_IMAGES Set to 'true' to push after build + +Registry Configuration: + Current registry: ${REGISTRY} + To change: modify REGISTRY variable in script + +Generated Files: + - deploy/kubernetes/kustomization-images.yaml + Contains image references for Kubernetes deployment + +EOF +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -t|--tag) + TAG="$2" + shift 2 + ;; + -p|--push) + export PUSH_IMAGES=true + shift + ;; + -h|--help) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Run main function +main "$@" \ No newline at end of file diff --git a/config/clickhouse-config.xml b/config/clickhouse-config.xml new file mode 100644 index 00000000..fb1dc0b8 --- /dev/null +++ b/config/clickhouse-config.xml @@ -0,0 +1,47 @@ + + + + 0.0.0.0 + + + warning + true + + + + 6 + 120000 + 604800 + + + + warning + 604800 + 120000 + + + + 1000 + 60000 + 604800 + + + + 60000 + 120000 + 604800 + + + + 60000 + 120000 + 604800 + + + + warning + 120000 + 604800 + + + diff --git a/config/docker_related_config.xml b/config/docker_related_config.xml new file mode 100644 index 00000000..29c0aaa3 --- /dev/null +++ b/config/docker_related_config.xml @@ -0,0 +1,6 @@ + + + 0.0.0.0 + 1 + + diff --git a/config/otel-collector-config.yaml b/config/otel-collector-config.yaml new file mode 100644 index 00000000..30628149 --- /dev/null +++ b/config/otel-collector-config.yaml @@ -0,0 +1,49 @@ +# OpenTelemetry Collector configuration for OpenLit +# Receives telemetry from openlit SDK and exports to ClickHouse + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + memory_limiter: + limit_mib: 1500 + spike_limit_mib: 512 + check_interval: 5s + +exporters: + clickhouse: + endpoint: tcp://${env:INIT_DB_HOST}:9000?dial_timeout=10s + database: ${env:INIT_DB_DATABASE} + username: ${env:INIT_DB_USERNAME} + password: ${env:INIT_DB_PASSWORD} + ttl: 730h + logs_table_name: otel_logs + traces_table_name: otel_traces + metrics_table_name: otel_metrics + timeout: 5s + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + +service: + pipelines: + logs: + receivers: [otlp] + processors: [batch] + exporters: [clickhouse] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [clickhouse] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [clickhouse] diff --git a/ctx-hook-simple.sh b/ctx-hook-simple.sh new file mode 100755 index 00000000..e6c07bd4 --- /dev/null +++ b/ctx-hook-simple.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +# Simplified Claude Code UserPromptSubmit hook for ctx.py +# Takes JSON input from Claude Code and outputs enhanced prompt + +# Read JSON input from stdin +INPUT=$(cat) + +# Extract the prompt text from Claude's JSON payload +if command -v jq >/dev/null 2>&1; then + USER_MESSAGE=$(echo "$INPUT" | jq -r '.prompt') + USER_CWD=$(echo "$INPUT" | jq -r '.cwd // empty') +else + # Fallback: treat entire input as the prompt text + USER_MESSAGE="$INPUT" +fi + +# Skip if empty message +if [ -z "$USER_MESSAGE" ] || [ "$USER_MESSAGE" = "null" ]; then + echo "$INPUT" + exit 0 +fi + +# Set working directory to where the hook script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Determine workspace directory: +# - If CTX_WORKSPACE_DIR is already set, honor it. +# - If running from an embedded extension under ~/.windsurf-server/extensions, +# default to the caller's CWD (Claude/VS Code workspace root). +# - Otherwise (repo-local hook), default to the script directory so it works +# even when Claude runs from a parent folder. +if [ -n "${CTX_WORKSPACE_DIR:-}" ]; then + WORKSPACE_DIR="$CTX_WORKSPACE_DIR" +elif [[ "$SCRIPT_DIR" == */.windsurf-server/extensions/* ]]; then + WORKSPACE_DIR="$PWD" +else + WORKSPACE_DIR="$SCRIPT_DIR" +fi +export CTX_WORKSPACE_DIR="$WORKSPACE_DIR" + +# If the workspace root does not contain ctx_config.json, but exactly one +# direct child directory does, treat that child directory as the effective +# workspace. This supports multi-repo workspaces where the ctx-enabled repo +# (with ctx_config.json and .env) lives one level below the VS Code root. +if [ ! -f "$WORKSPACE_DIR/ctx_config.json" ]; then + FOUND_SUBDIR="" + for candidate in "$WORKSPACE_DIR"/*; do + if [ -d "$candidate" ] && [ -f "$candidate/ctx_config.json" ]; then + if [ -z "$FOUND_SUBDIR" ]; then + FOUND_SUBDIR="$candidate" + else + # More than one candidate; ambiguous, keep original WORKSPACE_DIR + FOUND_SUBDIR="" + break + fi + fi + done + if [ -n "$FOUND_SUBDIR" ]; then + WORKSPACE_DIR="$FOUND_SUBDIR" + export CTX_WORKSPACE_DIR="$WORKSPACE_DIR" + fi +fi + +# Prefer workspace-level ctx_config.json, fall back to one next to the script +if [ -f "$WORKSPACE_DIR/ctx_config.json" ]; then + CONFIG_FILE="$WORKSPACE_DIR/ctx_config.json" +elif [ -f "$SCRIPT_DIR/ctx_config.json" ]; then + CONFIG_FILE="$SCRIPT_DIR/ctx_config.json" +else + CONFIG_FILE="" +fi + +# Optional: enable file logging when CTX_HOOK_LOG=1 or a .ctx_hook_log marker +# file exists in the workspace. When disabled, no log file is written. +if [ "${CTX_HOOK_LOG:-0}" = "1" ] || [ -f "$WORKSPACE_DIR/.ctx_hook_log" ]; then + LOG_FILE="$WORKSPACE_DIR/ctx-hook.log" + LOG_ENABLED=1 +else + LOG_ENABLED=0 +fi + +cd "$SCRIPT_DIR" + +# Optional: enable extra debug information in the JSON payload +# when CTX_HOOK_DEBUG=1 is set in the environment, or when a +# .ctx_hook_debug marker file exists in the workspace. +CTX_HOOK_DEBUG="${CTX_HOOK_DEBUG:-}" +if [ -z "$CTX_HOOK_DEBUG" ] && [ -f "$WORKSPACE_DIR/.ctx_hook_debug" ]; then + CTX_HOOK_DEBUG="1" +fi + +# Log the incoming payload when logging is enabled +if [ "$LOG_ENABLED" = "1" ]; then + { + echo "[$(date -Iseconds)] HOOK INVOKED" + echo "PWD = $PWD" + echo "WORKSPACE_DIR = $WORKSPACE_DIR" + echo "INPUT = <> "$LOG_FILE" +fi + +# Read all settings from ctx_config.json +if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then + CTX_COLLECTION=$(grep -o '"default_collection"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"default_collection"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' ) + REFRAG_RUNTIME=$(grep -o '"refrag_runtime"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"refrag_runtime"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' || echo "glm") + GLM_API_KEY=$(grep -o '"glm_api_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"glm_api_key"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' ) + GLM_API_BASE=$(grep -o '"glm_api_base"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"glm_api_base"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/') + GLM_MODEL=$(grep -o '"glm_model"[[:space:]]*:[[:space:]]*"[^\"]*"' "$CONFIG_FILE" | sed 's/.*"glm_model"[[:space:]]*:[[:space:]]*"\([^\"]*\)".*/\1/' || echo "glm-4.6") + CTX_DEFAULT_MODE=$(grep -o '"default_mode"[[:space:]]*:[[:space:]]*"[^\"]*"' "$CONFIG_FILE" | sed 's/.*"default_mode"[[:space:]]*:[[:space:]]*"\([^\"]*\)".*/\1/') + CTX_REQUIRE_CONTEXT=$(grep -o '"require_context"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"require_context"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/') + CTX_RELEVANCE_GATE=$(grep -o '"relevance_gate_enabled"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"relevance_gate_enabled"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/') + CTX_MIN_RELEVANCE=$(grep -o '"min_relevance"[[:space:]]*:[[:space:]]*[0-9.][0-9.]*' "$CONFIG_FILE" | sed 's/.*"min_relevance"[[:space:]]*:[[:space:]]*\([0-9.][0-9.]*\).*/\1/') + CTX_REWRITE_MAX_TOKENS=$(grep -o '"rewrite_max_tokens"[[:space:]]*:[[:space:]]*[0-9][0-9]*' "$CONFIG_FILE" | sed 's/.*"rewrite_max_tokens"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/') + CTX_SURFACE_COLLECTION_CFG=$(grep -o '"surface_qdrant_collection_hint"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"surface_qdrant_collection_hint"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/') +fi + +# Set defaults if not found in config +CTX_COLLECTION=${CTX_COLLECTION:-"codebase"} +REFRAG_RUNTIME=${REFRAG_RUNTIME:-"glm"} +GLM_API_KEY=${GLM_API_KEY:-} +GLM_API_BASE=${GLM_API_BASE:-} +GLM_MODEL=${GLM_MODEL:-"glm-4.6"} +CTX_DEFAULT_MODE=${CTX_DEFAULT_MODE:-"default"} +CTX_REQUIRE_CONTEXT=${CTX_REQUIRE_CONTEXT:-true} +CTX_RELEVANCE_GATE=${CTX_RELEVANCE_GATE:-false} +CTX_MIN_RELEVANCE=${CTX_MIN_RELEVANCE:-0.1} +CTX_REWRITE_MAX_TOKENS=${CTX_REWRITE_MAX_TOKENS:-320} + +# Normalize surface_qdrant_collection_hint from config (true/false) into 1/0 +CFG_HINT="" +if [ -n "$CTX_SURFACE_COLLECTION_CFG" ]; then + if [ "$CTX_SURFACE_COLLECTION_CFG" = "true" ]; then + CFG_HINT="1" + elif [ "$CTX_SURFACE_COLLECTION_CFG" = "false" ]; then + CFG_HINT="0" + fi +fi + +# Precedence: explicit env override > ctx_config flag > auto-on when collection known +if [ -n "${CTX_SURFACE_COLLECTION_HINT:-}" ]; then + : +elif [ -n "$CFG_HINT" ]; then + CTX_SURFACE_COLLECTION_HINT="$CFG_HINT" +elif [ -n "$CTX_COLLECTION" ]; then + CTX_SURFACE_COLLECTION_HINT="1" +fi + +# Export GLM/context environment variables from config +export REFRAG_RUNTIME GLM_API_KEY GLM_API_BASE GLM_MODEL CTX_REQUIRE_CONTEXT CTX_RELEVANCE_GATE CTX_MIN_RELEVANCE CTX_REWRITE_MAX_TOKENS + +# Easy bypass patterns - any of these will skip ctx enhancement +BYPASS_REASON="" +if [[ "$USER_MESSAGE" =~ ^(noctx|raw|bypass|skip|no-enhance): ]]; then + BYPASS_REASON="prefix_tag" +elif [[ "$USER_MESSAGE" =~ ^\\ ]]; then + BYPASS_REASON="leading_backslash" +elif [[ "$USER_MESSAGE" =~ ^\< ]]; then + BYPASS_REASON="leading_angle_bracket" +elif [[ "$USER_MESSAGE" =~ ^(/help|/clear|/exit|/quit) ]]; then + BYPASS_REASON="slash_command" +elif [[ "$USER_MESSAGE" =~ ^\?\s*$ ]]; then + BYPASS_REASON="short_question_mark" +elif [ ${#USER_MESSAGE} -lt 12 ]; then + BYPASS_REASON="too_short" +fi + +if [ -n "$BYPASS_REASON" ]; then + if [ "$CTX_HOOK_DEBUG" = "1" ]; then + echo "[ctx_debug status=bypassed reason=$BYPASS_REASON script_dir=$SCRIPT_DIR workspace_dir=$WORKSPACE_DIR config_file=$CONFIG_FILE] $USER_MESSAGE" + else + echo "$USER_MESSAGE" + fi + exit 0 +fi + +# Build ctx command with optional unicorn flag +if [ -f "$SCRIPT_DIR/ctx.py" ]; then + # Use embedded ctx.py when running from the packaged extension + CTX_CMD=(python3 "$SCRIPT_DIR/ctx.py") +else + # Fallback for repo-local usage + CTX_CMD=(python3 scripts/ctx.py) +fi +case "${CTX_DEFAULT_MODE,,}" in + unicorn) + CTX_CMD+=("--unicorn") + ;; + detail) + CTX_CMD+=("--detail") + ;; +esac +CTX_CMD+=("$USER_MESSAGE" --collection "$CTX_COLLECTION") + +# Run ctx with collection +# When CTX_DEBUG_PATHS is enabled, preserve stderr so path-level debug from ctx.py is visible +if [ -n "${CTX_DEBUG_PATHS:-}" ]; then + ENHANCED=$(timeout 120s "${CTX_CMD[@]}" 2>&1 || echo "$USER_MESSAGE") +else + ENHANCED=$(timeout 120s "${CTX_CMD[@]}" 2>/dev/null || echo "$USER_MESSAGE") +fi + +if [ -n "$WORKSPACE_DIR" ] && [ "${CTX_ROOT_HINT:-1}" != "0" ]; then + HINT="The user's project root directory is \"$WORKSPACE_DIR\" (WORKSPACE_DIR)." + if [ "${CTX_SURFACE_COLLECTION_HINT:-0}" = "1" ] && [ -n "$CTX_COLLECTION" ]; then + HINT="$HINT The Qdrant collection name for this workspace is \"$CTX_COLLECTION\". Specify this collection when using memory or qdrant-indexer MCP tool (if available)." + fi + if [ -n "${USER_CWD:-}" ]; then + HINT="$HINT Claude's current working directory is \"$USER_CWD\" (user_cwd). \ +When using tools like Read, Search, or Bash, treat WORKSPACE_DIR as the root \ +for repository files. If WORKSPACE_DIR and user_cwd differ, do not assume \ +files live under user_cwd; use the full paths under WORKSPACE_DIR or the \ +project-relative paths shown above." + fi + ENHANCED="$HINT + +$ENHANCED" +fi + +# Log ctx output when logging is enabled +if [ "$LOG_ENABLED" = "1" ]; then + { + echo "[$(date -Iseconds)] CTX_OUTPUT" + echo "PROMPT = $USER_MESSAGE" + echo "ENHANCED = <> "$LOG_FILE" +fi + +if [ "$CTX_HOOK_DEBUG" = "1" ]; then + HOOK_STATUS="unchanged" + if [ "$ENHANCED" != "$USER_MESSAGE" ]; then + HOOK_STATUS="enhanced" + fi + echo "[ctx_debug status=$HOOK_STATUS script_dir=$SCRIPT_DIR workspace_dir=$WORKSPACE_DIR config_file=$CONFIG_FILE] $ENHANCED" +else + echo "$ENHANCED" +fi \ No newline at end of file diff --git a/ctx-mcp-bridge/.gitignore b/ctx-mcp-bridge/.gitignore new file mode 100644 index 00000000..9e30eb9b --- /dev/null +++ b/ctx-mcp-bridge/.gitignore @@ -0,0 +1 @@ +*.tgz \ No newline at end of file diff --git a/ctx-mcp-bridge/README.md b/ctx-mcp-bridge/README.md new file mode 100644 index 00000000..9b571234 --- /dev/null +++ b/ctx-mcp-bridge/README.md @@ -0,0 +1,212 @@ +# Context Engine MCP Bridge + +`@context-engine-bridge/context-engine-mcp-bridge` provides the `ctxce` CLI, a +Model Context Protocol (MCP) bridge that speaks to the Context Engine indexer +and memory servers and exposes them as a single MCP server. + +It is primarily used by the VS Code **Context Engine Uploader** extension, +available on the Marketplace: + +- + +The bridge can also be run standalone (e.g. from a terminal, or wired into +other MCP clients) as long as the Context Engine stack is running. + +## Prerequisites + +- Node.js **>= 18** (see `engines` in `package.json`). +- A running Context Engine stack (e.g. via `docker-compose.yml`) with: + - MCP indexer HTTP endpoint (default: `http://localhost:8003/mcp`). + - MCP memory HTTP endpoint (optional, default: `http://localhost:8002/mcp`). +- For optional auth: + - The upload/auth services must be configured with `CTXCE_AUTH_ENABLED=1` and + a reachable auth backend URL (e.g. `http://localhost:8004`). + +## Installation + +You can install the package globally, or run it via `npx`. + +### Global install + +```bash +npm install -g @context-engine-bridge/context-engine-mcp-bridge +``` + +This installs the `ctxce` (and `ctxce-bridge`) CLI in your PATH. + +### Using npx (no global install) + +```bash +npx @context-engine-bridge/context-engine-mcp-bridge ctxce --help +``` + +The examples below assume `ctxce` is available on your PATH; if you use `npx`, +just prefix commands with `npx @context-engine-bridge/context-engine-mcp-bridge`. + +## CLI overview + +The main entrypoint is: + +```bash +ctxce [...args] +``` + +Supported commands (from `src/cli.js`): + +- `ctxce mcp-serve` – stdio MCP bridge (for stdio-based MCP clients). +- `ctxce mcp-http-serve` – HTTP MCP bridge (for HTTP-based MCP clients). +- `ctxce auth ` – auth helper commands (`login`, `status`, `logout`). + +### Environment variables + +These environment variables are respected by the bridge: + +- `CTXCE_INDEXER_URL` – MCP indexer URL (default: `http://localhost:8003/mcp`). +- `CTXCE_MEMORY_URL` – MCP memory URL, or empty/omitted to disable memory + (default: `http://localhost:8002/mcp`). +- `CTXCE_HTTP_PORT` – port for `mcp-http-serve` (default: `30810`). + +For auth (optional, shared with the upload/auth backend): + +- `CTXCE_AUTH_ENABLED` – whether auth is enabled in the backend. +- `CTXCE_AUTH_BACKEND_URL` – auth backend URL (e.g. `http://localhost:8004`). +- `CTXCE_AUTH_TOKEN` – dev/shared token for `ctxce auth login`. +- `CTXCE_AUTH_SESSION_TTL_SECONDS` – session TTL / sliding expiry (seconds). + +The CLI also stores auth sessions in `~/.ctxce/auth.json`, keyed by backend URL. + +## Running the MCP bridge (stdio) + +The stdio bridge is suitable for MCP clients that speak stdio directly (for +example, certain editors or tools that expect an MCP server on stdin/stdout). + +```bash +ctxce mcp-serve \ + --workspace /path/to/your/workspace \ + --indexer-url http://localhost:8003/mcp \ + --memory-url http://localhost:8002/mcp +``` + +Flags: + +- `--workspace` / `--path` – workspace root (default: current working directory). +- `--indexer-url` – override indexer URL (default: `CTXCE_INDEXER_URL` or + `http://localhost:8003/mcp`). +- `--memory-url` – override memory URL (default: `CTXCE_MEMORY_URL` or + disabled when empty). + +## Running the MCP bridge (HTTP) + +The HTTP bridge exposes the MCP server via an HTTP endpoint (default +`http://127.0.0.1:30810/mcp`) and is what the VS Code extension uses in its +`http` transport mode. + +```bash +ctxce mcp-http-serve \ + --workspace /path/to/your/workspace \ + --indexer-url http://localhost:8003/mcp \ + --memory-url http://localhost:8002/mcp \ + --port 30810 +``` + +Flags: + +- `--workspace` / `--path` – workspace root (default: current working directory). +- `--indexer-url` – MCP indexer URL. +- `--memory-url` – MCP memory URL (or omit/empty to disable memory). +- `--port` – HTTP port for the bridge (default: `CTXCE_HTTP_PORT` + or `30810`). + +Once running, you can point an MCP client at: + +```text +http://127.0.0.1:/mcp +``` + +## Auth helper commands (`ctxce auth ...`) + +These commands are used both by the VS Code extension and standalone flows to +log in and manage auth sessions for the backend. + +### Login (token) + +```bash +ctxce auth login \ + --backend-url http://localhost:8004 \ + --token $CTXCE_AUTH_SHARED_TOKEN +``` + +This hits the backend `/auth/login` endpoint and stores a session entry in +`~/.ctxce/auth.json` under the given backend URL. + +### Login (username/password) + +```bash +ctxce auth login \ + --backend-url http://localhost:8004 \ + --username your-user \ + --password your-password +``` + +This calls `/auth/login/password` and persists the returned session the same +way as the token flow. + +### Status + +Human-readable status: + +```bash +ctxce auth status --backend-url http://localhost:8004 +``` + +Machine-readable status (used by the VS Code extension): + +```bash +ctxce auth status --backend-url http://localhost:8004 --json +``` + +The `--json` variant prints a single JSON object to stdout, for example: + +```json +{ + "backendUrl": "http://localhost:8004", + "state": "ok", // "ok" | "missing" | "expired" | "missing_backend" + "sessionId": "...", + "userId": "user-123", + "expiresAt": 0 // 0 or a Unix timestamp +} +``` + +Exit codes: + +- `0` – `state: "ok"` (valid session present). +- `1` – `state: "missing"` or `"missing_backend"`. +- `2` – `state: "expired"`. + +### Logout + +```bash +ctxce auth logout --backend-url http://localhost:8004 +``` + +Removes the stored auth entry for the given backend URL from +`~/.ctxce/auth.json`. + +## Relationship to the VS Code extension + +The VS Code **Context Engine Uploader** extension is the recommended way to use +this bridge for day-to-day development. It: + +- Launches the standalone upload client to push code into the remote stack. +- Starts/stops the MCP HTTP bridge (`ctxce mcp-http-serve`) for the active + workspace when `autoStartMcpBridge` is enabled. +- Uses `ctxce auth status --json` and `ctxce auth login` under the hood to + manage user sessions via UI prompts. + +This package README is aimed at advanced users who want to: + +- Run the MCP bridge outside of VS Code. +- Integrate the Context Engine MCP servers with other MCP-compatible clients. + +You can safely mix both approaches: the extension and the standalone bridge +share the same auth/session storage in `~/.ctxce/auth.json`. diff --git a/ctx-mcp-bridge/bin/ctxce.js b/ctx-mcp-bridge/bin/ctxce.js new file mode 100644 index 00000000..d4fe0d56 --- /dev/null +++ b/ctx-mcp-bridge/bin/ctxce.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { runCli } from "../src/cli.js"; + +runCli().catch((err) => { + console.error("[ctxce] Fatal error:", err && err.stack ? err.stack : err); + process.exit(1); +}); diff --git a/ctx-mcp-bridge/docs/debugging.md b/ctx-mcp-bridge/docs/debugging.md new file mode 100644 index 00000000..e5904743 --- /dev/null +++ b/ctx-mcp-bridge/docs/debugging.md @@ -0,0 +1,20 @@ +{ + "mcpServers": { + "context-engine": { + "command": "node", + "args": [ + "C:/Users/Admin/Documents/GitHub/Context-Engine/ctx-mcp-bridge/bin/ctxce.js", + "mcp-serve", + "--indexer-url", + "http://192.168.100.249:30806/mcp", + "--memory-url", + "http://192.168.100.249:30804/mcp", + "--workspace", + "C:/Users/Admin/Documents/GitHub/Pirate Survivors" + ], + "env": { + "CTXCE_DEBUG_LOG": "C:/Users/Admin/ctxce-mcp.log" + } + } + } +} \ No newline at end of file diff --git a/ctx-mcp-bridge/package-lock.json b/ctx-mcp-bridge/package-lock.json new file mode 100644 index 00000000..04b28ef3 --- /dev/null +++ b/ctx-mcp-bridge/package-lock.json @@ -0,0 +1,1092 @@ +{ + "name": "@context-engine-bridge/context-engine-mcp-bridge", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@context-engine-bridge/context-engine-mcp-bridge", + "version": "0.0.1", + "dependencies": { + "@modelcontextprotocol/sdk": "^1.24.3", + "zod": "^3.25.0" + }, + "bin": { + "ctxce": "bin/ctxce.js", + "ctxce-bridge": "bin/ctxce.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.24.3.tgz", + "integrity": "sha512-YgSHW29fuzKKAHTGe9zjNoo+yF8KaQPzDC2W9Pv41E7/57IfY+AMGJ/aDFlgTLcVVELoggKE4syABCE75u3NCw==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.0.1", + "express-rate-limit": "^7.5.0", + "jose": "^6.1.1", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", + "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.1.tgz", + "integrity": "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz", + "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", + "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "mime-types": "^3.0.1", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } + } + } +} diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json new file mode 100644 index 00000000..d9f87d74 --- /dev/null +++ b/ctx-mcp-bridge/package.json @@ -0,0 +1,23 @@ +{ + "name": "@context-engine-bridge/context-engine-mcp-bridge", + "version": "0.0.13", + "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", + "bin": { + "ctxce": "bin/ctxce.js", + "ctxce-bridge": "bin/ctxce.js" + }, + "type": "module", + "scripts": { + "start": "node bin/ctxce.js" + }, + "dependencies": { + "@modelcontextprotocol/sdk": "^1.24.3", + "zod": "^3.25.0" + }, + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/ctx-mcp-bridge/publish.sh b/ctx-mcp-bridge/publish.sh new file mode 100755 index 00000000..7d86e455 --- /dev/null +++ b/ctx-mcp-bridge/publish.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Simple helper to login (if needed) and publish the package. +# Usage: +# ./publish.sh # publishes current version +# ./publish.sh 0.0.2 # bumps version to 0.0.2 then publishes + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +PACKAGE_NAME="@context-engine-bridge/context-engine-mcp-bridge" + +echo "[publish] Verifying npm authentication..." +if ! npm whoami >/dev/null 2>&1; then + echo "[publish] Not logged in; running npm login" + npm login +else + echo "[publish] Already authenticated as $(npm whoami)" +fi + +if [[ $# -gt 0 ]]; then + VERSION="$1" + echo "[publish] Bumping version to $VERSION" + npm version "$VERSION" --no-git-tag-version +fi + +echo "[publish] Packing $PACKAGE_NAME for verification..." +npm pack >/dev/null + +echo "[publish] Publishing $PACKAGE_NAME..." +npm publish --access public + +echo "[publish] Done!" diff --git a/ctx-mcp-bridge/src/authCli.js b/ctx-mcp-bridge/src/authCli.js new file mode 100644 index 00000000..45bc86b2 --- /dev/null +++ b/ctx-mcp-bridge/src/authCli.js @@ -0,0 +1,284 @@ +import process from "node:process"; +import { loadAuthEntry, saveAuthEntry, deleteAuthEntry, loadAnyAuthEntry } from "./authConfig.js"; + +function parseAuthArgs(args) { + let backendUrl = process.env.CTXCE_AUTH_BACKEND_URL || ""; + let token = process.env.CTXCE_AUTH_TOKEN || ""; + let username = process.env.CTXCE_AUTH_USERNAME || ""; + let password = process.env.CTXCE_AUTH_PASSWORD || ""; + let outputJson = false; + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if ((a === "--backend-url" || a === "--auth-url") && i + 1 < args.length) { + backendUrl = args[i + 1]; + i += 1; + continue; + } + if ((a === "--token" || a === "--api-key") && i + 1 < args.length) { + token = args[i + 1]; + i += 1; + continue; + } + if (a === "--username" || a === "--user") { + const hasNext = i + 1 < args.length; + const next = hasNext ? String(args[i + 1]) : ""; + if (hasNext && !next.startsWith("-")) { + username = args[i + 1]; + i += 1; + } else { + console.error("[ctxce] Missing value for --username/--user; expected a username."); + process.exit(1); + } + continue; + } + if ((a === "--password" || a === "--pass") && i + 1 < args.length) { + password = args[i + 1]; + i += 1; + continue; + } + if (a === "--json" || a === "-j") { + outputJson = true; + continue; + } + } + return { backendUrl, token, username, password, outputJson }; +} + +function getBackendUrl(backendUrl) { + return (backendUrl || process.env.CTXCE_AUTH_BACKEND_URL || "").trim(); +} + +function getDefaultUploadBackend() { + // Default to upload service when nothing else is configured + return (process.env.CTXCE_UPLOAD_ENDPOINT || process.env.UPLOAD_ENDPOINT || "http://localhost:8004").trim(); +} + +function requireBackendUrl(backendUrl) { + const url = getBackendUrl(backendUrl); + if (!url) { + console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + return url; +} + +function outputJsonStatus(url, state, entry, rawExpires) { + const expiresAt = typeof rawExpires === "number" + ? rawExpires + : entry && typeof entry.expiresAt === "number" + ? entry.expiresAt + : null; + console.log(JSON.stringify({ + backendUrl: url, + state, + sessionId: entry && entry.sessionId ? entry.sessionId : null, + userId: entry && entry.userId ? entry.userId : null, + expiresAt, + })); +} + +async function doLogin(args) { + const { backendUrl, token, username, password } = parseAuthArgs(args); + let url = getBackendUrl(backendUrl); + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + console.error("[ctxce] Using stored backend for login:", url); + } + } + if (!url) { + // Final fallback: default upload endpoint (extension's upload endpoint or localhost:8004) + url = getDefaultUploadBackend(); + if (url) { + console.error("[ctxce] Using default upload backend for login:", url); + } + } + if (!url) { + console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + const trimmedUser = (username || "").trim(); + const usePassword = trimmedUser && (password || "").length > 0; + + let body; + let target; + if (usePassword) { + body = { + username: trimmedUser, + password, + workspace: process.cwd(), + }; + target = url.replace(/\/+$/, "") + "/auth/login/password"; + } else { + body = { + client: "ctxce", + workspace: process.cwd(), + }; + if (token) { + body.token = token; + } + target = url.replace(/\/+$/, "") + "/auth/login"; + } + let resp; + try { + resp = await fetch(target, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + } catch (err) { + console.error("[ctxce] Auth login request failed:", String(err)); + process.exit(1); + } + if (!resp || !resp.ok) { + console.error("[ctxce] Auth login failed with status", resp ? resp.status : ""); + process.exit(1); + } + let data; + try { + data = await resp.json(); + } catch (err) { + data = {}; + } + const sessionId = data.session_id || data.sessionId || null; + const userId = data.user_id || data.userId || null; + const expiresAt = data.expires_at || data.expiresAt || null; + if (!sessionId) { + console.error("[ctxce] Auth login response missing session id."); + process.exit(1); + } + saveAuthEntry(url, { sessionId, userId, expiresAt }); + console.error("[ctxce] Auth login successful for", url); +} + +async function doStatus(args) { + const { backendUrl, outputJson } = parseAuthArgs(args); + let url = getBackendUrl(backendUrl); + let usedFallback = false; + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + usedFallback = true; + } + } + if (!url) { + // Final fallback: default upload endpoint + url = getDefaultUploadBackend(); + if (url) { + usedFallback = true; + console.error("[ctxce] Using default upload backend for status:", url); + } + } + if (!url) { + if (outputJson) { + outputJsonStatus("", "missing_backend", null, null); + process.exit(1); + } + console.error("[ctxce] Auth backend URL not configured and no stored sessions found. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + let entry; + try { + entry = loadAuthEntry(url); + } catch (err) { + entry = null; + } + const nowSecs = Math.floor(Date.now() / 1000); + const rawExpires = entry && typeof entry.expiresAt === "number" ? entry.expiresAt : null; + const hasSession = !!(entry && typeof entry.sessionId === "string" && entry.sessionId); + const expired = !!(rawExpires && rawExpires > 0 && rawExpires < nowSecs); + + if (!entry || !hasSession) { + if (outputJson) { + outputJsonStatus(url, "missing", null, rawExpires); + process.exit(1); + } + if (usedFallback) { + console.error("[ctxce] Not logged in for stored backend", url); + } else { + console.error("[ctxce] Not logged in for", url); + } + process.exit(1); + } + + if (expired) { + if (outputJson) { + outputJsonStatus(url, "expired", entry, rawExpires); + process.exit(2); + } + if (usedFallback) { + console.error("[ctxce] Stored auth session appears expired for stored backend", url); + } else { + console.error("[ctxce] Stored auth session appears expired for", url); + } + if (rawExpires) { + console.error("[ctxce] Session expired at", rawExpires); + } + process.exit(2); + } + + if (outputJson) { + outputJsonStatus(url, "ok", entry, rawExpires); + return; + } + if (usedFallback) { + console.error("[ctxce] Using stored backend for status:", url); + } + console.error("[ctxce] Logged in to", url, "as", entry.userId || ""); + if (rawExpires) { + console.error("[ctxce] Session expires at", rawExpires); + } +} + +async function doLogout(args) { + const { backendUrl } = parseAuthArgs(args); + let url = getBackendUrl(backendUrl); + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + console.error("[ctxce] Using stored backend for logout:", url); + } + } + if (!url) { + // Final fallback: default upload endpoint + url = getDefaultUploadBackend(); + if (url) { + console.error("[ctxce] Using default upload backend for logout:", url); + } + } + if (!url) { + console.error("[ctxce] Auth backend URL not configured and no stored sessions found. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + const entry = loadAuthEntry(url); + if (!entry) { + console.error("[ctxce] No stored auth session for", url); + return; + } + deleteAuthEntry(url); + console.error("[ctxce] Logged out from", url); +} + +export async function runAuthCommand(subcommand, args) { + const sub = (subcommand || "").toLowerCase(); + if (sub === "login") { + await doLogin(args || []); + return; + } + if (sub === "status") { + await doStatus(args || []); + return; + } + if (sub === "logout") { + await doLogout(args || []); + return; + } + console.error("Usage: ctxce auth [--backend-url ] [--token ]"); + process.exit(1); +} diff --git a/ctx-mcp-bridge/src/authConfig.js b/ctx-mcp-bridge/src/authConfig.js new file mode 100644 index 00000000..869b18ad --- /dev/null +++ b/ctx-mcp-bridge/src/authConfig.js @@ -0,0 +1,84 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const CONFIG_DIR_NAME = ".ctxce"; +const CONFIG_BASENAME = "auth.json"; + +function getConfigPath() { + const home = os.homedir() || process.cwd(); + const dir = path.join(home, CONFIG_DIR_NAME); + return path.join(dir, CONFIG_BASENAME); +} + +export function readConfig() { + try { + const cfgPath = getConfigPath(); + const raw = fs.readFileSync(cfgPath, "utf8"); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + } + return {}; +} + +function writeConfig(data) { + try { + const cfgPath = getConfigPath(); + const dir = path.dirname(cfgPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + fs.writeFileSync(cfgPath, JSON.stringify(data, null, 2), "utf8"); + } catch (err) { + } +} + +export function loadAuthEntry(backendUrl) { + if (!backendUrl) { + return null; + } + const all = readConfig(); + const key = String(backendUrl); + const entry = all[key]; + if (!entry || typeof entry !== "object") { + return null; + } + return entry; +} + +export function saveAuthEntry(backendUrl, entry) { + if (!backendUrl || !entry || typeof entry !== "object") { + return; + } + const all = readConfig(); + const key = String(backendUrl); + all[key] = entry; + writeConfig(all); +} + +export function deleteAuthEntry(backendUrl) { + if (!backendUrl) { + return; + } + const all = readConfig(); + const key = String(backendUrl); + if (Object.prototype.hasOwnProperty.call(all, key)) { + delete all[key]; + writeConfig(all); + } +} + +export function loadAnyAuthEntry() { + const all = readConfig(); + const keys = Object.keys(all); + for (const key of keys) { + const entry = all[key]; + if (entry && typeof entry === "object") { + return { backendUrl: key, entry }; + } + } + return null; +} diff --git a/ctx-mcp-bridge/src/cli.js b/ctx-mcp-bridge/src/cli.js new file mode 100644 index 00000000..4d995645 --- /dev/null +++ b/ctx-mcp-bridge/src/cli.js @@ -0,0 +1,123 @@ +// CLI entrypoint for ctxce + +import process from "node:process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runMcpServer, runHttpMcpServer } from "./mcpServer.js"; +import { runAuthCommand } from "./authCli.js"; + +export async function runCli() { + const argv = process.argv.slice(2); + const cmd = argv[0]; + + if (cmd === "auth") { + const sub = argv[1] || ""; + const args = argv.slice(2); + await runAuthCommand(sub, args); + return; + } + + if (cmd === "mcp-http-serve") { + const args = argv.slice(1); + let workspace = process.cwd(); + let indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + let memoryUrl = process.env.CTXCE_MEMORY_URL || null; + let port = Number.parseInt(process.env.CTXCE_HTTP_PORT || "30810", 10) || 30810; + + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if (a === "--workspace" || a === "--path") { + if (i + 1 < args.length) { + workspace = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--indexer-url") { + if (i + 1 < args.length) { + indexerUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--memory-url") { + if (i + 1 < args.length) { + memoryUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--port") { + if (i + 1 < args.length) { + const parsed = Number.parseInt(args[i + 1], 10); + if (!Number.isNaN(parsed) && parsed > 0) { + port = parsed; + } + i += 1; + continue; + } + } + } + + // eslint-disable-next-line no-console + console.error( + `[ctxce] Starting HTTP MCP bridge: workspace=${workspace}, port=${port}, indexerUrl=${indexerUrl}, memoryUrl=${memoryUrl || "disabled"}`, + ); + await runHttpMcpServer({ workspace, indexerUrl, memoryUrl, port }); + return; + } + + if (cmd === "mcp-serve") { + // Minimal flag parsing for PoC: allow passing workspace/root and indexer URL. + // Supported flags: + // --workspace / --path : workspace root (default: cwd) + // --indexer-url : override MCP indexer URL (default env CTXCE_INDEXER_URL or http://localhost:8003/mcp) + const args = argv.slice(1); + let workspace = process.cwd(); + let indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + let memoryUrl = process.env.CTXCE_MEMORY_URL || null; + + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if (a === "--workspace" || a === "--path") { + if (i + 1 < args.length) { + workspace = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--indexer-url") { + if (i + 1 < args.length) { + indexerUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--memory-url") { + if (i + 1 < args.length) { + memoryUrl = args[i + 1]; + i += 1; + continue; + } + } + } + + // eslint-disable-next-line no-console + console.error( + `[ctxce] Starting MCP bridge: workspace=${workspace}, indexerUrl=${indexerUrl}, memoryUrl=${memoryUrl || "disabled"}`, + ); + await runMcpServer({ workspace, indexerUrl, memoryUrl }); + return; + } + + // Default help + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + const binName = "ctxce"; + + // eslint-disable-next-line no-console + console.error( + `Usage: ${binName} mcp-serve [--workspace ] [--indexer-url ] [--memory-url ] | ${binName} mcp-http-serve [--workspace ] [--indexer-url ] [--memory-url ] [--port ] | ${binName} auth [--backend-url ] [--token ] [--username --password ]`, + ); + process.exit(1); +} diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js new file mode 100644 index 00000000..53cb05b7 --- /dev/null +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -0,0 +1,1061 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; +import { execSync } from "node:child_process"; +import { createServer } from "node:http"; +import { Server } from "@modelcontextprotocol/sdk/server/index.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; +import { loadAnyAuthEntry, loadAuthEntry, readConfig, saveAuthEntry } from "./authConfig.js"; +import { maybeRemapToolArgs, maybeRemapToolResult } from "./resultPathMapping.js"; +import * as oauthHandler from "./oauthHandler.js"; + +function debugLog(message) { + try { + const text = typeof message === "string" ? message : String(message); + console.error(text); + const dest = process.env.CTXCE_DEBUG_LOG; + if (dest) { + fs.appendFileSync(dest, `${new Date().toISOString()} ${text}\n`, "utf8"); + } + } catch { + } +} + +async function sendSessionDefaults(client, payload, label) { + if (!client) { + return; + } + try { + await client.callTool({ + name: "set_session_defaults", + arguments: payload, + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[ctxce] Failed to call set_session_defaults on ${label}:`, err); + } +} +function dedupeTools(tools) { + const seen = new Set(); + const out = []; + for (const tool of tools) { + const key = (tool && typeof tool.name === "string" && tool.name) || ""; + if (!key || seen.has(key)) { + if (key === "" || key !== "set_session_defaults") { + continue; + } + if (seen.has(key)) { + continue; + } + } + seen.add(key); + out.push(tool); + } + return out; +} + +async function listMemoryTools(client) { + if (!client) { + return []; + } + try { + const remote = await withTimeout( + client.listTools(), + 5000, + "memory tools/list", + ); + return Array.isArray(remote?.tools) ? remote.tools.slice() : []; + } catch (err) { + debugLog("[ctxce] Error calling memory tools/list: " + String(err)); + return []; + } +} + +function withTimeout(promise, ms, label) { + return new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + const errorMessage = + label != null + ? `[ctxce] Timeout after ${ms}ms in ${label}` + : `[ctxce] Timeout after ${ms}ms`; + reject(new Error(errorMessage)); + }, ms); + promise + .then((value) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resolve(value); + }) + .catch((err) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + reject(err); + }); + }); +} + +function getBridgeToolTimeoutMs() { + try { + const raw = process.env.CTXCE_TOOL_TIMEOUT_MSEC; + if (!raw) { + return 300000; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return 300000; + } + return parsed; + } catch { + return 300000; + } +} + +function selectClientForTool(name, indexerClient, memoryClient) { + if (!name) { + return indexerClient; + } + const lowered = name.toLowerCase(); + if (memoryClient && (lowered.startsWith("memory.") || lowered.startsWith("mcp_memory_"))) { + return memoryClient; + } + return indexerClient; +} + +function isSessionError(error) { + try { + const msg = + (error && typeof error.message === "string" && error.message) || + (typeof error === "string" ? error : String(error || "")); + if (!msg) { + return false; + } + return ( + msg.includes("No valid session ID") || + msg.includes("Mcp-Session-Id header is required") || + msg.includes("Server not initialized") || + msg.includes("Session not found") + ); + } catch { + return false; + } +} + +/** + * Detect actual backend auth rejection (from mcp_auth.py ValidationError). + * These indicate the session is truly invalid on the backend, not just + * an MCP SDK transport issue that can be fixed by reinit. + */ +function isAuthRejectionError(error) { + try { + const msg = + (error && typeof error.message === "string" && error.message) || + (typeof error === "string" ? error : String(error || "")); + if (!msg) { + return false; + } + return ( + msg.includes("Invalid or expired session") || + msg.includes("Missing session for authorized operation") || + msg.includes("Not authenticated") + ); + } catch { + return false; + } +} + +function getBridgeRetryAttempts() { + try { + const raw = process.env.CTXCE_TOOL_RETRY_ATTEMPTS; + if (!raw) { + return 2; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return 1; + } + return parsed; + } catch { + return 2; + } +} + +function getBridgeRetryDelayMs() { + try { + const raw = process.env.CTXCE_TOOL_RETRY_DELAY_MSEC; + if (!raw) { + return 200; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return 0; + } + return parsed; + } catch { + return 200; + } +} + +function isTransientToolError(error) { + try { + const msg = + (error && typeof error.message === "string" && error.message) || + (typeof error === "string" ? error : String(error || "")); + if (!msg) { + return false; + } + const lower = msg.toLowerCase(); + + if ( + lower.includes("timed out") || + lower.includes("timeout") || + lower.includes("time-out") + ) { + return true; + } + + if ( + lower.includes("econnreset") || + lower.includes("econnrefused") || + lower.includes("etimedout") || + lower.includes("enotfound") || + lower.includes("ehostunreach") || + lower.includes("enetunreach") + ) { + return true; + } + + if ( + lower.includes("bad gateway") || + lower.includes("gateway timeout") || + lower.includes("service unavailable") || + lower.includes(" 502 ") || + lower.includes(" 503 ") || + lower.includes(" 504 ") + ) { + return true; + } + + if (lower.includes("network error")) { + return true; + } + + if (typeof error.code === "number" && error.code === -32001 && !isSessionError(error)) { + return true; + } + if ( + typeof error.code === "string" && + error.code.toLowerCase && + error.code.toLowerCase().includes("timeout") + ) { + return true; + } + + return false; + } catch { + return false; + } +} +// MCP stdio server implemented using the official MCP TypeScript SDK. +// Acts as a low-level proxy for tools, forwarding tools/list and tools/call +// to the remote qdrant-indexer MCP server while adding a local `ping` tool. + +const ADMIN_SESSION_COOKIE_NAME = "ctxce_session"; +const SLUGGED_REPO_RE = /.+-[0-9a-f]{16}(?:_old)?$/i; +const BRIDGE_STATE_TOKEN = (process.env.CTXCE_BRIDGE_STATE_TOKEN || "").trim(); + +function getHostname(candidate) { + try { + const url = new URL(candidate); + return url.hostname; + } catch { + return ""; + } +} + +function normalizeBackendUrl(candidate) { + const trimmed = (candidate || "").trim(); + if (!trimmed) { + return ""; + } + try { + const parsed = new URL(trimmed); + if (parsed.protocol && parsed.host) { + return `${parsed.protocol}//${parsed.host}`; + } + } catch { + // ignore parse failures + } + return trimmed.replace(/\/+$/, ""); +} + +function resolveAuthBackendContext(indexerUrl, memoryUrl) { + const envBackend = normalizeBackendUrl(process.env.CTXCE_AUTH_BACKEND_URL || ""); + if (envBackend) { + return { backendUrl: envBackend, source: "CTXCE_AUTH_BACKEND_URL" }; + } + + // If no env override, try to find a saved session. + // We prefer one that matches the host of indexer or memory URL if they look like backend roots. + const targetHosts = new Set(); + const ih = getHostname(indexerUrl); + if (ih) { + targetHosts.add(ih); + } + const mh = getHostname(memoryUrl); + if (mh) { + targetHosts.add(mh); + } + + if (targetHosts.size > 0) { + const all = readConfig(); + const backends = Object.keys(all); + for (const backendUrl of backends) { + const bh = getHostname(backendUrl); + if (bh && targetHosts.has(bh)) { + return { backendUrl, source: "auth_entry_host_match" }; + } + } + } + + try { + const any = loadAnyAuthEntry(); + const stored = normalizeBackendUrl(any?.backendUrl || ""); + if (stored) { + return { backendUrl: stored, source: "auth_entry" }; + } + } catch { + // ignore auth config read failures + } + return { backendUrl: "", source: "" }; +} + +async function fetchBridgeCollectionState({ + workspace, + collection, + sessionId, + repoName, + bridgeStateToken, + backendHint, + uploadServiceUrl, +}) { + try { + if (!uploadServiceUrl) { + debugLog("[ctxce] Skipping bridge/state fetch: no upload endpoint configured."); + return null; + } + const url = new URL("/bridge/state", uploadServiceUrl); + // ... + if (collection && collection.trim()) { + url.searchParams.set("collection", collection.trim()); + } else if (workspace && workspace.trim()) { + url.searchParams.set("workspace", workspace.trim()); + } + if (repoName && repoName.trim()) { + url.searchParams.set("repo_name", repoName.trim()); + } + + const headers = { + Accept: "application/json", + }; + if (bridgeStateToken && bridgeStateToken.trim()) { + headers["X-Bridge-State-Token"] = bridgeStateToken.trim(); + } + if (sessionId) { + headers.Cookie = `${ADMIN_SESSION_COOKIE_NAME}=${sessionId}`; + } + + debugLog(`[ctxce] Fetching bridge/state from ${url.toString()} (repo=${repoName || ""}).`); + const resp = await fetch(url, { + method: "GET", + headers, + }); + if (!resp.ok) { + if (resp.status === 401 || resp.status === 403) { + debugLog( + `[ctxce] /bridge/state responded ${resp.status}; missing or invalid token/session, marking local session as expired.`, + ); + if (backendHint) { + try { + const entry = loadAuthEntry(backendHint); + if (entry) { + saveAuthEntry(backendHint, { ...entry, expiresAt: 1 }); + } + } catch { + // ignore failures + } + } + return null; + } + throw new Error(`bridge/state responded ${resp.status}`); + } + debugLog(`[ctxce] bridge/state responded ${resp.status}`); + const data = await resp.json(); + return data && typeof data === "object" ? data : null; + } catch (err) { + debugLog("[ctxce] Failed to fetch /bridge/state: " + String(err)); + return null; + } +} + +async function createBridgeServer(options) { + const workspace = options.workspace || process.cwd(); + const indexerUrl = options.indexerUrl; + const memoryUrl = options.memoryUrl; + + const config = loadConfig(workspace); + const defaultCollection = + config && typeof config.default_collection === "string" + ? config.default_collection + : null; + const defaultMode = + config && typeof config.default_mode === "string" ? config.default_mode : null; + const defaultUnder = + config && typeof config.default_under === "string" ? config.default_under : null; + + debugLog( + `[ctxce] MCP low-level stdio bridge starting: workspace=${workspace}, indexerUrl=${indexerUrl}`, + ); + + if (defaultCollection) { + // eslint-disable-next-line no-console + console.error( + `[ctxce] Using default collection from ctx_config.json: ${defaultCollection}`, + ); + } + + let indexerClient = null; + let memoryClient = null; + + // Derive a simple session identifier for this bridge process. In the + // future this can be made user-aware (e.g. from auth), but for now we + // keep it deterministic per workspace to help the indexer reuse + // session-scoped defaults. + const { + backendUrl: authBackendUrl, + source: authBackendSource, + } = resolveAuthBackendContext(indexerUrl, memoryUrl); + + const uploadServiceUrl = authBackendUrl; + const uploadAuthBackend = authBackendUrl; + + if (uploadServiceUrl) { + debugLog(`[ctxce] Upload/auth backend resolved from ${authBackendSource}: ${uploadServiceUrl}`); + } else { + debugLog("[ctxce] No auth backend detected; bridge/state overrides disabled."); + } + + const explicitSession = process.env.CTXCE_SESSION_ID || ""; + const authBackendEnv = (process.env.CTXCE_AUTH_BACKEND_URL || "").trim(); + let backendHint = authBackendEnv || uploadAuthBackend || ""; + let sessionId = explicitSession; + + function sessionFromEntry(entry) { + if (!entry || typeof entry.sessionId !== "string" || !entry.sessionId) { + return ""; + } + const expiresAt = entry.expiresAt; + if ( + typeof expiresAt === "number" && + Number.isFinite(expiresAt) && + expiresAt > 0 && + expiresAt < Math.floor(Date.now() / 1000) + ) { + debugLog("[ctxce] Stored auth session appears expired; please run `ctxce auth login` again."); + return ""; + } + return entry.sessionId; + } + + function findSavedSession(backends) { + for (const backend of backends) { + const trimmed = (backend || "").trim(); + if (!trimmed) { + continue; + } + try { + const entry = loadAuthEntry(trimmed); + const session = sessionFromEntry(entry); + if (session) { + backendHint = trimmed; + return session; + } + } catch { + // ignore lookup failures + } + } + try { + const any = loadAnyAuthEntry(); + const session = any ? sessionFromEntry(any.entry) : ""; + if (session && any?.backendUrl) { + backendHint = any.backendUrl; + return session; + } + } catch { + // ignore lookup failures + } + return ""; + } + + function resolveSessionId() { + const explicit = (process.env.CTXCE_SESSION_ID || "").trim(); + if (explicit) { + return explicit; + } + return findSavedSession([backendHint, uploadAuthBackend, authBackendEnv]); + } + + if (!sessionId) { + sessionId = resolveSessionId(); + } + + if (!sessionId) { + sessionId = `ctxce-${Buffer.from(workspace).toString("hex").slice(0, 24)}`; + } + + // Best-effort: inform the indexer of default collection and session. + // If this fails we still proceed, falling back to per-call injection. + const defaultsPayload = { session: sessionId }; + if (defaultCollection) { + defaultsPayload.collection = defaultCollection; + } + + const repoName = detectRepoName(workspace, config); + + try { + const state = await fetchBridgeCollectionState({ + workspace, + collection: defaultCollection, + sessionId, + repoName, + bridgeStateToken: BRIDGE_STATE_TOKEN, + backendHint, + uploadServiceUrl, + }); + if (state) { + const serving = state.serving_collection || state.active_collection; + if (serving) { + defaultsPayload.collection = serving; + if (!defaultCollection || defaultCollection !== serving) { + debugLog( + `[ctxce] Using serving collection from /bridge/state: ${serving}`, + ); + } + } + } + } catch (err) { + debugLog("[ctxce] bridge/state lookup failed: " + String(err)); + } + + if (defaultMode) { + defaultsPayload.mode = defaultMode; + } + if (defaultUnder) { + defaultsPayload.under = defaultUnder; + } + + async function initializeRemoteClients(forceRecreate = false) { + if (!forceRecreate && indexerClient) { + return; + } + + if (forceRecreate) { + try { + debugLog("[ctxce] Reinitializing remote MCP clients after session error."); + } catch { + // ignore logging failures + } + } + + let nextIndexerClient = null; + try { + const indexerTransport = new StreamableHTTPClientTransport(indexerUrl); + const client = new Client( + { + name: "ctx-context-engine-bridge-http-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + await client.connect(indexerTransport); + nextIndexerClient = client; + } catch (err) { + debugLog("[ctxce] Failed to connect MCP HTTP client to indexer: " + String(err)); + nextIndexerClient = null; + } + + let nextMemoryClient = null; + if (memoryUrl) { + try { + const memoryTransport = new StreamableHTTPClientTransport(memoryUrl); + const client = new Client( + { + name: "ctx-context-engine-bridge-memory-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + await client.connect(memoryTransport); + debugLog(`[ctxce] Connected memory MCP client: ${memoryUrl}`); + nextMemoryClient = client; + } catch (err) { + debugLog("[ctxce] Failed to connect memory MCP client: " + String(err)); + nextMemoryClient = null; + } + } + + indexerClient = nextIndexerClient; + memoryClient = nextMemoryClient; + + if (Object.keys(defaultsPayload).length > 1 && indexerClient) { + await sendSessionDefaults(indexerClient, defaultsPayload, "indexer"); + if (memoryClient) { + await sendSessionDefaults(memoryClient, defaultsPayload, "memory"); + } + } + } + + await initializeRemoteClients(false); + + const server = new Server( // TODO: marked as depreciated + { + name: "ctx-context-engine-bridge", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + }, + }, + ); + + // tools/list → fetch tools from remote indexer + server.setRequestHandler(ListToolsRequestSchema, async () => { + let remote; + try { + debugLog("[ctxce] tools/list: fetching tools from indexer"); + await initializeRemoteClients(false); + if (!indexerClient) { + throw new Error("Indexer MCP client not initialized"); + } + remote = await withTimeout( + indexerClient.listTools(), + 10000, + "indexer tools/list", + ); + } catch (err) { + debugLog("[ctxce] Error calling remote tools/list: " + String(err)); + const memoryToolsFallback = await listMemoryTools(memoryClient); + const toolsFallback = dedupeTools([...memoryToolsFallback]); + return { tools: toolsFallback }; + } + + try { + const toolNames = + remote && Array.isArray(remote.tools) + ? remote.tools.map((t) => (t && typeof t.name === "string" ? t.name : "")) + : []; + debugLog("[ctxce] tools/list remote result tools: " + JSON.stringify(toolNames)); + } catch (err) { + debugLog("[ctxce] tools/list remote result: " + String(err)); + } + + const indexerTools = Array.isArray(remote?.tools) ? remote.tools.slice() : []; + const memoryTools = await listMemoryTools(memoryClient); + const tools = dedupeTools([...indexerTools, ...memoryTools]); + debugLog(`[ctxce] tools/list: returning ${tools.length} tools`); + return { tools }; + }); + + // tools/call → proxied to indexer or memory server + server.setRequestHandler(CallToolRequestSchema, async (request) => { + const params = request.params || {}; + const name = params.name; + let args = params.arguments; + + debugLog(`[ctxce] tools/call: ${name || ""}`); + + // Refresh session before each call; re-init clients if session changes. + const freshSession = resolveSessionId() || sessionId; + if (freshSession && freshSession !== sessionId) { + sessionId = freshSession; + try { + await initializeRemoteClients(true); + } catch (err) { + debugLog("[ctxce] Failed to reinitialize clients after session refresh: " + String(err)); + } + } + if (sessionId && (args === undefined || args === null || typeof args === "object")) { + const obj = args && typeof args === "object" ? { ...args } : {}; + if (!Object.prototype.hasOwnProperty.call(obj, "session")) { + obj.session = sessionId; + } + args = obj; + } + + args = maybeRemapToolArgs(name, args, workspace); + + if (name === "set_session_defaults") { + const indexerResult = await indexerClient.callTool({ name, arguments: args }); + if (memoryClient) { + try { + await memoryClient.callTool({ name, arguments: args }); + } catch (err) { + debugLog("[ctxce] Memory set_session_defaults failed: " + String(err)); + } + } + return indexerResult; + } + + await initializeRemoteClients(false); + + const timeoutMs = getBridgeToolTimeoutMs(); + const maxAttempts = getBridgeRetryAttempts(); + const retryDelayMs = getBridgeRetryDelayMs(); + let sessionRetried = false; + let lastError; + + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + if (attempt > 0 && retryDelayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + + const targetClient = selectClientForTool(name, indexerClient, memoryClient); + if (!targetClient) { + throw new Error(`Tool ${name} not available on any configured MCP server`); + } + + try { + const result = await targetClient.callTool( + { + name, + arguments: args, + }, + undefined, + { timeout: timeoutMs }, + ); + return maybeRemapToolResult(name, result, workspace); + } catch (err) { + lastError = err; + + if (isSessionError(err) && !sessionRetried) { + debugLog( + "[ctxce] tools/call: detected remote MCP session error; reinitializing clients and retrying once: " + + String(err), + ); + await initializeRemoteClients(true); + sessionRetried = true; + continue; + } + + // Backend auth rejection (mcp_auth.py ValidationError) - expire local auth + if (isAuthRejectionError(err)) { + debugLog( + "[ctxce] tools/call: backend auth rejection; marking local session as expired: " + + String(err), + ); + if (backendHint) { + try { + const entry = loadAuthEntry(backendHint); + if (entry) { + saveAuthEntry(backendHint, { ...entry, expiresAt: 1 }); + } + } catch { + // ignore failures + } + } + } + + if (!isTransientToolError(err) || attempt === maxAttempts - 1) { + throw err; + } + + debugLog( + `[ctxce] tools/call: transient error (attempt ${attempt + 1}/${maxAttempts}), retrying: ` + + String(err), + ); + // Loop will retry + } + } + + throw lastError || new Error("Unknown MCP tools/call error"); + }); + + return server; +} + +export async function runMcpServer(options) { + const server = await createBridgeServer(options); + const transport = new StdioServerTransport(); + await server.connect(transport); + + const exitOnStdinClose = process.env.CTXCE_EXIT_ON_STDIN_CLOSE !== "0"; + if (exitOnStdinClose) { + const handleStdioClosed = () => { + try { + debugLog("[ctxce] Stdio transport closed; exiting MCP bridge process."); + } catch { + // ignore + } + // Allow any in-flight logs to flush, then exit. + setTimeout(() => { + process.exit(0); + }, 10).unref(); + }; + + if (process.stdin && typeof process.stdin.on === "function") { + process.stdin.on("end", handleStdioClosed); + process.stdin.on("close", handleStdioClosed); + process.stdin.on("error", handleStdioClosed); + } + } +} + +export async function runHttpMcpServer(options) { + const server = await createBridgeServer(options); + const port = + typeof options.port === "number" + ? options.port + : Number.parseInt(process.env.CTXCE_HTTP_PORT || "30810", 10) || 30810; + + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + }); + + await server.connect(transport); + + // Build issuer URL for OAuth + // Note: Local-only bridge uses 127.0.0.1. For remote access, this would need to be configurable. + const issuerUrl = `http://127.0.0.1:${port}`; + + const httpServer = createServer((req, res) => { + try { + const url = req.url || "/"; + + // Parse URL for query params + const parsedUrl = new URL(url, `http://${req.headers.host || 'localhost'}`); + + // ================================================================ + // OAuth 2.0 Endpoints (RFC9728 Protected Resource Metadata + RFC7591) + // ================================================================ + + // OAuth metadata endpoint (RFC9728) + if (parsedUrl.pathname === "/.well-known/oauth-authorization-server") { + oauthHandler.handleOAuthMetadata(req, res, issuerUrl); + return; + } + + // OAuth Dynamic Client Registration endpoint (RFC7591) + if (parsedUrl.pathname === "/oauth/register" && req.method === "POST") { + oauthHandler.handleOAuthRegister(req, res); + return; + } + + // OAuth authorize endpoint + if (parsedUrl.pathname === "/oauth/authorize") { + oauthHandler.handleOAuthAuthorize(req, res, parsedUrl.searchParams); + return; + } + + // Store session endpoint (helper for login page) + if (parsedUrl.pathname === "/oauth/store-session" && req.method === "POST") { + oauthHandler.handleOAuthStoreSession(req, res); + return; + } + + // OAuth token endpoint + if (parsedUrl.pathname === "/oauth/token" && req.method === "POST") { + oauthHandler.handleOAuthToken(req, res); + return; + } + + // ================================================================ + // MCP Endpoint + // ================================================================ + + // Check Bearer token for MCP endpoint (accept /mcp and /mcp/ for compatibility) + if (parsedUrl.pathname === "/mcp" || parsedUrl.pathname === "/mcp/") { + const authHeader = req.headers["authorization"] || ""; + const token = authHeader.startsWith("Bearer ") ? authHeader.slice(7) : null; + + // TODO: Validate token and inject session + // For now, allow unauthenticated (backward compatible) + + if (req.method !== "POST") { + res.statusCode = 405; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32000, message: "Method not allowed" }, + id: null, + }), + ); + return; + } + + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", async () => { + let parsed; + try { + parsed = body ? JSON.parse(body) : {}; + } catch (err) { + debugLog("[ctxce] Failed to parse HTTP MCP request body: " + String(err)); + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32700, message: "Invalid JSON" }, + id: null, + }), + ); + return; + } + + try { + await transport.handleRequest(req, res, parsed); + } catch (err) { + debugLog("[ctxce] Error handling HTTP MCP request: " + String(err)); + if (!res.headersSent) { + res.statusCode = 500; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32603, message: "Internal server error" }, + id: null, + }), + ); + } + } + }); + return; + } + + // 404 for everything else + res.statusCode = 404; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32000, message: "Not found" }, + id: null, + }), + ); + } catch (err) { + debugLog("[ctxce] Unexpected error in HTTP MCP server: " + String(err)); + if (!res.headersSent) { + res.statusCode = 500; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32603, message: "Internal server error" }, + id: null, + }), + ); + } + } + }); + + // Bind to 127.0.0.1 only (localhost) for local-only OAuth security + httpServer.listen(port, '127.0.0.1', () => { + debugLog(`[ctxce] HTTP MCP bridge listening on 127.0.0.1:${port}`); + }); +} + +function loadConfig(startDir) { + try { + let dir = startDir; + for (let i = 0; i < 5; i += 1) { + const cfgPath = path.join(dir, "ctx_config.json"); + if (fs.existsSync(cfgPath)) { + try { + const raw = fs.readFileSync(cfgPath, "utf8"); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Failed to parse ctx_config.json:", err); + return null; + } + } + const parent = path.dirname(dir); + if (!parent || parent === dir) { + break; + } + dir = parent; + } + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Error while loading ctx_config.json:", err); + } + return null; +} + +function detectGitBranch(workspace) { + try { + const out = execSync("git rev-parse --abbrev-ref HEAD", { + cwd: workspace, + stdio: ["ignore", "pipe", "ignore"], + }); + const name = out.toString("utf8").trim(); + return name || null; + } catch { + return null; + } +} + +function detectRepoName(workspace, config) { + const envRepo = + (process.env.CURRENT_REPO && process.env.CURRENT_REPO.trim()) || + (process.env.REPO_NAME && process.env.REPO_NAME.trim()); + if (envRepo) { + return envRepo; + } + + if (config) { + const cfgRepo = + (typeof config.repo_name === "string" && config.repo_name.trim()) || + (typeof config.default_repo === "string" && config.default_repo.trim()); + if (cfgRepo) { + return cfgRepo; + } + } + + const leaf = workspace ? path.basename(workspace) : ""; + return leaf && SLUGGED_REPO_RE.test(leaf) ? leaf : null; +} + diff --git a/ctx-mcp-bridge/src/oauthHandler.js b/ctx-mcp-bridge/src/oauthHandler.js new file mode 100644 index 00000000..8d798ecb --- /dev/null +++ b/ctx-mcp-bridge/src/oauthHandler.js @@ -0,0 +1,585 @@ +// OAuth 2.0 Handler for HTTP MCP Server +// Implements RFC9728 Protected Resource Metadata and RFC7591 Dynamic Client Registration + +import fs from "node:fs"; +import { randomBytes } from "node:crypto"; +import { loadAnyAuthEntry, saveAuthEntry } from "./authConfig.js"; + +// ============================================================================ +// OAuth Storage (in-memory for bridge process) +// ============================================================================ + +// Maps bearer tokens to session IDs +const tokenStore = new Map(); +// Maps authorization codes to session info +const pendingCodes = new Map(); +// Maps client_id to client info +const registeredClients = new Map(); + +// ============================================================================ +// OAuth Utilities +// ============================================================================ + +/** + * Clean up expired tokens from tokenStore + * Called periodically to prevent unbounded memory growth + */ +function cleanupExpiredTokens() { + const now = Date.now(); + const expiryMs = 86400000; // 24 hours + for (const [token, data] of tokenStore.entries()) { + if (now - data.createdAt > expiryMs) { + tokenStore.delete(token); + } + } +} + +function generateToken() { + return randomBytes(32).toString("hex"); +} + +function generateCode() { + return randomBytes(16).toString("base64url"); +} + +function debugLog(message) { + try { + const text = typeof message === "string" ? message : String(message); + console.error(text); + const dest = process.env.CTXCE_DEBUG_LOG; + if (dest) { + fs.appendFileSync(dest, `${new Date().toISOString()} ${text}\n`, "utf8"); + } + } catch { + // ignore logging errors + } +} + +// ============================================================================ +// OAuth 2.0 Metadata (RFC9728) +// ============================================================================ + +export function getOAuthMetadata(issuerUrl) { + return { + issuer: issuerUrl, + authorization_endpoint: `${issuerUrl}/oauth/authorize`, + token_endpoint: `${issuerUrl}/oauth/token`, + registration_endpoint: `${issuerUrl}/oauth/register`, // RFC7591 Dynamic Client Registration + response_types_supported: ["code"], + grant_types_supported: ["authorization_code"], + token_endpoint_auth_methods_supported: ["none"], + code_challenge_methods_supported: ["S256"], + scopes_supported: ["mcp"], + }; +} + +// ============================================================================ +// HTML Login Page +// ============================================================================ + +/** + * Safely escape JSON for embedding in HTML script context + * Escapes special characters that could break out of a script tag + */ +function escapeJsonForHtml(obj) { + const json = JSON.stringify(obj); + // Replace dangerous characters with HTML-safe equivalents + // can break out of script tag, so replace /g, '\\u003E'); +} + +export function getLoginPage(redirectUri, clientId, state, codeChallenge, codeChallengeMethod) { + const params = new URLSearchParams({ + redirect_uri: redirectUri || "", + client_id: clientId || "", + state: state || "", + code_challenge: codeChallenge || "", + code_challenge_method: codeChallengeMethod || "", + }); + + return ` + + + Context Engine MCP - Login + + + +

Context Engine MCP Bridge

+
+ This MCP bridge requires authentication. Please log in to your Context Engine backend. +
+ +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ + + +`; +} + +// ============================================================================ +// OAuth Endpoint Handlers +// ============================================================================ + +/** + * Validate client_id and redirect_uri against registered clients + * @param {string} clientId - OAuth client_id + * @param {string} redirectUri - OAuth redirect_uri + * @returns {boolean} - true if both client_id and redirect_uri are valid + */ +function validateClientAndRedirect(clientId, redirectUri) { + if (!clientId || !redirectUri) { + return false; + } + const client = registeredClients.get(clientId); + if (!client) { + return false; + } + // Check if redirect_uri exactly matches one of the registered URIs + const redirectUris = client.redirectUris || []; + return redirectUris.includes(redirectUri); +} + +/** + * Handle OAuth metadata endpoint (RFC9728) + * GET /.well-known/oauth-authorization-server + */ +export function handleOAuthMetadata(_req, res, issuerUrl) { + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify(getOAuthMetadata(issuerUrl))); +} + +/** + * Handle OAuth Dynamic Client Registration (RFC7591) + * POST /oauth/register + */ +export function handleOAuthRegister(req, res) { + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", () => { + try { + const data = JSON.parse(body); + + // Validate required fields + if (!data.redirect_uris || !Array.isArray(data.redirect_uris) || data.redirect_uris.length === 0) { + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ error: "invalid_redirect_uri" })); + return; + } + + // Auto-approve any client registration for local bridge + const clientId = generateToken().slice(0, 32); + const client_id = `mcp_${clientId}`; + + registeredClients.set(client_id, { + clientId: client_id, + redirectUris: data.redirect_uris, + grantTypes: data.grant_types || ["authorization_code"], + createdAt: Date.now(), + }); + + res.setHeader("Content-Type", "application/json"); + res.statusCode = 201; + res.end(JSON.stringify({ + client_id: client_id, + client_id_issued_at: Math.floor(Date.now() / 1000), + grant_types: ["authorization_code"], + redirect_uris: data.redirect_uris, + response_types: ["code"], + token_endpoint_auth_method: "none", + })); + } catch (err) { + debugLog("[ctxce] /oauth/register error: " + String(err)); + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ error: "invalid_client_metadata", error_description: String(err) })); + } + }); +} + +/** + * Handle OAuth authorize endpoint + * GET /oauth/authorize + */ +export function handleOAuthAuthorize(_req, res, searchParams) { + const redirectUri = searchParams.get("redirect_uri"); + const clientId = searchParams.get("client_id"); + const state = searchParams.get("state"); + const responseType = searchParams.get("response_type"); + const codeChallenge = searchParams.get("code_challenge"); + const codeChallengeMethod = searchParams.get("code_challenge_method") || "S256"; + + // Validate response_type is "code" (authorization code flow) + if (responseType !== "code") { + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ error: "unsupported_response_type", error_description: "Only response_type=code is supported" })); + return; + } + + // Validate client_id and redirect_uri against registered clients + if (!validateClientAndRedirect(clientId, redirectUri)) { + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ error: "invalid_client", error_description: "Unknown client_id or unauthorized redirect_uri" })); + return; + } + + // If already logged in (has valid session), auto-approve + const existingAuth = loadAnyAuthEntry(); + if (existingAuth && existingAuth.entry && existingAuth.entry.sessionId) { + // Auto-generate code and redirect + const code = generateCode(); + pendingCodes.set(code, { + clientId, + sessionId: existingAuth.entry.sessionId, + backendUrl: existingAuth.backendUrl, + codeChallenge, + codeChallengeMethod, + redirectUri, + createdAt: Date.now(), + }); + + const redirectUrl = new URL(redirectUri || "http://localhost/callback"); + redirectUrl.searchParams.set("code", code); + if (state) redirectUrl.searchParams.set("state", state); + res.setHeader("Location", redirectUrl.toString()); + res.statusCode = 302; + res.end(); + return; + } + + // Otherwise, show login page + res.setHeader("Content-Type", "text/html"); + res.end(getLoginPage(redirectUri, clientId, state, codeChallenge, codeChallengeMethod)); +} + +/** + * Handle OAuth store-session endpoint (helper for login page) + * POST /oauth/store-session + * + * Security note: This endpoint is called from the browser after login. + * Since the HTTP server binds to 127.0.0.1 only, this is only accessible from localhost. + * For additional CSRF protection, we validate client_id and redirect_uri match a + * previously registered client. + */ +export function handleOAuthStoreSession(req, res) { + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", () => { + res.setHeader("Content-Type", "application/json"); + try { + const data = JSON.parse(body); + const { session_id, backend_url, redirect_uri, state, code_challenge, code_challenge_method, client_id } = data; + + if (!session_id || !backend_url) { + res.statusCode = 400; + res.end(JSON.stringify({ error: "Missing session_id or backend_url" })); + return; + } + + // Validate backend_url is a valid URL string (prevent prototype pollution) + // Only allow http/https schemes for backend API URLs + try { + const url = new URL(backend_url); + if (url.protocol !== "http:" && url.protocol !== "https:") { + res.statusCode = 400; + res.end(JSON.stringify({ error: "Invalid backend_url", error_description: "Only http/https URLs are allowed" })); + return; + } + } catch { + res.statusCode = 400; + res.end(JSON.stringify({ error: "Invalid backend_url" })); + return; + } + + // Validate client_id and redirect_uri against registered clients + // Note: client_id is passed from the login page which gets it from the initial auth request + if (!validateClientAndRedirect(client_id, redirect_uri)) { + res.statusCode = 400; + res.end(JSON.stringify({ error: "invalid_client", error_description: "Unknown client_id or unauthorized redirect_uri" })); + return; + } + + // Additional CSRF protection: verify request came from a local browser origin + // Require Origin or Referer header to be present and from localhost + const origin = req.headers["origin"] || req.headers["referer"]; + if (!origin) { + res.statusCode = 403; + res.end(JSON.stringify({ error: "forbidden", error_description: "Origin or Referer header required" })); + return; + } + try { + const originUrl = new URL(origin); + const hostname = originUrl.hostname; + // Only allow localhost or 127.0.0.1 origins + if (hostname !== "localhost" && hostname !== "127.0.0.1") { + res.statusCode = 403; + res.end(JSON.stringify({ error: "forbidden", error_description: "Request must originate from localhost" })); + return; + } + } catch { + // If origin parsing fails, reject the request + res.statusCode = 403; + res.end(JSON.stringify({ error: "forbidden", error_description: "Invalid origin" })); + return; + } + + // Save the auth entry + saveAuthEntry(backend_url, { + sessionId: session_id, + userId: "oauth-user", + expiresAt: null, + }); + + // Generate auth code + const code = generateCode(); + pendingCodes.set(code, { + clientId: client_id, + sessionId: session_id, + backendUrl: backend_url, + codeChallenge: code_challenge, + codeChallengeMethod: code_challenge_method, + redirectUri: redirect_uri, + createdAt: Date.now(), + }); + + // Return redirect URL + const redirectUrl = new URL(redirect_uri || "http://localhost/callback"); + redirectUrl.searchParams.set("code", code); + if (state) redirectUrl.searchParams.set("state", state); + + res.end(JSON.stringify({ redirect: redirectUrl.toString() })); + } catch (err) { + debugLog("[ctxce] /oauth/store-session error: " + String(err)); + res.statusCode = 400; + res.end(JSON.stringify({ error: String(err) })); + } + }); +} + +/** + * Handle OAuth token endpoint + * POST /oauth/token + */ +export function handleOAuthToken(req, res) { + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", () => { + try { + const data = new URLSearchParams(body); + const code = data.get("code"); + const redirectUri = data.get("redirect_uri"); + const clientId = data.get("client_id"); + // PKCE code_verifier - extracted but not validated yet (local bridge, trusted) + data.get("code_verifier"); + const grantType = data.get("grant_type"); + + res.setHeader("Content-Type", "application/json"); + + if (grantType !== "authorization_code") { + res.statusCode = 400; + res.end(JSON.stringify({ error: "unsupported_grant_type" })); + return; + } + + const pendingData = pendingCodes.get(code); + if (!pendingData) { + res.statusCode = 400; + res.end(JSON.stringify({ error: "invalid_grant", error_description: "Invalid or expired code" })); + return; + } + + // Check code age (10 minute expiry) + if (Date.now() - pendingData.createdAt > 600000) { + pendingCodes.delete(code); + res.statusCode = 400; + res.end(JSON.stringify({ error: "invalid_grant", error_description: "Code expired" })); + return; + } + + // Validate client_id matches the one used in authorize request + // This prevents code leakage from being used by a different client + if (pendingData.clientId !== clientId) { + pendingCodes.delete(code); + res.statusCode = 400; + res.end(JSON.stringify({ error: "invalid_client", error_description: "client_id mismatch" })); + return; + } + + // Validate redirect_uri matches the one used in authorize request + // This prevents code interception from being used on a different redirect URI + if (pendingData.redirectUri !== redirectUri) { + pendingCodes.delete(code); + res.statusCode = 400; + res.end(JSON.stringify({ error: "invalid_grant", error_description: "redirect_uri mismatch" })); + return; + } + + // TODO: Validate PKCE code_verifier against code_challenge + // For now, skip validation (local bridge, trusted) + + // Clean up expired tokens periodically to prevent unbounded growth + cleanupExpiredTokens(); + + // Generate access token + const accessToken = generateToken(); + tokenStore.set(accessToken, { + sessionId: pendingData.sessionId, + backendUrl: pendingData.backendUrl, + createdAt: Date.now(), + }); + + // Clean up pending code + pendingCodes.delete(code); + + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ + access_token: accessToken, + token_type: "Bearer", + expires_in: 86400, // 24 hours + scope: "mcp", + })); + } catch (err) { + debugLog("[ctxce] /oauth/token error: " + String(err)); + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ error: "invalid_request" })); + } + }); +} + +/** + * Validate Bearer token and return session info + * @param {string} token - Bearer token + * @returns {{sessionId: string, backendUrl: string} | null} + */ +export function validateBearerToken(token) { + const tokenData = tokenStore.get(token); + if (!tokenData) { + return null; + } + + // Check token age (24 hour expiry) + const tokenAge = Date.now() - tokenData.createdAt; + if (tokenAge > 86400000) { + tokenStore.delete(token); + return null; + } + + return { + sessionId: tokenData.sessionId, + backendUrl: tokenData.backendUrl, + }; +} + +/** + * Check if a given pathname is an OAuth endpoint + * @param {string} pathname - URL pathname + * @returns {boolean} + */ +export function isOAuthEndpoint(pathname) { + return ( + pathname === "/.well-known/oauth-authorization-server" || + pathname === "/oauth/register" || + pathname === "/oauth/authorize" || + pathname === "/oauth/store-session" || + pathname === "/oauth/token" + ); +} diff --git a/ctx-mcp-bridge/src/resultPathMapping.js b/ctx-mcp-bridge/src/resultPathMapping.js new file mode 100644 index 00000000..a3309876 --- /dev/null +++ b/ctx-mcp-bridge/src/resultPathMapping.js @@ -0,0 +1,501 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; + +function envTruthy(value, defaultVal = false) { + try { + if (value === undefined || value === null) { + return defaultVal; + } + const s = String(value).trim().toLowerCase(); + if (!s) { + return defaultVal; + } + return s === "1" || s === "true" || s === "yes" || s === "on"; + } catch { + return defaultVal; + } +} + +function _posixToNative(rel) { + try { + if (!rel) { + return ""; + } + return String(rel).split("/").join(path.sep); + } catch { + return rel; + } +} + +function _nativeToPosix(p) { + try { + if (!p) { + return ""; + } + return String(p).split(path.sep).join("/"); + } catch { + return p; + } +} + +function _workPathToRepoRelPosix(p) { + try { + const s = typeof p === "string" ? p.trim() : ""; + if (!s || !s.startsWith("/work/")) { + return null; + } + const rest = s.slice("/work/".length); + const parts = rest.split("/").filter(Boolean); + if (parts.length >= 2) { + return parts.slice(1).join("/"); + } + if (parts.length === 1) { + return parts[0]; + } + return ""; + } catch { + return null; + } +} + +function normalizeToolArgPath(p, workspaceRoot) { + try { + const s = typeof p === "string" ? p.trim() : ""; + if (!s) { + return p; + } + + const root = typeof workspaceRoot === "string" ? workspaceRoot : ""; + const sPosix = s.replace(/\\/g, "/"); + + const fromWork = _workPathToRepoRelPosix(sPosix); + if (typeof fromWork === "string" && fromWork) { + return fromWork; + } + if (fromWork === "") { + return p; + } + + if (root) { + try { + const sNorm = s.replace(/\\/g, path.sep); + const rootNorm = root.replace(/\\/g, path.sep); + if (sNorm === rootNorm || sNorm.startsWith(rootNorm + path.sep)) { + const relNative = path.relative(rootNorm, sNorm); + const relPosix = _nativeToPosix(relNative); + if (relPosix && relPosix !== "." && relPosix !== ".." && !relPosix.startsWith("../")) { + return relPosix; + } + } + } catch { + // ignore + } + try { + const base = path.posix.basename(root.replace(/\\/g, "/")); + if (base && sPosix.startsWith(base + "/")) { + const rest = sPosix.slice((base + "/").length); + if (rest && rest !== "." && rest !== ".." && !rest.startsWith("../")) { + return rest; + } + } + } catch { + // ignore + } + } + + if (sPosix.startsWith("./")) { + const rest = sPosix.slice(2); + if (rest && rest !== "." && rest !== ".." && !rest.startsWith("../")) { + return rest; + } + } + if (sPosix === ".") { + return ""; + } + return p; + } catch { + return p; + } +} + +function normalizeToolArgGlob(p, workspaceRoot) { + try { + const s = typeof p === "string" ? p : ""; + if (!s) { + return p; + } + // TODO(ctxce): If this becomes annoying, consider making glob normalization + // more conservative (e.g. only strip a repo prefix when followed by "/", + // and avoid collapsing "/**" into "**" which can broaden scope). + if (s.startsWith("!")) { + const rest = s.slice(1); + const mapped = normalizeToolArgPath(rest, workspaceRoot); + if (typeof mapped === "string") { + return "!" + mapped; + } + return p; + } + return normalizeToolArgPath(s, workspaceRoot); + } catch { + return p; + } +} + +function applyPathMappingToArgs(value, workspaceRoot, keyHint = "") { + try { + if (value === null || value === undefined) { + return value; + } + + const key = typeof keyHint === "string" ? keyHint : ""; + const lowered = key.toLowerCase(); + const shouldMapString = + lowered === "path" || + lowered === "under" || + lowered === "root" || + lowered === "subdir" || + lowered === "path_glob" || + lowered === "not_glob"; + + if (typeof value === "string") { + if (!shouldMapString) { + return value; + } + if (lowered === "path_glob" || lowered === "not_glob") { + return normalizeToolArgGlob(value, workspaceRoot); + } + return normalizeToolArgPath(value, workspaceRoot); + } + + if (Array.isArray(value)) { + return value.map((v) => applyPathMappingToArgs(v, workspaceRoot, keyHint)); + } + + if (typeof value === "object") { + const out = { ...value }; + for (const [k, v] of Object.entries(out)) { + out[k] = applyPathMappingToArgs(v, workspaceRoot, k); + } + return out; + } + + return value; + } catch { + return value; + } +} + +function computeWorkspaceRelativePath(containerPath, hostPath) { + try { + const cont = typeof containerPath === "string" ? containerPath.trim() : ""; + const rel = _workPathToRepoRelPosix(cont); + if (typeof rel === "string" && rel) { + return rel; + } + } catch { + } + try { + const hp = typeof hostPath === "string" ? hostPath.trim() : ""; + if (!hp) { + return ""; + } + // If we don't have a container path, at least try to return a basename. + return path.posix.basename(hp.replace(/\\/g, "/")); + } catch { + return ""; + } +} + +function remapRelatedPathToClient(p, workspaceRoot) { + try { + const s = typeof p === "string" ? p : ""; + const root = typeof workspaceRoot === "string" ? workspaceRoot : ""; + if (!s || !root) { + return p; + } + + const sNorm = s.replace(/\\/g, path.sep); + if (sNorm.startsWith(root + path.sep) || sNorm === root) { + return sNorm; + } + + const rel = _workPathToRepoRelPosix(s); + if (typeof rel === "string" && rel) { + const relNative = _posixToNative(rel); + return path.join(root, relNative); + } + + // If it's already a relative path, join it to the workspace root. + if (!s.startsWith("/") && !s.includes(":") && !s.includes("\\")) { + const relPosix = s.trim(); + if (relPosix && relPosix !== "." && !relPosix.startsWith("../") && relPosix !== "..") { + const relNative = _posixToNative(relPosix); + const joined = path.join(root, relNative); + const relCheck = path.relative(root, joined); + if (relCheck && !relCheck.startsWith(`..${path.sep}`) && relCheck !== "..") { + return joined; + } + } + } + + return p; + } catch { + return p; + } +} + +function remapHitPaths(hit, workspaceRoot) { + if (!hit || typeof hit !== "object") { + return hit; + } + const rawPath = typeof hit.path === "string" ? hit.path : ""; + let hostPath = typeof hit.host_path === "string" ? hit.host_path : ""; + let containerPath = typeof hit.container_path === "string" ? hit.container_path : ""; + if (!hostPath && rawPath) { + hostPath = rawPath; + } + if (!containerPath && rawPath) { + containerPath = rawPath; + } + const relPath = computeWorkspaceRelativePath(containerPath, hostPath); + const out = { ...hit }; + if (relPath) { + out.rel_path = relPath; + } + // Remap related_paths nested under each hit (repo_search/hybrid_search emit this per result). + try { + if (Array.isArray(out.related_paths)) { + out.related_paths = out.related_paths.map((p) => remapRelatedPathToClient(p, workspaceRoot)); + } + } catch { + // ignore + } + if (workspaceRoot && relPath) { + try { + const relNative = _posixToNative(relPath); + const candidate = path.join(workspaceRoot, relNative); + const diagnostics = envTruthy(process.env.CTXCE_BRIDGE_PATH_DIAGNOSTICS, false); + const strictClientPath = envTruthy(process.env.CTXCE_BRIDGE_CLIENT_PATH_STRICT, false); + if (strictClientPath) { + out.client_path = candidate; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "workspace_join"; + } + } else { + // Prefer a host_path that is within the current bridge workspace. + // This keeps provenance (host_path) intact while providing a user-local + // absolute path even when the bridge workspace is a parent directory. + const hp = typeof hostPath === "string" ? hostPath : ""; + const hpNorm = hp ? hp.replace(/\\/g, path.sep) : ""; + if ( + hpNorm && + hpNorm.startsWith(workspaceRoot) && + (!fs.existsSync(candidate) || fs.existsSync(hpNorm)) + ) { + out.client_path = hpNorm; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "host_path"; + } + } else { + out.client_path = candidate; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "workspace_join"; + } + } + } + } catch { + // ignore + } + } + const overridePath = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (overridePath) { + if (typeof out.client_path === "string" && out.client_path) { + out.path = out.client_path; + } else if (relPath) { + out.path = relPath; + } + } + // Strip internal container_path before returning to client. + delete out.container_path; + return out; +} + +function remapStringPath(p, workspaceRoot) { + try { + const s = typeof p === "string" ? p : ""; + if (!s) { + return p; + } + // If this is already a path within the current client workspace, rewrite to a + // workspace-relative string when override is enabled. + try { + const root = typeof workspaceRoot === "string" ? workspaceRoot : ""; + if (root) { + const sNorm = s.replace(/\\/g, path.sep); + if (sNorm.startsWith(root + path.sep) || sNorm === root) { + const relNative = path.relative(root, sNorm); + const relPosix = String(relNative).split(path.sep).join("/"); + if (relPosix && !relPosix.startsWith("../") && relPosix !== "..") { + const override = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (override) { + return relPosix; + } + } + } + } + } catch { + // ignore + } + const rel = _workPathToRepoRelPosix(s); + if (typeof rel === "string" && rel) { + const override = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (override) { + return rel; + } + return p; + } + return p; + } catch { + return p; + } +} + +function maybeParseToolJson(result) { + try { + if ( + result && + typeof result === "object" && + result.structuredContent && + typeof result.structuredContent === "object" + ) { + return { mode: "structured", value: result.structuredContent }; + } + } catch { + } + try { + const content = result && result.content; + if (!Array.isArray(content)) { + return null; + } + const first = content.find( + (c) => c && c.type === "text" && typeof c.text === "string", + ); + if (!first) { + return null; + } + const txt = String(first.text || "").trim(); + if (!txt || !(txt.startsWith("{") || txt.startsWith("["))) { + return null; + } + return { mode: "text", value: JSON.parse(txt) }; + } catch { + return null; + } +} + +function applyPathMappingToPayload(payload, workspaceRoot) { + if (!payload || typeof payload !== "object") { + return payload; + } + const out = Array.isArray(payload) ? payload.slice() : { ...payload }; + + const mapHitsArray = (arr) => { + if (!Array.isArray(arr)) { + return arr; + } + return arr.map((h) => remapHitPaths(h, workspaceRoot)); + }; + + // Common result shapes across tools + if (Array.isArray(out.results)) { + out.results = mapHitsArray(out.results); + } + if (Array.isArray(out.citations)) { + out.citations = mapHitsArray(out.citations); + } + if (Array.isArray(out.related_paths)) { + out.related_paths = out.related_paths.map((p) => remapRelatedPathToClient(p, workspaceRoot)); + } + + // Some tools nest under {result:{...}} + if (out.result && typeof out.result === "object") { + out.result = applyPathMappingToPayload(out.result, workspaceRoot); + } + + return out; +} + +export function maybeRemapToolResult(name, result, workspaceRoot) { + try { + if (!name || !result || !workspaceRoot) { + return result; + } + const enabled = envTruthy(process.env.CTXCE_BRIDGE_MAP_PATHS, true); + if (!enabled) { + return result; + } + const lower = String(name).toLowerCase(); + const shouldMap = ( + lower === "repo_search" || + lower === "context_search" || + lower === "context_answer" || + lower.endsWith("search_tests_for") || + lower.endsWith("search_config_for") || + lower.endsWith("search_callers_for") || + lower.endsWith("search_importers_for") + ); + if (!shouldMap) { + return result; + } + + const parsed = maybeParseToolJson(result); + if (!parsed) { + return result; + } + + const mapped = applyPathMappingToPayload(parsed.value, workspaceRoot); + let outResult = result; + if (parsed.mode === "structured") { + outResult = { ...result, structuredContent: mapped }; + } + + // Replace text payload for clients that only read `content[].text` + try { + const content = Array.isArray(outResult.content) ? outResult.content.slice() : []; + const idx = content.findIndex( + (c) => c && c.type === "text" && typeof c.text === "string", + ); + if (idx >= 0) { + content[idx] = { ...content[idx], text: JSON.stringify(mapped) }; + outResult = { ...outResult, content }; + } + } catch { + // ignore + } + return outResult; + } catch { + return result; + } +} + +export function maybeRemapToolArgs(name, args, workspaceRoot) { + try { + if (!name || !workspaceRoot) { + return args; + } + const enabled = envTruthy(process.env.CTXCE_BRIDGE_MAP_ARGS, true); + if (!enabled) { + return args; + } + if (args === null || args === undefined || typeof args !== "object") { + return args; + } + return applyPathMappingToArgs(args, workspaceRoot, ""); + } catch { + return args; + } +} diff --git a/ctx_config.example.json b/ctx_config.example.json new file mode 100644 index 00000000..81eb779e --- /dev/null +++ b/ctx_config.example.json @@ -0,0 +1,19 @@ +{ + "default_collection": "codebase", + "refrag_runtime": "glm", + "glm_api_key": "", + "glm_api_base": "https://api.z.ai/api/coding/paas/v4/", + "glm_model": "glm-4.6", + "always_include_tests": true, + "prefer_bullet_commands": false, + "extra_instructions": "Always consider error handling and edge cases", + "default_mode": "unicorn", + "streaming": true, + "require_context": true, + "relevance_gate_enabled": false, + "min_relevance": 0.1, + "rewrite_max_tokens": 420, + "surface_qdrant_collection_hint": true + +} + diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile new file mode 100644 index 00000000..76d9df7d --- /dev/null +++ b/deploy/kubernetes/Makefile @@ -0,0 +1,205 @@ +# Context-Engine Kubernetes Deployment Makefile + +# Configuration +NAMESPACE ?= context-engine +IMAGE_REGISTRY ?= context-engine +IMAGE_TAG ?= latest + +# Default target +.PHONY: help +help: ## Show this help message + @echo "Context-Engine Kubernetes Deployment Commands" + @echo "" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +# Prerequisites +.PHONY: check-kubectl +check-kubectl: ## Check if kubectl is available and cluster is accessible + @which kubectl > /dev/null || (echo "kubectl not found. Please install kubectl." && exit 1) + @kubectl cluster-info > /dev/null || (echo "Cannot connect to Kubernetes cluster." && exit 1) + @echo "✓ Kubernetes connection verified" + +# Deployment targets +.PHONY: deploy +deploy: check-kubectl ## Deploy all Context-Engine services + ./deploy.sh --namespace $(NAMESPACE) --registry $(IMAGE_REGISTRY) --tag $(IMAGE_TAG) + +.PHONY: deploy-core +deploy-core: check-kubectl ## Deploy only core services (Qdrant + MCP servers) + @echo "Deploying core services..." + kubectl apply -f namespace.yaml + kubectl apply -f configmap.yaml + kubectl apply -f qdrant.yaml + kubectl apply -f mcp-memory.yaml + kubectl apply -f mcp-indexer.yaml + +.PHONY: deploy-full +deploy-full: check-kubectl ## Deploy all services including optional ones + ./deploy.sh --namespace $(NAMESPACE) --registry $(IMAGE_REGISTRY) --tag $(IMAGE_TAG) --deploy-ingress + +.PHONY: deploy-minimal +deploy-minimal: check-kubectl ## Deploy minimal setup (skip Llama.cpp and Ingress) + ./deploy.sh --namespace $(NAMESPACE) --registry $(IMAGE_REGISTRY) --tag $(IMAGE_TAG) --skip-llamacpp + +# Kustomize targets +.PHONY: kustomize-build +kustomize-build: ## Build manifests with Kustomize + kustomize build . + +.PHONY: kustomize-apply +kustomize-apply: check-kubectl ## Apply manifests with Kustomize + kustomize build . | kubectl apply -f - + +.PHONY: kustomize-delete +kustomize-delete: check-kubectl ## Delete manifests with Kustomize + kustomize build . | kubectl delete -f - +# Management targets +.PHONY: status +status: check-kubectl ## Show deployment status + @echo "=== Namespace Status ===" + kubectl get namespace $(NAMESPACE) || echo "Namespace $(NAMESPACE) not found" + @echo "" + @echo "=== Pods ===" + kubectl get pods -n $(NAMESPACE) -o wide || echo "No pods found" + @echo "" + @echo "=== Services ===" + kubectl get services -n $(NAMESPACE) || echo "No services found" + @echo "" + @echo "=== Deployments ===" + kubectl get deployments -n $(NAMESPACE) || echo "No deployments found" + @echo "" + @echo "=== StatefulSets ===" + kubectl get statefulsets -n $(NAMESPACE) || echo "No statefulsets found" + @echo "" + @echo "=== PersistentVolumeClaims ===" + kubectl get pvc -n $(NAMESPACE) || echo "No PVCs found" + @echo "" + @echo "=== Jobs ===" + kubectl get jobs -n $(NAMESPACE) || echo "No jobs found" + +.PHONY: logs +logs: check-kubectl ## Show logs for core services (tail 100) + @echo "=== Qdrant Logs ===" + kubectl logs -f statefulset/qdrant -n $(NAMESPACE) --tail=100 || echo "Qdrant logs not available" + @echo "" + @echo "=== MCP Memory Logs ===" + kubectl logs -f deployment/mcp-memory -n $(NAMESPACE) --tail=100 || echo "MCP Memory logs not available" + @echo "" + @echo "=== MCP Indexer Logs ===" + kubectl logs -f deployment/mcp-indexer -n $(NAMESPACE) --tail=100 || echo "MCP Indexer logs not available" + @echo "" + @echo "=== Watcher Logs ===" + kubectl logs -f deployment/watcher -n $(NAMESPACE) --tail=100 || echo "Watcher logs not available" + +.PHONY: logs-service +logs-service: check-kubectl ## Show logs for specific service (usage: make logs-service SERVICE=mcp-memory) + @if [ -z "$(SERVICE)" ]; then echo "Usage: make logs-service SERVICE="; exit 1; fi + kubectl logs -f deployment/$(SERVICE) -n $(NAMESPACE) --tail=100 || kubectl logs -f statefulset/$(SERVICE) -n $(NAMESPACE) --tail=100 || kubectl logs -f job/$(SERVICE) -n $(NAMESPACE) --tail=100 || echo "Service $(SERVICE) not found" + +.PHONY: shell +shell: check-kubectl ## Get a shell in a running pod (usage: make shell POD=mcp-memory-xxx) + @if [ -z "$(POD)" ]; then echo "Usage: make shell POD="; echo "Available pods:"; kubectl get pods -n $(NAMESPACE); exit 1; fi + kubectl exec -it $(POD) -n $(NAMESPACE) -- /bin/bash || kubectl exec -it $(POD) -n $(NAMESPACE) -- /bin/sh + +# Cleanup targets +.PHONY: cleanup +cleanup: check-kubectl ## Remove all Context-Engine resources + ./cleanup.sh --namespace $(NAMESPACE) + +.PHONY: clean-force +clean-force: check-kubectl ## Force cleanup without confirmation + ./cleanup.sh --namespace $(NAMESPACE) --force + +# Development targets +.PHONY: restart +restart: check-kubectl ## Restart all deployments + kubectl rollout restart deployment -n $(NAMESPACE) + kubectl rollout restart statefulset -n $(NAMESPACE) + +.PHONY: restart-service +restart-service: check-kubectl ## Restart specific service (usage: make restart-service SERVICE=mcp-memory) + @if [ -z "$(SERVICE)" ]; then echo "Usage: make restart-service SERVICE="; exit 1; fi + kubectl rollout restart deployment/$(SERVICE) -n $(NAMESPACE) || kubectl rollout restart statefulset/$(SERVICE) -n $(NAMESPACE) + +.PHONY: scale +scale: check-kubectl ## Scale a deployment (usage: make scale SERVICE=mcp-memory REPLICAS=3) + @if [ -z "$(SERVICE)" ] || [ -z "$(REPLICAS)" ]; then echo "Usage: make scale SERVICE= REPLICAS="; exit 1; fi + kubectl scale deployment $(SERVICE) -n $(NAMESPACE) --replicas=$(REPLICAS) + +# Port forwarding targets +.PHONY: port-forward +port-forward: check-kubectl ## Port forward all services + @echo "Opening port forwards in background..." + @kubectl port-forward -n $(NAMESPACE) service/qdrant 6333:6333 & + @kubectl port-forward -n $(NAMESPACE) service/mcp-memory 8000:8000 & + @kubectl port-forward -n $(NAMESPACE) service/mcp-indexer 8001:8001 & + @echo "Port forwards started. Use 'make stop-port-forward' to stop." + +.PHONY: port-forward-service +port-forward-service: check-kubectl ## Port forward specific service (usage: make port-forward-service SERVICE=qdrant LOCAL=6333 REMOTE=6333) + @if [ -z "$(SERVICE)" ] || [ -z "$(LOCAL)" ] || [ -z "$(REMOTE)" ]; then echo "Usage: make port-forward-service SERVICE= LOCAL= REMOTE="; exit 1; fi + kubectl port-forward -n $(NAMESPACE) service/$(SERVICE) $(LOCAL):$(REMOTE) + +.PHONY: stop-port-forward +stop-port-forward: ## Stop all port forwards + pkill -f "kubectl port-forward" || echo "No port forwards found" + +.PHONY: build-image +build-image: ## Build Docker image (requires Docker) + docker build -t $(IMAGE_REGISTRY)/context-engine:$(IMAGE_TAG) ../../ + +.PHONY: push-image +push-image: build-image ## Push Docker image to registry + docker push $(IMAGE_REGISTRY)/context-engine:$(IMAGE_TAG) + +# Test targets +.PHONY: test-connection +test-connection: check-kubectl ## Test connectivity to all services + @echo "Testing service connectivity..." + @echo "Qdrant:" + @kubectl run qdrant-test --image=curlimages/curl --rm -i --restart=Never -n $(NAMESPACE) -- curl -f http://qdrant.$(NAMESPACE).svc.cluster.local:6333/health || echo "Qdrant test failed" + @echo "MCP Memory:" + @kubectl run memory-test --image=curlimages/curl --rm -i --restart=Never -n $(NAMESPACE) -- curl -f http://mcp-memory.$(NAMESPACE).svc.cluster.local:8000/health || echo "MCP Memory test failed" + @echo "MCP Indexer:" + @kubectl run indexer-test --image=curlimages/curl --rm -i --restart=Never -n $(NAMESPACE) -- curl -f http://mcp-indexer.$(NAMESPACE).svc.cluster.local:8001/health || echo "MCP Indexer test failed" + +# Configuration targets +.PHONY: show-config +show-config: ## Show current configuration + @echo "Configuration:" + @echo " NAMESPACE: $(NAMESPACE)" + @echo " IMAGE_REGISTRY: $(IMAGE_REGISTRY)" + @echo " IMAGE_TAG: $(IMAGE_TAG)" + @echo "" + @echo "Quick start commands:" + @echo " make deploy # Deploy all services" + @echo " make status # Show deployment status" + @echo " make logs # Show logs" + @echo " make cleanup # Remove everything" + +.PHONY: show-urls +show-urls: check-kubectl ## Show access URLs for services + @echo "Service URLs (via NodePort):" + @echo " Qdrant: http://:30333" + @echo " MCP Memory (SSE): http://:30800" + @echo " MCP Indexer (SSE): http://:30802" + @echo " MCP Memory (HTTP): http://:30804" + @echo " MCP Indexer (HTTP): http://:30806" + @echo " Llama.cpp: http://:30808" + @echo "" + @echo "Service URLs (via port-forward):" + @echo " make port-forward # Then access via localhost ports" + +# Advanced targets +.PHONY: watch-deployment +watch-deployment: check-kubectl ## Watch deployment progress + watch kubectl get pods,services,deployments -n $(NAMESPACE) + +.PHONY: describe-service +describe-service: check-kubectl ## Describe a service (usage: make describe-service SERVICE=mcp-memory) + @if [ -z "$(SERVICE)" ]; then echo "Usage: make describe-service SERVICE="; echo "Available services:"; kubectl get services -n $(NAMESPACE); exit 1; fi + kubectl describe service $(SERVICE) -n $(NAMESPACE) + +.PHONY: events +events: check-kubectl ## Show recent events + kubectl get events -n $(NAMESPACE) --sort-by=.metadata.creationTimestamp diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md new file mode 100644 index 00000000..a84050e8 --- /dev/null +++ b/deploy/kubernetes/README.md @@ -0,0 +1,561 @@ +# Kubernetes Deployment Guide + +**Documentation:** [README](../../README.md) · [Configuration](../../docs/CONFIGURATION.md) · [IDE Clients](../../docs/IDE_CLIENTS.md) · [MCP API](../../docs/MCP_API.md) · [ctx CLI](../../docs/CTX_CLI.md) · [Memory Guide](../../docs/MEMORY_GUIDE.md) · [Architecture](../../docs/ARCHITECTURE.md) · [Multi-Repo](../../docs/MULTI_REPO_COLLECTIONS.md) · Kubernetes · [VS Code Extension](../../docs/vscode-extension.md) · [Troubleshooting](../../docs/TROUBLESHOOTING.md) · [Development](../../docs/DEVELOPMENT.md) + +--- + +## Overview + +This directory contains Kubernetes manifests for deploying Context Engine on a remote cluster using **Kustomize**. This enables: + +- **Remote development** from thin clients with cluster-based heavy lifting +- **Multi-repository indexing** with unified `codebase` collection +- **Scalable architecture** with independent watcher deployments per repo +- **Kustomize-based configuration** for easy customization and overlays + +## Architecture + +```mermaid +graph TB + subgraph cluster["Kubernetes Cluster (namespace: context-engine)"] + subgraph ingress["Ingress Layer"] + nginx["NGINX Ingress
Routes: /qdrant, /mcp/*, /mcp-http/*, /llamacpp"] + end + + subgraph services["Core Services"] + qdrant["Qdrant StatefulSet
Port: 6333
Vector Database"] + + subgraph mcp["MCP Services (4 Deployments)"] + mcp_mem_sse["Memory SSE
Port: 8000"] + mcp_mem_http["Memory HTTP
Port: 8002"] + mcp_idx_sse["Indexer SSE
Port: 8001
(HPA: 1-5 replicas)"] + mcp_idx_http["Indexer HTTP
Port: 8003"] + end + + llama["Llama.cpp Deployment
Port: 8080
Init: Model Download"] + watcher["Watcher Deployment
Watches: /work"] + end + + subgraph security["Security & Scaling"] + rbac["RBAC: ServiceAccount
(context-engine)"] + netpol["NetworkPolicy
Intra-namespace ingress
for watcher/indexer/init"] + hpa["HPA: mcp-indexer
1-5 replicas @ 70% CPU"] + end + + subgraph storage["Persistent Storage (HostPath)"] + qdrant_vol["Qdrant Data
/tmp/context-engine-qdrant"] + models_vol["LLM Models
/tmp/context-engine-models"] + work_vol["Workspace
/tmp/context-engine-work"] + end + end + + nginx --> qdrant + nginx --> mcp + nginx --> llama + + mcp --> qdrant + llama -.-> mcp + watcher --> qdrant + + qdrant --> qdrant_vol + llama --> models_vol + watcher --> work_vol + + style nginx fill:#e1f5ff + style qdrant fill:#fff4e1 + style mcp fill:#e8f5e9 + style llama fill:#f3e5f5 + style watcher fill:#fce4ec + style security fill:#fff9c4 + style storage fill:#e0e0e0 +``` + +## Quick Start + +### Prerequisites + +- Kubernetes cluster (1.19+) +- `kubectl` configured to access your cluster +- `kustomize` (optional, kubectl has built-in support) +- Docker image built and pushed to a registry + +### 1. Build and Push Image + +```bash +# Build unified image +docker build -t your-registry/context-engine:latest . + +# Push to registry +docker push your-registry/context-engine:latest +``` + +### 2. Update Image References + +Edit `kustomization.yaml` to use your registry: + +```yaml +images: + - name: context-engine + newName: your-registry/context-engine + newTag: latest +``` + +### 3. Deploy Using Kustomize + +```bash +# Option 1: Using the deploy script with Kustomize (recommended) +./deploy.sh --use-kustomize --registry your-registry/context-engine --tag latest --deploy-ingress + +# Option 2: Using kubectl with kustomize directly +kubectl apply -k . + +# Option 3: Using kustomize CLI +kustomize build . | kubectl apply -f - + +# Option 4: Using the deploy script without Kustomize (legacy) +./deploy.sh --registry your-registry/context-engine --tag latest --deploy-ingress +``` + +**Deploy Script Flags:** +- `--use-kustomize`: Use Kustomize for declarative image management (recommended) +- `--registry `: Docker registry and image name (default: context-engine) +- `--tag `: Image tag (default: latest) +- `--deploy-ingress`: Deploy NGINX ingress routes +- `--skip-llamacpp`: Skip llama.cpp decoder deployment + +### 4. Deploy Using Makefile + +```bash +# Deploy all services +make deploy + +# Or deploy core services only +make deploy-core + +# Check status +make status +``` + +### 5. Verify Deployment + +```bash +# Check all pods are running +kubectl get pods -n context-engine + +# Check services +kubectl get svc -n context-engine + +# View logs +make logs-service SERVICE=mcp-memory +``` + +### 6. Access Services + +```bash +# Port forward to localhost +make port-forward + +# Or access via NodePort +# Qdrant: http://:30333 +# MCP Memory: http://:30800 +# MCP Indexer: http://:30802 +``` + +## Configuration + +### Automatic Model Download + +The Llama.cpp deployment includes an **init container** that automatically downloads the model on first startup: + +- **Default Model**: Qwen2.5-1.5B-Instruct (Q8_0 quantization, ~1.7GB) +- **Download Location**: `/tmp/context-engine-models/` on the Kubernetes node +- **Behavior**: Downloads only if model doesn't exist (idempotent) +- **Good balance**: Fast, accurate, small footprint + +To use a different model, edit `configmap.yaml`: + +```yaml +# Model download configuration +LLAMACPP_MODEL_URL: "https://huggingface.co/your-org/your-model/resolve/main/model.gguf" +LLAMACPP_MODEL_NAME: "model.gguf" +``` + +**Alternative Models**: +- **Qwen2.5-0.5B-Instruct-Q8** (~500MB) - Tiny, very fast +- **Qwen2.5-1.5B-Instruct-Q8** (default, ~1.7GB) - Best balance +- **Granite-3.0-3B-Instruct-Q8** (~3.2GB) - Higher quality +- **Phi-3-mini-4k-instruct-Q8** (~4GB) - High quality + +### Environment Variables (ConfigMap) + +Key environment variables in `configmap.yaml`: + +```yaml +COLLECTION_NAME: "codebase" # Unified collection for all repos +EMBEDDING_MODEL: "BAAI/bge-base-en-v1.5" +QDRANT_URL: "http://qdrant:6333" +INDEX_MICRO_CHUNKS: "1" +MAX_MICRO_CHUNKS_PER_FILE: "200" +WATCH_DEBOUNCE_SECS: "1.5" +``` + +#### Syncing `configmap.yaml` from `.env` + +If you treat a `.env` file as the source of truth for configuration, you can use the helper script `scripts/sync_env_to_k8s.py` to keep `deploy/kubernetes/configmap.yaml` and the workloads in sync: + +```bash +cd /path/to/Context-Engine +python3 scripts/sync_env_to_k8s.py --env-file .env --k8s-dir deploy/kubernetes +``` + +This will: + +- Regenerate `deploy/kubernetes/configmap.yaml` so its `data:` keys match the provided `.env` (excluding sensitive keys such as `GLM_API_KEY` by default). +- Ensure all Deployments and Jobs in `deploy/kubernetes/` include: + + ```yaml + envFrom: + - configMapRef: + name: context-engine-config + ``` + +In CI (for example Bamboo), you can run the same script against the workspace copy of the manifests before `kustomize build . | kubectl apply -f -`, and then provide any sensitive values (such as `GLM_API_KEY`) via Kubernetes `Secret` resources or per-environment overrides instead of committing them to git. + +### Persistent Volumes + +The deployment uses HostPath volumes for simplicity (suitable for single-node clusters like minikube): + +1. **qdrant-storage**: Stores Qdrant vector database + - Path: `/tmp/context-engine-qdrant` + - Type: DirectoryOrCreate + +2. **models-storage**: Stores LLM models for llama.cpp + - Path: `/tmp/context-engine-models` + - Type: DirectoryOrCreate + +3. **work-storage**: Stores workspace/repository code + - Path: `/tmp/context-engine-work` + - Type: DirectoryOrCreate + - Mounted at: `/work` in containers + +For multi-node production clusters, replace HostPath with PersistentVolumeClaims (PVCs) backed by network storage (NFS, Ceph, cloud provider volumes). + +### Resource Requests/Limits + +Adjust based on your cluster capacity: + +```yaml +# Qdrant (memory-intensive) +resources: + requests: + memory: "4Gi" + cpu: "2" + limits: + memory: "8Gi" + cpu: "4" + +# MCP Servers (moderate) +resources: + requests: + memory: "2Gi" + cpu: "1" + limits: + memory: "4Gi" + cpu: "2" + +# Watchers (light) +resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" +``` + +## Workspace Setup + +### Indexing Your Codebase + +The deployment indexes the codebase mounted at `/work` inside containers, which maps to `/tmp/context-engine-work` on the host. + +1. **Copy your codebase to the workspace volume**: + ```bash + # For minikube (single-node) + minikube ssh + sudo mkdir -p /tmp/context-engine-work + exit + + # Copy your code + kubectl cp /local/path/to/your/repo context-engine/watcher-:/work + + # Or mount directly on the host + cp -r /local/path/to/your/repo /tmp/context-engine-work/ + ``` + +2. **Verify indexing**: + ```bash + # Check watcher logs + kubectl logs -f deployment/watcher -n context-engine + + # Check indexer job completion + kubectl get jobs -n context-engine + + # Check collection status via MCP + kubectl port-forward -n context-engine svc/mcp-indexer 8001:8001 + curl -X POST http://localhost:8001/mcp \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"qdrant_status","arguments":{}}}' + ``` + +### Workspace Volume Structure + +``` +/work/ # Container mount point +├── .codebase/ # Indexing metadata +│ └── state.json +├── src/ # Your source code +├── tests/ +├── docs/ +└── ... +``` + +**Note**: The current deployment uses a single workspace at `/work`. For multi-repository setups, you can: +- Use subdirectories under `/work` (e.g., `/work/backend`, `/work/frontend`) +- Deploy multiple watcher instances with different `WATCH_ROOT` environment variables +- Use a unified collection or separate collections per repository + +## Accessing Services + +### From Within Cluster + +Services are accessible via Kubernetes DNS: + +- Qdrant: `http://qdrant:6333` +- Memory MCP: `http://mcp-memory:8000/sse` +- Indexer MCP: `http://mcp-indexer:8001/sse` + +### From Outside Cluster + +#### Option 1: Port Forwarding (Development) + +```bash +# Forward MCP services to localhost +kubectl port-forward -n context-engine svc/mcp-memory 8000:8000 +kubectl port-forward -n context-engine svc/mcp-indexer 8001:8001 +kubectl port-forward -n context-engine svc/qdrant 6333:6333 +``` + +Then configure your IDE to use `http://localhost:8000/sse` and `http://localhost:8001/sse`. + +#### Option 2: Ingress (Production) + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: context-engine-ingress + namespace: context-engine +spec: + rules: + - host: mcp.your-domain.com + http: + paths: + - path: /memory + pathType: Prefix + backend: + service: + name: mcp-memory + port: + number: 8000 + - path: /indexer + pathType: Prefix + backend: + service: + name: mcp-indexer + port: + number: 8001 +``` + +#### Option 3: LoadBalancer (Cloud) + +Change service type to `LoadBalancer` in service manifests: + +```yaml +spec: + type: LoadBalancer + ports: + - port: 8000 + targetPort: 8000 +``` + +## Monitoring and Maintenance + +### Health Checks + +```bash +# Check Qdrant health +kubectl exec -n context-engine qdrant-0 -- curl -f http://localhost:6333/readyz + +# Check MCP server health +kubectl exec -n context-engine deployment/mcp-memory -- curl -f http://localhost:18000/health +kubectl exec -n context-engine deployment/mcp-indexer -- curl -f http://localhost:18001/health +``` + +### Logs + +```bash +# View logs for specific service +kubectl logs -f -n context-engine deployment/mcp-memory +kubectl logs -f -n context-engine deployment/mcp-indexer +kubectl logs -f -n context-engine deployment/watcher + +# View logs for all watchers (if multiple) +kubectl logs -f -n context-engine -l component=watcher +``` + +### Collection Status + +```bash +# Port forward indexer MCP +kubectl port-forward -n context-engine svc/mcp-indexer 8001:8001 + +# Check collection status +curl -X POST http://localhost:8001/mcp \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"qdrant_status","arguments":{}}}' +``` + +### Backup and Restore + +#### Backup Qdrant Data + +```bash +# Create snapshot +kubectl exec -n context-engine qdrant-0 -- \ + curl -X POST http://localhost:6333/collections/codebase/snapshots + +# Copy snapshot to local +kubectl cp context-engine/qdrant-0:/qdrant/storage/snapshots/codebase-snapshot.tar \ + ./backup/codebase-snapshot.tar +``` + +#### Restore Qdrant Data + +```bash +# Copy snapshot to pod +kubectl cp ./backup/codebase-snapshot.tar \ + context-engine/qdrant-0:/qdrant/storage/snapshots/ + +# Restore snapshot +kubectl exec -n context-engine qdrant-0 -- \ + curl -X PUT http://localhost:6333/collections/codebase/snapshots/upload \ + -F 'snapshot=@/qdrant/storage/snapshots/codebase-snapshot.tar' +``` + +## Troubleshooting + +### Pods Not Starting + +```bash +# Check pod status +kubectl describe pod -n context-engine + +# Check events +kubectl get events -n context-engine --sort-by='.lastTimestamp' +``` + +### Persistent Volume Issues + +```bash +# Check PV/PVC status +kubectl get pv,pvc -n context-engine + +# Check PVC events +kubectl describe pvc -n context-engine +``` + +### Watcher Not Indexing + +```bash +# Check watcher logs +kubectl logs -f -n context-engine deployment/watcher + +# Verify volume mount +kubectl exec -n context-engine deployment/watcher -- ls -la /work + +# Check Qdrant connectivity +kubectl exec -n context-engine deployment/watcher -- \ + curl -f http://qdrant:6333/readyz +``` + +### MCP Connection Issues + +```bash +# Test SSE endpoint +kubectl exec -n context-engine deployment/mcp-indexer -- \ + curl -H "Accept: text/event-stream" http://localhost:8001/sse + +# Check service endpoints +kubectl get endpoints -n context-engine +``` + +## Scaling + +### Horizontal Scaling + +- **MCP Servers**: Can run multiple replicas behind a service +- **Watchers**: One per repository (do not scale horizontally) +- **Qdrant**: Single instance (StatefulSet with replicas=1) + +### Vertical Scaling + +Adjust resource requests/limits based on workload: + +```bash +# Edit deployment +kubectl edit deployment -n context-engine mcp-indexer + +# Or patch +kubectl patch deployment -n context-engine mcp-indexer -p \ + '{"spec":{"template":{"spec":{"containers":[{"name":"mcp-indexer","resources":{"requests":{"memory":"4Gi"}}}]}}}}' +``` + +## Security Considerations + +### Implemented Security Features + +1. **RBAC (Role-Based Access Control)** + - ServiceAccount: `context-engine` created in `rbac.yaml` + - Applied to all Deployments and Jobs + - Provides pod identity for Kubernetes API authentication + - Future: Add Role/RoleBinding for fine-grained permissions + +2. **NetworkPolicy (Soft Hardening - Option B)** + - Policy: `allow-intra-namespace-ingress-internal` in `networkpolicy.yaml` + - Scope: Applies to watcher, indexer, and init pods + - Rules: Allows ingress only from pods in the same namespace + - No egress restrictions (external downloads and Qdrant access work) + - MCP services and Qdrant remain accessible via Ingress/NodePort + - Future: Implement Option A (default-deny with explicit allow rules) + +3. **HorizontalPodAutoscaler (HPA)** + - Target: mcp-indexer deployment + - Min replicas: 1, Max replicas: 5 + - Trigger: 70% CPU utilization + - Prevents resource exhaustion under load + +### Additional Security Recommendations + +4. **Secrets Management**: Use Kubernetes secrets or external secret managers for sensitive data +5. **TLS**: Enable TLS for external access via Ingress with cert-manager +6. **Resource Quotas**: Set namespace resource quotas to prevent resource exhaustion +7. **Pod Security Standards**: Apply restricted pod security standards +8. **Image Security**: Use signed images and vulnerability scanning + +## See Also + +- [Multi-Repository Collections Guide](../../docs/MULTI_REPO_COLLECTIONS.md) +- [MCP API Reference](../../docs/MCP_API.md) +- [Architecture Overview](../../docs/ARCHITECTURE.md) + diff --git a/deploy/kubernetes/cleanup.sh b/deploy/kubernetes/cleanup.sh new file mode 100755 index 00000000..dadfa6ec --- /dev/null +++ b/deploy/kubernetes/cleanup.sh @@ -0,0 +1,247 @@ +#!/bin/bash + +# Context-Engine Kubernetes Cleanup Script +# This script removes all Context-Engine resources from Kubernetes + +set -e + +# Configuration +NAMESPACE="context-engine" +FORCE=false + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if kubectl is available +check_kubectl() { + if ! command -v kubectl &> /dev/null; then + log_error "kubectl is not installed or not in PATH" + exit 1 + fi + + if ! kubectl cluster-info &> /dev/null; then + log_error "Cannot connect to Kubernetes cluster" + exit 1 + fi + + log_success "Kubernetes connection verified" +} + +# Check if namespace exists +check_namespace() { + if ! kubectl get namespace $NAMESPACE &> /dev/null; then + log_warning "Namespace $NAMESPACE does not exist" + return 1 + fi + return 0 +} + +# Show what will be deleted +show_deletion_plan() { + log_info "The following resources will be deleted:" + echo + + # Show current resources + echo "Pods:" + kubectl get pods -n $NAMESPACE 2>/dev/null || echo " No pods found" + echo + echo "Services:" + kubectl get services -n $NAMESPACE 2>/dev/null || echo " No services found" + echo + echo "Deployments:" + kubectl get deployments -n $NAMESPACE 2>/dev/null || echo " No deployments found" + echo + echo "StatefulSets:" + kubectl get statefulsets -n $NAMESPACE 2>/dev/null || echo " No statefulsets found" + echo + echo "Jobs:" + kubectl get jobs -n $NAMESPACE 2>/dev/null || echo " No jobs found" + echo + echo "PersistentVolumeClaims:" + kubectl get pvc -n $NAMESPACE 2>/dev/null || echo " No PVCs found" + echo + echo "ConfigMaps:" + kubectl get configmaps -n $NAMESPACE 2>/dev/null || echo " No configmaps found" + echo + if kubectl get ingress -n $NAMESPACE &> /dev/null; then + echo "Ingress:" + kubectl get ingress -n $NAMESPACE + echo + fi + + log_warning "This will permanently delete all data in Qdrant and any other persistent storage!" +} + +confirm_cleanup() { + if [[ "$FORCE" == "true" ]]; then + return 0 + fi + read -p "Are you sure you want to delete all Context-Engine resources? (yes/no): " -r + echo + if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then + log_info "Cleanup cancelled" + exit 0 + fi +} + +# Delete namespace and all resources +delete_namespace() { + log_info "Deleting namespace: $NAMESPACE" + kubectl delete namespace $NAMESPACE --ignore-not-found=true + log_success "Namespace deleted" +} + +# Wait for namespace deletion +wait_for_deletion() { + log_info "Waiting for namespace deletion to complete..." + + local timeout=60 + local count=0 + + while kubectl get namespace $NAMESPACE &> /dev/null; do + if [[ $count -ge $timeout ]]; then + log_warning "Namespace deletion is taking longer than expected" + log_info "You may need to manually delete remaining resources" + return 1 + fi + + echo -n "." + sleep 1 + ((count++)) + done + + echo + log_success "Namespace deletion completed" +} + +# Force delete if needed +force_delete() { + log_warning "Attempting to force delete remaining resources..." + + # Force delete any remaining pods + kubectl delete pods --all -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true + + # Force delete any remaining PVCs + kubectl delete pvc --all -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true + + log_success "Force delete completed" +} + +# Verify cleanup +verify_cleanup() { + log_info "Verifying cleanup..." + + if kubectl get namespace $NAMESPACE &> /dev/null; then + log_error "Namespace $NAMESPACE still exists" + return 1 + fi + + log_success "Cleanup completed successfully" +} + +# Main cleanup function +main() { + log_info "Starting Context-Engine Kubernetes cleanup" + + # Check prerequisites + check_kubectl + + # Check if namespace exists + if ! check_namespace; then + log_success "Nothing to clean up - namespace $NAMESPACE does not exist" + exit 0 + fi + + # Show what will be deleted + show_deletion_plan + + # Ask for confirmation (unless forced) + confirm_cleanup + + # Delete namespace + delete_namespace + + # Wait for deletion + if ! wait_for_deletion; then + log_warning "Standard deletion incomplete, attempting force delete..." + force_delete + fi + + # Verify cleanup + verify_cleanup + + log_success "Context-Engine cleanup completed!" +} + +# Help function +show_help() { + echo "Context-Engine Kubernetes Cleanup Script" + echo + echo "Usage: $0 [OPTIONS]" + echo + echo "Options:" + echo " -h, --help Show this help message" + echo " -n, --namespace NAMESPACE Kubernetes namespace (default: context-engine)" + echo " -f, --force Skip confirmation prompt" + echo + echo "Environment variables:" + echo " NAMESPACE=context-engine Kubernetes namespace" + echo + echo "Examples:" + echo " $0 # Interactive cleanup with confirmation" + echo " $0 --force # Cleanup without confirmation" + echo " $0 -n my-namespace # Cleanup different namespace" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + show_help + exit 0 + ;; + -n|--namespace) + NAMESPACE="$2" + shift 2 + ;; + -f|--force|--force=true) + FORCE=true + shift + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Check if we're in the right directory +if [[ ! -f "qdrant.yaml" ]]; then + log_error "Please run this script from the deploy/kubernetes directory" + exit 1 +fi + +# Run main cleanup +main diff --git a/deploy/kubernetes/code-models-pvc.yaml b/deploy/kubernetes/code-models-pvc.yaml new file mode 100644 index 00000000..0b01a027 --- /dev/null +++ b/deploy/kubernetes/code-models-pvc.yaml @@ -0,0 +1,18 @@ +--- +# Persistent Volume Claim for model storage (CephFS RWX) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: code-models-pvc + namespace: context-engine + labels: + app: context-engine + component: llamacpp + type: storage +spec: + accessModes: + - ReadWriteMany # CephFS supports RWX for multiple pods + storageClassName: ceph-filesystem # Adjust based on your storage class + resources: + requests: + storage: 20Gi # Adjust size based on expected model footprint diff --git a/deploy/kubernetes/configmap.yaml b/deploy/kubernetes/configmap.yaml new file mode 100644 index 00000000..9c91a893 --- /dev/null +++ b/deploy/kubernetes/configmap.yaml @@ -0,0 +1,145 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: context-engine-config + namespace: context-engine + labels: + app: context-engine + component: configuration +data: + COLLECTION_NAME: codebase + COMMIT_VECTOR_SEARCH: '0' + CTXCE_AUTH_ADMIN_TOKEN: change-me-admin-token + CTXCE_AUTH_ENABLED: '0' + CTX_SNIPPET_CHARS: '400' + CTX_SUMMARY_CHARS: '0' + CURRENT_REPO: '' + DECODER_MAX_TOKENS: '4000' + EMBEDDING_MODEL: BAAI/bge-base-en-v1.5 + EMBEDDING_PROVIDER: fastembed + EMBEDDING_WARMUP: '0' + FASTMCP_HOST: 0.0.0.0 + FASTMCP_HTTP_HEALTH_PORT: '18002' + FASTMCP_HTTP_PORT: '8002' + FASTMCP_HTTP_TRANSPORT: http + FASTMCP_INDEXER_HTTP_HEALTH_PORT: '18003' + FASTMCP_INDEXER_HTTP_PORT: '8003' + FASTMCP_INDEXER_PORT: '8001' + FASTMCP_PORT: '8000' + GLM_API_BASE: https://api.z.ai/api/coding/paas/v4/ + GLM_MODEL: glm-4.6 + HOST_INDEX_PATH: ./dev-workspace + HYBRID_EXPAND: '0' + HYBRID_IN_PROCESS: '1' + HYBRID_MINI_WEIGHT: '1.0' + HYBRID_PER_PATH: '1' + HYBRID_RECENCY_WEIGHT: '0.1' + HYBRID_RESULTS_CACHE: '128' + HYBRID_RESULTS_CACHE_ENABLED: '1' + HYBRID_SYMBOL_BOOST: '0.35' + INDEX_CHUNK_LINES: '60' + INDEX_CHUNK_OVERLAP: '10' + INDEX_MICRO_CHUNKS: '0' + INDEX_SEMANTIC_CHUNKS: '1' + INDEX_UPSERT_BACKOFF: '0.5' + INDEX_UPSERT_BATCH: '128' + INDEX_UPSERT_RETRIES: '5' + INDEX_USE_ENHANCED_AST: '1' + INFO_REQUEST_CONTEXT_LINES: '5' + INFO_REQUEST_EXPLAIN_DEFAULT: '0' + INFO_REQUEST_LIMIT: '10' + INFO_REQUEST_RELATIONSHIPS: '0' + LLAMACPP_EXTRA_ARGS: '' + LLAMACPP_GPU_LAYERS: '32' + LLAMACPP_GPU_SPLIT: '' + LLAMACPP_THREADS: '6' + LLAMACPP_TIMEOUT_SEC: '300' + LLAMACPP_URL: http://host.docker.internal:8081 + LLAMACPP_USE_GPU: '1' + LLM_EXPAND_MAX: '0' + LLM_EXPAND_MODEL: phi3:mini + LLM_PROVIDER: ollama + MAX_CHANGED_SYMBOLS_RATIO: '0.6' + MAX_EMBED_CACHE: '16384' + MAX_MICRO_CHUNKS_PER_FILE: '500' + MCP_INDEXER_URL: http://localhost:8003/mcp + MEMORY_AUTODETECT: '1' + MEMORY_COLLECTION_TTL_SECS: '300' + MEMORY_MCP_TIMEOUT: '6' + MEMORY_MCP_URL: http://mcp:8000/sse + MEMORY_SSE_ENABLED: 'true' + MICRO_BUDGET_TOKENS: '1500' + MICRO_CHUNK_STRIDE: '48' + MICRO_CHUNK_TOKENS: '24' + MICRO_MERGE_LINES: '4' + MICRO_OUT_MAX_SPANS: '3' + MICRO_TOKENS_PER_LINE: '32' + MINI_VECTOR_NAME: mini + MINI_VEC_DIM: '64' + MINI_VEC_SEED: '1337' + MULTI_REPO_MODE: '1' + OLLAMA_HOST: http://host.docker.internal:11434 + POST_RERANK_SYMBOL_BOOST: '1.0' + PRF_ENABLED: '1' + QDRANT_API_KEY: '' + QDRANT_EF_SEARCH: '128' + QDRANT_TIMEOUT: '20' + QDRANT_URL: http://qdrant:6333 + QUERY_OPTIMIZER_ADAPTIVE: '1' + QUERY_OPTIMIZER_COLLECTION_SIZE: '10000' + QUERY_OPTIMIZER_COMPLEX_FACTOR: '2.0' + QUERY_OPTIMIZER_COMPLEX_THRESHOLD: '0.7' + QUERY_OPTIMIZER_DENSE_THRESHOLD: '0.2' + QUERY_OPTIMIZER_MAX_EF: '512' + QUERY_OPTIMIZER_MIN_EF: '64' + QUERY_OPTIMIZER_SEMANTIC_FACTOR: '1.0' + QUERY_OPTIMIZER_SIMPLE_FACTOR: '0.5' + QUERY_OPTIMIZER_SIMPLE_THRESHOLD: '0.3' + REFRAG_CANDIDATES: '200' + REFRAG_COMMIT_DESCRIBE: '1' + REFRAG_DECODER: '1' + REFRAG_DECODER_MODE: prompt + REFRAG_ENCODER_MODEL: BAAI/bge-base-en-v1.5 + REFRAG_GATE_FIRST: '1' + REFRAG_MODE: '1' + REFRAG_PHI_PATH: /work/models/refrag_phi_768_to_dmodel.bin + REFRAG_PSEUDO_DESCRIBE: '1' + REFRAG_RUNTIME: glm + REFRAG_SENSE: heuristic + REFRAG_SOFT_SCALE: '1.0' + REMOTE_UPLOAD_GIT_MAX_COMMITS: '500' + REPO_AUTO_FILTER: '1' + RERANKER_ENABLED: '1' + RERANKER_ONNX_PATH: /app/models/reranker.onnx + RERANKER_RETURN_M: '20' + RERANKER_TIMEOUT_MS: '3000' + RERANKER_TOKENIZER_PATH: /app/models/tokenizer.json + RERANKER_TOPN: '100' + RERANK_LEARNING: '0' + RERANKER_WEIGHTS_DIR: /tmp/rerank_weights + RERANK_EVENTS_DIR: /tmp/rerank_events + RERANK_EVENTS_ENABLED: '0' + RERANK_EVENTS_SAMPLE_RATE: '0.33' + RERANK_EVENTS_RETENTION_DAYS: '0' + RERANK_LEARNING_BATCH_SIZE: '32' + RERANK_LEARNING_POLL_INTERVAL: '30' + RERANK_LEARNING_RATE: '0.001' + RERANK_BLEND_WEIGHT: '0.6' + RERANK_EXPAND: '1' + RERANK_IN_PROCESS: '1' + RERANK_TIMEOUT_FLOOR_MS: '1000' + RERANK_WARMUP: '0' + SMART_SYMBOL_REINDEXING: '1' + STRICT_MEMORY_RESTORE: '1' + TOOL_FIND_DESCRIPTION: Search for relevant code snippets using multiple phrasings + of the query (multi-query). Prefer results where metadata.language matches the + target file and metadata.path is relevant. You may pass optional filters (language, + path_prefix, kind) which the server applies server-side. Include 'metadata.code', + 'metadata.path', and 'metadata.language' in responses. + TOOL_STORE_DESCRIPTION: Store reusable code snippets for later retrieval. The 'information' + is a clear NL description; include the actual code in 'metadata.code' and add + 'metadata.language' (e.g., python, typescript) and 'metadata.path' when known. + Use this whenever you generate or refine a code snippet. + USE_GPU_DECODER: '0' + USE_TREE_SITTER: '1' + WATCH_DEBOUNCE_SECS: '4' diff --git a/deploy/kubernetes/deploy.sh b/deploy/kubernetes/deploy.sh new file mode 100755 index 00000000..61fdf1c6 --- /dev/null +++ b/deploy/kubernetes/deploy.sh @@ -0,0 +1,334 @@ +#!/bin/bash + +# Context-Engine Kubernetes Deployment Script +# This script deploys Context-Engine services to Kubernetes + +set -e + +# Configuration +NAMESPACE="context-engine" +IMAGE_REGISTRY="context-engine" # Change to your registry if needed +IMAGE_TAG="latest" +USE_KUSTOMIZE=${USE_KUSTOMIZE:-"false"} + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if kubectl is available +check_kubectl() { + if ! command -v kubectl &> /dev/null; then + log_error "kubectl is not installed or not in PATH" + exit 1 + fi + + if ! kubectl cluster-info &> /dev/null; then + log_error "Cannot connect to Kubernetes cluster" + exit 1 + fi + + log_success "Kubernetes connection verified" +} + +# Create namespace if it doesn't exist +create_namespace() { + log_info "Creating namespace: $NAMESPACE" + kubectl apply -f namespace.yaml + log_success "Namespace created/verified" +} + +# Deploy configuration +deploy_config() { + log_info "Deploying configuration" + kubectl apply -f configmap.yaml + log_success "Configuration deployed" +} + +# Deploy core services +deploy_core() { + log_info "Deploying core services" + + # Deploy Qdrant + log_info "Deploying Qdrant database..." + kubectl apply -f qdrant.yaml + + # Wait for Qdrant to be ready + log_info "Waiting for Qdrant to be ready..." + kubectl wait --for=condition=ready pod -l component=qdrant -n "$NAMESPACE" --timeout=300s + + log_success "Core services deployed" +} + +# Deploy MCP servers +deploy_mcp_servers() { + log_info "Deploying MCP servers" + + # Deploy SSE versions + kubectl apply -f mcp-memory.yaml + kubectl apply -f mcp-indexer.yaml + + # Wait for MCP servers to be ready + log_info "Waiting for MCP servers to be ready..." + kubectl wait --for=condition=ready pod -l component=mcp-memory -n "$NAMESPACE" --timeout=300s + kubectl wait --for=condition=ready pod -l component=mcp-indexer -n "$NAMESPACE" --timeout=300s + + log_success "MCP servers deployed" +} + +# Deploy HTTP servers (optional) +deploy_http_servers() { + log_info "Deploying HTTP servers (optional)" + kubectl apply -f mcp-http.yaml + + log_info "Waiting for HTTP servers to be ready..." + kubectl wait --for=condition=ready pod -l component=mcp-memory-http -n "$NAMESPACE" --timeout=300s + kubectl wait --for=condition=ready pod -l component=mcp-indexer-http -n "$NAMESPACE" --timeout=300s + + log_success "HTTP servers deployed" +} + +# Deploy indexer services +deploy_indexer_services() { + log_info "Deploying indexer services" + kubectl apply -f indexer-services.yaml + + log_success "Indexer services deployed" +} + +# Deploy optional Llama.cpp service +deploy_llamacpp() { + if [[ "$SKIP_LLAMACPP" != "true" ]]; then + log_info "Deploying Llama.cpp service (optional)" + kubectl apply -f llamacpp.yaml + log_success "Llama.cpp service deployed" + else + log_warning "Skipping Llama.cpp deployment" + fi +} + +# Deploy Ingress (optional) +deploy_ingress() { + if [[ "$DEPLOY_INGRESS" == "true" ]]; then + log_info "Deploying Ingress" + kubectl apply -f ingress.yaml + log_success "Ingress deployed" + else + log_warning "Skipping Ingress deployment (set DEPLOY_INGRESS=true or pass --deploy-ingress to enable)" + fi +} + +# Show deployment status +show_status() { + log_info "Deployment status:" + echo + echo "Namespace: $NAMESPACE" + echo + echo "Pods:" + kubectl get pods -n $NAMESPACE -o wide + echo + echo "Services:" + kubectl get services -n $NAMESPACE + echo + echo "Persistent Volumes:" + kubectl get pvc -n $NAMESPACE || echo "No PVCs found" + echo + + log_success "Deployment complete!" + echo + log_info "Access URLs:" + echo " Qdrant: http://:30333" + echo " MCP Memory (SSE): http://:30800" + echo " MCP Memory (HTTP): http://:30804" + echo " MCP Indexer (SSE): http://:30802" + echo " MCP Indexer (HTTP): http://:30806" + if [[ "$SKIP_LLAMACPP" != "true" ]]; then + echo " Llama.cpp: http://:30808" + fi +} + +# Patch images to the chosen registry:tag and refresh jobs +set_images() { + local full="${IMAGE_REGISTRY}:${IMAGE_TAG}" + log_info "Setting images to ${full}" + kubectl set image deployment/mcp-memory mcp-memory="${full}" -n "$NAMESPACE" || true + kubectl set image deployment/mcp-indexer mcp-indexer="${full}" -n "$NAMESPACE" || true + kubectl set image deployment/mcp-memory-http mcp-memory-http="${full}" -n "$NAMESPACE" || true + kubectl set image deployment/mcp-indexer-http mcp-indexer-http="${full}" -n "$NAMESPACE" || true + kubectl set image deployment/watcher watcher="${full}" -n "$NAMESPACE" || true + # Refresh Jobs so they pick up the new image + kubectl delete job indexer-job init-payload -n "$NAMESPACE" --ignore-not-found=true + kubectl apply -f indexer-services.yaml + log_success "Images set to ${full} and jobs refreshed" +} + +# Build a temporary Kustomize overlay and apply it with images/flags respected +apply_with_kustomize() { + local base_dir + base_dir="$(pwd)" + local tmp_dir + tmp_dir="$(mktemp -d)" + log_info "Building temporary kustomize overlay at ${tmp_dir}" + + # Copy manifests to temp dir to avoid absolute path issues + cp namespace.yaml configmap.yaml qdrant.yaml mcp-memory.yaml mcp-indexer.yaml \ + mcp-http.yaml indexer-services.yaml rbac.yaml hpa.yaml networkpolicy.yaml "${tmp_dir}/" + + if [[ "${SKIP_LLAMACPP}" != "true" ]]; then + cp llamacpp.yaml "${tmp_dir}/" + fi + + if [[ "${DEPLOY_INGRESS}" == "true" ]]; then + cp ingress.yaml "${tmp_dir}/" + fi + + # Compose resources list based on flags + { + echo "apiVersion: kustomize.config.k8s.io/v1beta1" + echo "kind: Kustomization" + echo "namespace: ${NAMESPACE}" + echo "resources:" + echo " - namespace.yaml" + echo " - configmap.yaml" + echo " - qdrant.yaml" + echo " - mcp-memory.yaml" + echo " - mcp-indexer.yaml" + echo " - mcp-http.yaml" + echo " - indexer-services.yaml" + echo " - rbac.yaml" + echo " - hpa.yaml" + echo " - networkpolicy.yaml" + if [[ "${SKIP_LLAMACPP}" != "true" ]]; then + echo " - llamacpp.yaml" + fi + if [[ "${DEPLOY_INGRESS}" == "true" ]]; then + echo " - ingress.yaml" + fi + echo "images:" + echo " - name: context-engine" + echo " newName: ${IMAGE_REGISTRY}" + echo " newTag: ${IMAGE_TAG}" + } > "${tmp_dir}/kustomization.yaml" + + log_info "Applying kustomize overlay" + kubectl apply -k "${tmp_dir}" + log_success "Applied manifests via kustomize" + + # Clean up temp dir + rm -rf "${tmp_dir}" +} + +# Main deployment function +main() { + log_info "Starting Context-Engine Kubernetes deployment" + + # Check prerequisites + check_kubectl + + if [[ "$USE_KUSTOMIZE" == "true" ]]; then + # Single-shot apply via kustomize overlay (respects flags and images) + apply_with_kustomize + else + # Deploy in order via raw manifests + create_namespace + deploy_config + deploy_core + deploy_mcp_servers + deploy_http_servers + deploy_indexer_services + deploy_llamacpp + deploy_ingress + set_images + fi + + # Show status + show_status +} + +# Help function +show_help() { + echo "Context-Engine Kubernetes Deployment Script" + echo + echo "Usage: $0 [OPTIONS]" + echo + echo "Options:" + echo " -h, --help Show this help message" + echo " -r, --registry REGISTRY Docker image registry (default: context-engine)" + echo " -t, --tag TAG Docker image tag (default: latest)" + echo " --skip-llamacpp Skip Llama.cpp deployment" + echo " --deploy-ingress Deploy Ingress configuration" + echo " --use-kustomize Apply via kustomize overlay (respects --registry/--tag and flags)" + echo " --namespace NAMESPACE Kubernetes namespace (default: context-engine)" + echo + echo "Examples:" + echo " $0 # Basic deployment" + echo " $0 --skip-llamacpp # Skip Llama.cpp" + echo " $0 --deploy-ingress # Deploy with Ingress" + echo " $0 -r myregistry.com -t v1.0 # Use custom image" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + show_help + exit 0 + ;; + -r|--registry) + IMAGE_REGISTRY="$2" + shift 2 + ;; + -t|--tag) + IMAGE_TAG="$2" + shift 2 + ;; + --skip-llamacpp) + SKIP_LLAMACPP=true + shift + ;; + --deploy-ingress) + DEPLOY_INGRESS=true + shift + ;; + --use-kustomize) + USE_KUSTOMIZE=true + shift + ;; + --namespace) + NAMESPACE="$2" + shift 2 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Check if we're in the right directory +if [[ ! -f "qdrant.yaml" ]]; then + log_error "Please run this script from the deploy/kubernetes directory" + exit 1 +fi + +main diff --git a/deploy/kubernetes/hpa.yaml b/deploy/kubernetes/hpa.yaml new file mode 100644 index 00000000..90171d0b --- /dev/null +++ b/deploy/kubernetes/hpa.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: mcp-indexer-hpa + namespace: context-engine +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mcp-indexer + minReplicas: 1 + maxReplicas: 5 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + diff --git a/deploy/kubernetes/indexer-services.yaml b/deploy/kubernetes/indexer-services.yaml new file mode 100644 index 00000000..123582c6 --- /dev/null +++ b/deploy/kubernetes/indexer-services.yaml @@ -0,0 +1,353 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: watcher + namespace: context-engine + labels: + app: context-engine + component: indexer-service +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: indexer-service + template: + metadata: + labels: + app: context-engine + component: indexer-service + spec: + serviceAccountName: context-engine + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /mnt/rerank_weights /mnt/rerank_events && chmod 777 /mnt/rerank_weights /mnt/rerank_events + volumeMounts: + - name: metadata-volume + mountPath: /mnt + - name: wait-for-qdrant + image: context-engine-indexer-service + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - /app/scripts/wait-for-qdrant.sh + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + containers: + - name: watcher + image: context-engine-indexer-service + imagePullPolicy: IfNotPresent + command: + - python + - /app/scripts/watch_index.py + workingDir: /work + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + - name: COLLECTION_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: COLLECTION_NAME + - name: EMBEDDING_MODEL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: EMBEDDING_MODEL + - name: HF_HOME + value: /work/models/hf-cache + - name: XDG_CACHE_HOME + value: /work/models/hf-cache + - name: WATCH_ROOT + value: /work + - name: QDRANT_TIMEOUT + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_TIMEOUT + - name: MAX_MICRO_CHUNKS_PER_FILE + valueFrom: + configMapKeyRef: + name: context-engine-config + key: MAX_MICRO_CHUNKS_PER_FILE + - name: INDEX_UPSERT_BATCH + valueFrom: + configMapKeyRef: + name: context-engine-config + key: INDEX_UPSERT_BATCH + - name: INDEX_UPSERT_RETRIES + valueFrom: + configMapKeyRef: + name: context-engine-config + key: INDEX_UPSERT_RETRIES + - name: WATCH_DEBOUNCE_SECS + valueFrom: + configMapKeyRef: + name: context-engine-config + key: WATCH_DEBOUNCE_SECS + - name: HF_HOME + value: /work/models/hf-cache + - name: XDG_CACHE_HOME + value: /work/models/hf-cache + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + volumeMounts: + - name: work-volume + mountPath: /work + - name: metadata-volume + mountPath: /work/.codebase + - name: models-volume + mountPath: /work/models + - name: metadata-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: metadata-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: metadata-volume + persistentVolumeClaim: + claimName: code-metadata-pvc + - name: models-volume + persistentVolumeClaim: + claimName: code-models-pvc +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: indexer-job + namespace: context-engine + labels: + app: context-engine + component: indexer +spec: + template: + metadata: + labels: + app: context-engine + component: indexer + spec: + serviceAccountName: context-engine + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /mnt/rerank_weights /mnt/rerank_events && chmod 777 /mnt/rerank_weights /mnt/rerank_events + volumeMounts: + - name: metadata-volume + mountPath: /mnt + - name: wait-for-qdrant + image: context-engine-indexer-service + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - /app/scripts/wait-for-qdrant.sh + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + restartPolicy: OnFailure + containers: + - name: indexer + image: context-engine-indexer-service + imagePullPolicy: IfNotPresent + command: + - python + - /app/scripts/ingest_code.py + workingDir: /work + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + - name: COLLECTION_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: COLLECTION_NAME + - name: EMBEDDING_MODEL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: EMBEDDING_MODEL + - name: HF_HOME + value: /work/models/hf-cache + - name: XDG_CACHE_HOME + value: /work/models/hf-cache + resources: + requests: + memory: 1Gi + cpu: 500m + limits: + memory: 4Gi + cpu: 2000m + volumeMounts: + - name: work-volume + mountPath: /work + - name: metadata-volume + mountPath: /work/.codebase + - name: models-volume + mountPath: /work/models + - name: metadata-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: metadata-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: metadata-volume + persistentVolumeClaim: + claimName: code-metadata-pvc + - name: models-volume + persistentVolumeClaim: + claimName: code-models-pvc +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: init-payload + namespace: context-engine + labels: + app: context-engine + component: init +spec: + template: + metadata: + labels: + app: context-engine + component: init + spec: + serviceAccountName: context-engine + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /mnt/rerank_weights /mnt/rerank_events && chmod 777 /mnt/rerank_weights /mnt/rerank_events + volumeMounts: + - name: metadata-volume + mountPath: /mnt + - name: wait-for-qdrant + image: context-engine-indexer-service + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - /app/scripts/wait-for-qdrant.sh + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + restartPolicy: OnFailure + containers: + - name: init-payload + image: context-engine-indexer-service + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - PYTHONPATH=/app python /app/scripts/create_indexes.py && PYTHONPATH=/app python /app/scripts/warm_all_collections.py && PYTHONPATH=/app python /app/scripts/health_check.py + workingDir: /work + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + - name: COLLECTION_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: COLLECTION_NAME + - name: HF_HOME + value: /work/models/hf-cache + - name: XDG_CACHE_HOME + value: /work/models/hf-cache + - name: HF_HUB_CACHE + value: /work/models/hf-cache/huggingface + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 500m + volumeMounts: + - name: work-volume + mountPath: /work + - name: metadata-volume + mountPath: /work/.codebase + - name: models-volume + mountPath: /work/models + - name: metadata-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: metadata-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: metadata-volume + persistentVolumeClaim: + claimName: code-metadata-pvc + - name: models-volume + persistentVolumeClaim: + claimName: code-models-pvc diff --git a/deploy/kubernetes/ingress.yaml b/deploy/kubernetes/ingress.yaml new file mode 100644 index 00000000..99c1b5a6 --- /dev/null +++ b/deploy/kubernetes/ingress.yaml @@ -0,0 +1,74 @@ +--- +# Ingress for Context-Engine services (optional) +# Requires an Ingress controller (e.g., nginx-ingress, traefik) +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: context-engine-ingress + namespace: context-engine + labels: + app: context-engine + component: ingress + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/proxy-body-size: "100m" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + # nginx.ingress.kubernetes.io/enable-cors: "true" + # nginx.ingress.kubernetes.io/cors-allow-origin: "*" +spec: + ingressClassName: nginx # Adjust based on your ingress controller + rules: + - host: context-engine.example.com # Change to your domain + http: + paths: + - path: /qdrant(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: qdrant + port: + number: 6333 + - path: /mcp/memory(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: mcp-memory + port: + number: 8000 + - path: /mcp/indexer(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: mcp-indexer + port: + number: 8001 + - path: /mcp-http/memory(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: mcp-memory-http + port: + number: 8002 + - path: /mcp-http/indexer(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: mcp-indexer-http + port: + number: 8003 + - path: /llamacpp(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: llamacpp + port: + number: 8080 + + # TLS configuration (optional) + # tls: + # - hosts: + # - context-engine.example.com + # secretName: context-engine-tls diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml new file mode 100644 index 00000000..bf622c85 --- /dev/null +++ b/deploy/kubernetes/kustomization.yaml @@ -0,0 +1,85 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: context-engine + namespace: context-engine + +resources: + # Namespace and configuration + - namespace.yaml + - configmap.yaml + + # Core services + - qdrant.yaml + - code-models-pvc.yaml + + # MCP servers + - mcp-memory.yaml + - mcp-indexer.yaml + - mcp-http.yaml + - learning-reranker-worker.yaml + + # Indexer services + - indexer-services.yaml + - rbac.yaml + - networkpolicy.yaml + - hpa.yaml + + # Optional services + - llamacpp.yaml + - upload-service.yaml + - ingress.yaml + +# Common labels +labels: + - includeSelectors: true + pairs: + app.kubernetes.io/name: context-engine + app.kubernetes.io/component: kubernetes-deployment + app.kubernetes.io/managed-by: kustomize + +# Patches for production customization +patches: [] + +# ConfigMap generator (optional - for overrides) +configMapGenerator: + - name: context-engine-overrides + literals: [] + +# Secret generator (optional - for sensitive data) +secretGenerator: + - name: context-engine-secrets + literals: [] + +# Images configuration (customize for your registry) +images: + - name: context-engine + newTag: latest + # newTag: v1.0.0 + # newName: your-registry/context-engine + +# Namespace override +namespace: context-engine + +# Replicas configuration +replicas: + # Scale MCP servers for high availability + - name: mcp-memory + count: 1 # Set to 2+ for production + - name: mcp-indexer + count: 1 # Set to 2+ for production + +# Resource patches +patches: + # Example resource customization + - patch: |- + - op: replace + path: /spec/template/spec/containers/0/resources/requests/memory + value: "1Gi" + - op: replace + path: /spec/template/spec/containers/0/resources/limits/memory + value: "4Gi" + target: + kind: Deployment + name: mcp-memory diff --git a/deploy/kubernetes/learning-reranker-worker.yaml b/deploy/kubernetes/learning-reranker-worker.yaml new file mode 100644 index 00000000..c1431c22 --- /dev/null +++ b/deploy/kubernetes/learning-reranker-worker.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: learning-reranker-worker + namespace: context-engine + labels: + app: context-engine + component: learning-reranker-worker +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: learning-reranker-worker + template: + metadata: + labels: + app: context-engine + component: learning-reranker-worker + spec: + serviceAccountName: context-engine + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /mnt/rerank_weights /mnt/rerank_events && chmod 777 /mnt/rerank_weights /mnt/rerank_events + volumeMounts: + - name: metadata-volume + mountPath: /mnt + containers: + - name: learning-reranker-worker + image: context-engine-indexer + imagePullPolicy: IfNotPresent + command: + - python + - /app/scripts/learning_reranker_worker.py + - --daemon + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + volumeMounts: + - name: metadata-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: metadata-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: metadata-volume + persistentVolumeClaim: + claimName: code-metadata-pvc diff --git a/deploy/kubernetes/llamacpp.yaml b/deploy/kubernetes/llamacpp.yaml new file mode 100644 index 00000000..398771a4 --- /dev/null +++ b/deploy/kubernetes/llamacpp.yaml @@ -0,0 +1,160 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llamacpp + namespace: context-engine + labels: + app: context-engine + component: llamacpp +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: llamacpp + template: + metadata: + labels: + app: context-engine + component: llamacpp + spec: + initContainers: + - name: model-downloader + image: curlimages/curl:latest + env: + - name: LLAMACPP_MODEL_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: LLAMACPP_MODEL_URL + - name: LLAMACPP_MODEL_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: LLAMACPP_MODEL_NAME + command: + - sh + - -c + - "MODEL_PATH=\"/models/${LLAMACPP_MODEL_NAME}\"\n\nif [ -f \"$MODEL_PATH\"\ + \ ]; then\n echo \"Model already exists at $MODEL_PATH\"\n ls -lh \"$MODEL_PATH\"\ + \n exit 0\nfi\n\necho \"Downloading model from ${LLAMACPP_MODEL_URL}...\"\ + \necho \"Target: $MODEL_PATH\"\n\ncurl -L --progress-bar -o \"$MODEL_PATH.tmp\"\ + \ \"${LLAMACPP_MODEL_URL}\"\n\nif [ $? -eq 0 ]; then\n mv \"$MODEL_PATH.tmp\"\ + \ \"$MODEL_PATH\"\n echo \"Model downloaded successfully\"\n ls -lh \"\ + $MODEL_PATH\"\nelse\n echo \"Failed to download model\"\n rm -f \"$MODEL_PATH.tmp\"\ + \n exit 1\nfi\n" + volumeMounts: + - name: models + mountPath: /models + resources: + requests: + memory: 512Mi + cpu: 100m + limits: + memory: 2Gi + cpu: 500m + containers: + - name: llamacpp + image: ghcr.io/ggerganov/llama.cpp:server + imagePullPolicy: IfNotPresent + env: + - name: LLAMACPP_MODEL_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: LLAMACPP_MODEL_NAME + - name: LLAMA_ARG_MODEL + value: /models/model.gguf + - name: LLAMA_ARG_CTX_SIZE + value: '8192' + - name: LLAMA_ARG_HOST + value: 0.0.0.0 + - name: LLAMA_ARG_PORT + value: '8080' + ports: + - name: http + containerPort: 8080 + protocol: TCP + command: + - llama-server + args: + - --model + - /models/model.gguf + - --host + - 0.0.0.0 + - --port + - '8080' + - --ctx-size + - '8192' + - --no-warmup + resources: + requests: + memory: 2Gi + cpu: 1000m + limits: + memory: 8Gi + cpu: 4000m + volumeMounts: + - name: models + mountPath: /models + readOnly: false + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: models + persistentVolumeClaim: + claimName: code-models-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: llamacpp + namespace: context-engine + labels: + app: context-engine + component: llamacpp +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + targetPort: http + protocol: TCP + selector: + app: context-engine + component: llamacpp +--- +apiVersion: v1 +kind: Service +metadata: + name: llamacpp-external + namespace: context-engine + labels: + app: context-engine + component: llamacpp +spec: + type: NodePort + ports: + - name: http + port: 8080 + targetPort: http + nodePort: 30808 + protocol: TCP + selector: + app: context-engine + component: llamacpp diff --git a/deploy/kubernetes/mcp-http.yaml b/deploy/kubernetes/mcp-http.yaml new file mode 100644 index 00000000..c3c71fe2 --- /dev/null +++ b/deploy/kubernetes/mcp-http.yaml @@ -0,0 +1,405 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcp-memory-http + namespace: context-engine + labels: + app: context-engine + component: mcp-memory-http +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: mcp-memory-http + template: + metadata: + labels: + app: context-engine + component: mcp-memory-http + spec: + serviceAccountName: context-engine + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /mnt/rerank_weights /mnt/rerank_events && chmod 777 /mnt/rerank_weights /mnt/rerank_events + volumeMounts: + - name: metadata-volume + mountPath: /mnt + containers: + - name: mcp-memory-http + image: context-engine-memory + imagePullPolicy: IfNotPresent + command: + - python + - /app/scripts/mcp_memory_server.py + ports: + - name: http + containerPort: 8000 + protocol: TCP + - name: health + containerPort: 18000 + protocol: TCP + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + - name: COLLECTION_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: COLLECTION_NAME + - name: EMBEDDING_MODEL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: EMBEDDING_MODEL + - name: HF_HOME + value: /work/models/hf-cache + - name: XDG_CACHE_HOME + value: /work/models/hf-cache + - name: EMBEDDING_PROVIDER + valueFrom: + configMapKeyRef: + name: context-engine-config + key: EMBEDDING_PROVIDER + - name: TOOL_STORE_DESCRIPTION + valueFrom: + configMapKeyRef: + name: context-engine-config + key: TOOL_STORE_DESCRIPTION + - name: TOOL_FIND_DESCRIPTION + valueFrom: + configMapKeyRef: + name: context-engine-config + key: TOOL_FIND_DESCRIPTION + - name: FASTMCP_HOST + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_HOST + - name: FASTMCP_PORT + value: '8000' + - name: FASTMCP_TRANSPORT + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_HTTP_TRANSPORT + - name: FASTMCP_HEALTH_PORT + value: '18000' + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + volumeMounts: + - name: work-volume + mountPath: /work + readOnly: true + - name: metadata-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: metadata-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + livenessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 10 + periodSeconds: 5 + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: metadata-volume + persistentVolumeClaim: + claimName: code-metadata-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-memory-http + namespace: context-engine + labels: + app: context-engine + component: mcp-memory-http +spec: + type: ClusterIP + ports: + - name: http + port: 8002 + targetPort: http + protocol: TCP + - name: health + port: 18002 + targetPort: health + protocol: TCP + selector: + app: context-engine + component: mcp-memory-http +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-memory-http-external + namespace: context-engine + labels: + app: context-engine + component: mcp-memory-http +spec: + type: NodePort + ports: + - name: http + port: 8002 + targetPort: http + nodePort: 30804 + protocol: TCP + - name: health + port: 18002 + targetPort: health + nodePort: 30805 + protocol: TCP + selector: + app: context-engine + component: mcp-memory-http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcp-indexer-http + namespace: context-engine + labels: + app: context-engine + component: mcp-indexer-http +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: mcp-indexer-http + template: + metadata: + labels: + app: context-engine + component: mcp-indexer-http + spec: + serviceAccountName: context-engine + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /work/.codebase/rerank_weights /work/.codebase/rerank_events && chmod 777 /work/.codebase/rerank_weights /work/.codebase/rerank_events + volumeMounts: + - name: codebase-volume + mountPath: /work/.codebase + containers: + - name: mcp-indexer-http + image: context-engine-indexer + imagePullPolicy: IfNotPresent + command: + - python + - /app/scripts/mcp_indexer_server.py + ports: + - name: http + containerPort: 8001 + protocol: TCP + - name: health + containerPort: 18001 + protocol: TCP + env: + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + - name: COLLECTION_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: COLLECTION_NAME + - name: EMBEDDING_MODEL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: EMBEDDING_MODEL + - name: HF_HOME + value: /work/models/hf-cache + - name: XDG_CACHE_HOME + value: /work/models/hf-cache + - name: HF_HUB_CACHE + value: /work/models/hf-cache/huggingface + - name: INDEX_MICRO_CHUNKS + valueFrom: + configMapKeyRef: + name: context-engine-config + key: INDEX_MICRO_CHUNKS + - name: MAX_MICRO_CHUNKS_PER_FILE + valueFrom: + configMapKeyRef: + name: context-engine-config + key: MAX_MICRO_CHUNKS_PER_FILE + - name: REFRAG_MODE + valueFrom: + configMapKeyRef: + name: context-engine-config + key: REFRAG_MODE + - name: REFRAG_DECODER + valueFrom: + configMapKeyRef: + name: context-engine-config + key: REFRAG_DECODER + - name: LLAMACPP_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: LLAMACPP_URL + - name: MEMORY_SSE_ENABLED + valueFrom: + configMapKeyRef: + name: context-engine-config + key: MEMORY_SSE_ENABLED + - name: MEMORY_MCP_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: MEMORY_MCP_URL + - name: FASTMCP_HOST + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_HOST + - name: FASTMCP_INDEXER_PORT + value: '8001' + - name: FASTMCP_TRANSPORT + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_HTTP_TRANSPORT + - name: FASTMCP_HEALTH_PORT + value: '18001' + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + volumeMounts: + - name: work-volume + mountPath: /work + - name: codebase-volume + mountPath: /work/.codebase + - name: models-volume + mountPath: /work/models + - name: codebase-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: codebase-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + livenessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 60 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 6 + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: codebase-volume + persistentVolumeClaim: + claimName: code-metadata-pvc + - name: models-volume + persistentVolumeClaim: + claimName: code-models-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-indexer-http + namespace: context-engine + labels: + app: context-engine + component: mcp-indexer-http +spec: + type: ClusterIP + ports: + - name: http + port: 8003 + targetPort: http + protocol: TCP + - name: health + port: 18003 + targetPort: health + protocol: TCP + selector: + app: context-engine + component: mcp-indexer-http +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-indexer-http-external + namespace: context-engine + labels: + app: context-engine + component: mcp-indexer-http +spec: + type: NodePort + ports: + - name: http + port: 8003 + targetPort: http + nodePort: 30806 + protocol: TCP + - name: health + port: 18003 + targetPort: health + nodePort: 30807 + protocol: TCP + selector: + app: context-engine + component: mcp-indexer-http diff --git a/deploy/kubernetes/mcp-indexer.yaml b/deploy/kubernetes/mcp-indexer.yaml new file mode 100644 index 00000000..2fbec1a1 --- /dev/null +++ b/deploy/kubernetes/mcp-indexer.yaml @@ -0,0 +1,183 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcp-indexer + namespace: context-engine + labels: + app: context-engine + component: mcp-indexer +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: mcp-indexer + template: + metadata: + labels: + app: context-engine + component: mcp-indexer + spec: + serviceAccountName: context-engine + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /work/.codebase/rerank_weights /work/.codebase/rerank_events && chmod 777 /work/.codebase/rerank_weights /work/.codebase/rerank_events + volumeMounts: + - name: codebase-volume + mountPath: /work/.codebase + containers: + - name: mcp-indexer + image: context-engine-indexer + imagePullPolicy: IfNotPresent + command: + - python + - /app/scripts/mcp_indexer_server.py + ports: + - name: sse + containerPort: 8001 + protocol: TCP + - name: health + containerPort: 18001 + protocol: TCP + env: + - name: FASTMCP_HOST + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_HOST + - name: FASTMCP_INDEXER_PORT + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_INDEXER_PORT + - name: FASTMCP_HEALTH_PORT + value: '18001' + - name: FASTMCP_TRANSPORT + value: sse + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + - name: COLLECTION_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: COLLECTION_NAME + - name: EMBEDDING_MODEL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: EMBEDDING_MODEL + - name: HF_HOME + value: /work/models/hf-cache + - name: XDG_CACHE_HOME + value: /work/models/hf-cache + - name: HF_HUB_CACHE + value: /work/models/hf-cache/huggingface + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + volumeMounts: + - name: work-volume + mountPath: /work + - name: codebase-volume + mountPath: /work/.codebase + - name: models-volume + mountPath: /work/models + - name: codebase-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: codebase-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + livenessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 60 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 6 + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: codebase-volume + persistentVolumeClaim: + claimName: code-metadata-pvc + - name: models-volume + persistentVolumeClaim: + claimName: code-models-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-indexer + namespace: context-engine + labels: + app: context-engine + component: mcp-indexer +spec: + type: ClusterIP + ports: + - name: sse + port: 8001 + targetPort: sse + protocol: TCP + - name: health + port: 18001 + targetPort: health + protocol: TCP + selector: + app: context-engine + component: mcp-indexer +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-indexer-external + namespace: context-engine + labels: + app: context-engine + component: mcp-indexer +spec: + type: NodePort + ports: + - name: sse + port: 8001 + targetPort: sse + nodePort: 30802 + protocol: TCP + - name: health + port: 18001 + targetPort: health + nodePort: 30803 + protocol: TCP + selector: + app: context-engine + component: mcp-indexer diff --git a/deploy/kubernetes/mcp-memory.yaml b/deploy/kubernetes/mcp-memory.yaml new file mode 100644 index 00000000..165076db --- /dev/null +++ b/deploy/kubernetes/mcp-memory.yaml @@ -0,0 +1,163 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcp-memory + namespace: context-engine + labels: + app: context-engine + component: mcp-memory +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: mcp-memory + template: + metadata: + labels: + app: context-engine + component: mcp-memory + spec: + serviceAccountName: context-engine + initContainers: + - name: init-rerank-dirs + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - mkdir -p /mnt/rerank_weights /mnt/rerank_events && chmod 777 /mnt/rerank_weights /mnt/rerank_events + volumeMounts: + - name: metadata-volume + mountPath: /mnt + containers: + - name: mcp-memory + image: context-engine-memory + imagePullPolicy: IfNotPresent + command: + - python + - /app/scripts/mcp_memory_server.py + ports: + - name: sse + containerPort: 8000 + protocol: TCP + - name: health + containerPort: 18000 + protocol: TCP + env: + - name: FASTMCP_HOST + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_HOST + - name: FASTMCP_PORT + valueFrom: + configMapKeyRef: + name: context-engine-config + key: FASTMCP_PORT + - name: FASTMCP_HEALTH_PORT + value: '18000' + - name: FASTMCP_TRANSPORT + value: sse + - name: QDRANT_URL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: QDRANT_URL + - name: COLLECTION_NAME + valueFrom: + configMapKeyRef: + name: context-engine-config + key: COLLECTION_NAME + - name: EMBEDDING_MODEL + valueFrom: + configMapKeyRef: + name: context-engine-config + key: EMBEDDING_MODEL + resources: + requests: + memory: 1Gi + cpu: 500m + limits: + memory: 4Gi + cpu: '2' + volumeMounts: + - name: work-volume + mountPath: /work + readOnly: true + - name: metadata-volume + mountPath: /tmp/rerank_weights + subPath: rerank_weights + - name: metadata-volume + mountPath: /tmp/rerank_events + subPath: rerank_events + livenessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 10 + periodSeconds: 5 + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: metadata-volume + persistentVolumeClaim: + claimName: code-metadata-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-memory + namespace: context-engine + labels: + app: context-engine + component: mcp-memory +spec: + type: ClusterIP + ports: + - name: sse + port: 8000 + targetPort: sse + protocol: TCP + - name: health + port: 18000 + targetPort: health + protocol: TCP + selector: + app: context-engine + component: mcp-memory +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-memory-external + namespace: context-engine + labels: + app: context-engine + component: mcp-memory +spec: + type: NodePort + ports: + - name: sse + port: 8000 + targetPort: sse + nodePort: 30800 + protocol: TCP + - name: health + port: 18000 + targetPort: health + nodePort: 30801 + protocol: TCP + selector: + app: context-engine + component: mcp-memory diff --git a/deploy/kubernetes/namespace.yaml b/deploy/kubernetes/namespace.yaml new file mode 100644 index 00000000..537c0982 --- /dev/null +++ b/deploy/kubernetes/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: context-engine + labels: + app: context-engine + component: infrastructure diff --git a/deploy/kubernetes/networkpolicy.yaml b/deploy/kubernetes/networkpolicy.yaml new file mode 100644 index 00000000..27b031b7 --- /dev/null +++ b/deploy/kubernetes/networkpolicy.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-intra-namespace-ingress-internal + namespace: context-engine +spec: + podSelector: + matchLabels: + app: context-engine + matchExpressions: + - key: component + operator: In + values: ["watcher", "indexer", "init"] + policyTypes: + - Ingress + ingress: + - from: + - podSelector: {} + diff --git a/deploy/kubernetes/qdrant.yaml b/deploy/kubernetes/qdrant.yaml new file mode 100644 index 00000000..ba645364 --- /dev/null +++ b/deploy/kubernetes/qdrant.yaml @@ -0,0 +1,119 @@ +--- +# Qdrant StatefulSet +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: qdrant + namespace: context-engine + labels: + app: context-engine + component: qdrant +spec: + serviceName: qdrant + replicas: 1 + selector: + matchLabels: + app: context-engine + component: qdrant + template: + metadata: + labels: + app: context-engine + component: qdrant + spec: + containers: + - name: qdrant + image: qdrant/qdrant:latest + imagePullPolicy: Always + ports: + - name: http + containerPort: 6333 + protocol: TCP + - name: grpc + containerPort: 6334 + protocol: TCP + env: + - name: QDRANT__SERVICE__HTTP_PORT + value: "6333" + - name: QDRANT__SERVICE__GRPC_PORT + value: "6334" + resources: + requests: + memory: "2Gi" + cpu: "1" + limits: + memory: "8Gi" + cpu: "4" + volumeMounts: + - name: qdrant-storage + mountPath: /qdrant/storage + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + volumeClaimTemplates: + - metadata: + name: qdrant-storage + labels: + app: context-engine + component: qdrant + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: ceph-block # Ceph RWO block storage + resources: + requests: + storage: 20Gi + +--- +# Qdrant Service +apiVersion: v1 +kind: Service +metadata: + name: qdrant + namespace: context-engine + labels: + app: context-engine + component: qdrant +spec: + type: ClusterIP + ports: + - name: http + port: 6333 + targetPort: http + protocol: TCP + - name: grpc + port: 6334 + targetPort: grpc + protocol: TCP + selector: + app: context-engine + component: qdrant + +--- +# Optional: Qdrant External Service (for direct access) +apiVersion: v1 +kind: Service +metadata: + name: qdrant-external + namespace: context-engine + labels: + app: context-engine + component: qdrant +spec: + type: NodePort # Change to LoadBalancer if your cluster supports it + ports: + - name: http + port: 6333 + targetPort: http + nodePort: 30333 # Optional: specify node port + protocol: TCP + - name: grpc + port: 6334 + targetPort: grpc + nodePort: 30334 # Optional: specify node port + protocol: TCP + selector: + app: context-engine + component: qdrant diff --git a/deploy/kubernetes/rbac.yaml b/deploy/kubernetes/rbac.yaml new file mode 100644 index 00000000..0f0bc1eb --- /dev/null +++ b/deploy/kubernetes/rbac.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: context-engine + namespace: context-engine + diff --git a/deploy/kubernetes/upload-codebase-pvc.yaml b/deploy/kubernetes/upload-codebase-pvc.yaml new file mode 100644 index 00000000..e6fbc52a --- /dev/null +++ b/deploy/kubernetes/upload-codebase-pvc.yaml @@ -0,0 +1,7 @@ +## Deprecated: upload-codebase-pvc +## +## This file previously defined a separate PVC for upload-service metadata. +## The architecture now shares a single metadata volume (code-metadata-pvc) +## across upload-service and indexers, so this PVC is intentionally removed. +## +## Left as a stub to avoid accidental kubectl apply of an unused resource. \ No newline at end of file diff --git a/deploy/kubernetes/upload-pvc.yaml b/deploy/kubernetes/upload-pvc.yaml new file mode 100644 index 00000000..c708e4eb --- /dev/null +++ b/deploy/kubernetes/upload-pvc.yaml @@ -0,0 +1,47 @@ +--- +# Persistent Volume Claim for code repositories storage (CephFS RWX) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: code-repos-pvc + namespace: context-engine + labels: + app: context-engine + component: upload-service + type: storage +spec: + accessModes: + - ReadWriteMany # CephFS supports RWX for multiple pods + storageClassName: ceph-filesystem # Adjust based on your CephFS storage class + resources: + requests: + storage: 10Gi # Adjust size based on your needs + # Optional: selector for specific PV + # selector: + # matchLabels: + # app: context-engine + # component: code-repos + +--- +# Persistent Volume Claim for code metadata storage (CephFS RWX) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: code-metadata-pvc + namespace: context-engine + labels: + app: context-engine + component: upload-service + type: storage +spec: + accessModes: + - ReadWriteMany # CephFS supports RWX for multiple pods + storageClassName: ceph-filesystem # Adjust based on your CephFS storage class + resources: + requests: + storage: 5Gi # Smaller size for metadata/cache + # Optional: selector for specific PV + # selector: + # matchLabels: + # app: context-engine + # component: code-metadata diff --git a/deploy/kubernetes/upload-service.yaml b/deploy/kubernetes/upload-service.yaml new file mode 100644 index 00000000..2accf809 --- /dev/null +++ b/deploy/kubernetes/upload-service.yaml @@ -0,0 +1,109 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: upload-service + namespace: context-engine + labels: + app: context-engine + component: upload-service +spec: + replicas: 1 + selector: + matchLabels: + app: context-engine + component: upload-service + template: + metadata: + labels: + app: context-engine + component: upload-service + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + containers: + - name: upload-service + image: context-engine-upload-service + imagePullPolicy: IfNotPresent + command: + - uvicorn + args: + - scripts.upload_service:app + - --host + - 0.0.0.0 + - --port + - '8002' + - --workers + - '2' + ports: + - name: http + containerPort: 8002 + protocol: TCP + env: + - name: UPLOAD_SERVICE_HOST + value: 0.0.0.0 + - name: UPLOAD_SERVICE_PORT + value: '8002' + - name: WORK_DIR + value: /work + - name: MAX_BUNDLE_SIZE_MB + value: '100' + - name: UPLOAD_TIMEOUT_SECS + value: '300' + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + volumeMounts: + - name: work-volume + mountPath: /work + - name: codebase-volume + mountPath: /work/.codebase + # livenessProbe: + # httpGet: + # path: /health + # port: http + # initialDelaySeconds: 30 + # periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + timeoutSeconds: 15 + periodSeconds: 10 + failureThreshold: 10 + envFrom: + - configMapRef: + name: context-engine-config + volumes: + - name: work-volume + persistentVolumeClaim: + claimName: code-repos-pvc + - name: codebase-volume + persistentVolumeClaim: + claimName: code-metadata-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: upload-service + namespace: context-engine + labels: + app: context-engine + component: upload-service +spec: + type: NodePort + ports: + - name: http + port: 8002 + targetPort: http + nodePort: 30810 + protocol: TCP + selector: + app: context-engine + component: upload-service diff --git a/deploy/nginx/context-engine-rmcp.conf b/deploy/nginx/context-engine-rmcp.conf new file mode 100644 index 00000000..d6743083 --- /dev/null +++ b/deploy/nginx/context-engine-rmcp.conf @@ -0,0 +1,180 @@ +# Context-Engine RMCP + Upload reverse proxy +# Copy to /etc/nginx/conf.d/context-engine.conf (or similar) and reload nginx. + +upstream rmcp_memory_backend { + server 127.0.0.1:8002; # mcp-search-http-dev-remote (RMCP memory) + keepalive 8; +} + +upstream rmcp_indexer_backend { + server 127.0.0.1:8003; # mcp-indexer-http-dev-remote (RMCP indexer) + keepalive 8; +} + +upstream upload_service_backend { + server 127.0.0.1:8004; # upload-service-dev-remote + keepalive 8; +} + +upstream fastmcp_core_backend { + server 127.0.0.1:8000; + keepalive 8; +} + +upstream fastmcp_indexer_core_backend { + server 127.0.0.1:8001; + keepalive 8; +} + +# HTTP listener kept minimal for ACME HTTP-01 challenges / health checks. +server { + listen 80; + server_name ; + + location /.well-known/acme-challenge/ { + root /var/lib/letsencrypt; + } + + location / { + return 404; + } +} + +# Core FastMCP service exposed on https://:8800/mcp +server { + listen 8800 ssl http2; + server_name ; + + ssl_certificate /etc/letsencrypt/live//fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live//privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + client_max_body_size 200m; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + location ^~ /mcp { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_pass http://fastmcp_core_backend; + } +} + +# Core FastMCP indexer service exposed on https://:8801/mcp +server { + listen 8801 ssl http2; + server_name ; + + ssl_certificate /etc/letsencrypt/live//fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live//privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + client_max_body_size 200m; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + location ^~ /mcp { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_pass http://fastmcp_indexer_core_backend; + } +} + +# RMCP memory service exposed on https://:8802/mcp +server { + listen 8802 ssl http2; + server_name ; + + ssl_certificate /etc/letsencrypt/live//fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live//privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + client_max_body_size 200m; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + location ^~ /mcp { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_pass http://rmcp_memory_backend; + } +} + +# RMCP indexer service exposed on https://:8803/mcp +server { + listen 8803 ssl http2; + server_name ; + + ssl_certificate /etc/letsencrypt/live//fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live//privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + client_max_body_size 200m; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + location ^~ /mcp { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_pass http://rmcp_indexer_backend; + } +} + +# Upload service exposed on https://:8804/ +server { + listen 8804 ssl http2; + server_name ; + + ssl_certificate /etc/letsencrypt/live//fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live//privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + client_max_body_size 200m; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + location / { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_pass http://upload_service_backend/; + } + + location /health { + proxy_pass http://upload_service_backend/health; + } +} \ No newline at end of file diff --git a/dev-workspace/.gitkeep b/dev-workspace/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker-compose-bindmount-checkout.yml b/docker-compose-bindmount-checkout.yml new file mode 100644 index 00000000..b4e6361c --- /dev/null +++ b/docker-compose-bindmount-checkout.yml @@ -0,0 +1,307 @@ +services: + qdrant: + image: qdrant/qdrant:latest + container_name: qdrant-db + # Expose Qdrant database APIs to the host + # 6333 = HTTP API, 6334 = gRPC + ports: + - "6333:6333" + - "6334:6334" + volumes: + - qdrant_storage:/qdrant/storage + + mcp: + build: + context: . + dockerfile: Dockerfile.mcp + container_name: mcp-search + depends_on: + - qdrant + env_file: + - .env + environment: + # Ensure the server binds to all interfaces for container networking + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_PORT=${FASTMCP_PORT} + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - TOOL_STORE_DESCRIPTION=${TOOL_STORE_DESCRIPTION} + - TOOL_FIND_DESCRIPTION=${TOOL_FIND_DESCRIPTION} + - FASTMCP_HEALTH_PORT=18000 + + # SSE endpoint for IDE agents at http://localhost:8000/sse + ports: + - "18000:18000" + + - "8000:8000" + + volumes: + - ${HOST_INDEX_PATH:-.}:/work:ro + + mcp_indexer: + build: + context: . + dockerfile: Dockerfile.mcp-indexer + container_name: mcp-indexer + depends_on: + - qdrant + env_file: + - .env + environment: + - FASTMCP_HEALTH_PORT=18001 + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_INDEXER_PORT=${FASTMCP_INDEXER_PORT} + - QDRANT_URL=${QDRANT_URL} + - DEBUG_CONTEXT_ANSWER=${DEBUG_CONTEXT_ANSWER:-1} + - REFRAG_DECODER=${REFRAG_DECODER:-1} + - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080} + - USE_GPU_DECODER=${USE_GPU_DECODER:-0} + - LLAMACPP_TIMEOUT_SEC=${LLAMACPP_TIMEOUT_SEC:-180} + - CTX_REQUIRE_IDENTIFIER=${CTX_REQUIRE_IDENTIFIER:-0} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + # SSE endpoint for IDE agents at http://localhost:${FASTMCP_INDEXER_PORT:-8001}/sse + ports: + - "${FASTMCP_INDEXER_PORT:-8001}:8001" + - "18001:18001" + volumes: + - ${HOST_INDEX_PATH:-.}:/work + + + mcp_http: + build: + context: . + dockerfile: Dockerfile.mcp + container_name: mcp-search-http + depends_on: + - qdrant + env_file: + - .env + environment: + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_PORT=8000 + - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - TOOL_STORE_DESCRIPTION=${TOOL_STORE_DESCRIPTION} + - TOOL_FIND_DESCRIPTION=${TOOL_FIND_DESCRIPTION} + - FASTMCP_HEALTH_PORT=18000 + # Streamable HTTP endpoint for IDE agents at http://localhost:${FASTMCP_HTTP_PORT:-8002}/mcp/ + ports: + - "${FASTMCP_HTTP_HEALTH_PORT:-18002}:18000" + - "${FASTMCP_HTTP_PORT:-8002}:8000" + volumes: + - ${HOST_INDEX_PATH:-.}:/work:ro + + mcp_indexer_http: + build: + context: . + dockerfile: Dockerfile.mcp-indexer + container_name: mcp-indexer-http + depends_on: + - qdrant + - llamacpp + env_file: + - .env + environment: + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_INDEXER_PORT=8001 + - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} + - QDRANT_URL=${QDRANT_URL} + - FASTMCP_HEALTH_PORT=18001 + - DEBUG_CONTEXT_ANSWER=${DEBUG_CONTEXT_ANSWER:-1} + - REFRAG_DECODER=${REFRAG_DECODER:-1} + - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080} + - USE_GPU_DECODER=${USE_GPU_DECODER:-0} + - LLAMACPP_TIMEOUT_SEC=${LLAMACPP_TIMEOUT_SEC:-180} + - CTX_REQUIRE_IDENTIFIER=${CTX_REQUIRE_IDENTIFIER:-0} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + # Streamable HTTP endpoint for IDE agents at http://localhost:${FASTMCP_INDEXER_HTTP_PORT:-8003}/mcp/ + ports: + - "${FASTMCP_INDEXER_HTTP_PORT:-8003}:8001" + - "${FASTMCP_INDEXER_HTTP_HEALTH_PORT:-18003}:18001" + volumes: + - ${HOST_INDEX_PATH:-.}:/work + + llamacpp: + image: ghcr.io/ggml-org/llama.cpp:server + container_name: llama-decoder + # Optional sidecar providing a text-generation API on :8080 + # No behavior change unless REFRAG_DECODER=1 + environment: + - LLAMACPP_CTX_SIZE=${LLAMACPP_CTX_SIZE:-8192} + - LLAMACPP_HOST=0.0.0.0 + - LLAMACPP_PORT=8080 + - LLAMACPP_USE_GPU=${LLAMACPP_USE_GPU:-0} + - LLAMACPP_GPU_LAYERS=${LLAMACPP_GPU_LAYERS:-0} + - LLAMACPP_GPU_SPLIT=${LLAMACPP_GPU_SPLIT:-} + - LLAMACPP_THREADS=${LLAMACPP_THREADS:-} + - LLAMACPP_EXTRA_ARGS=${LLAMACPP_EXTRA_ARGS:-} + - LLAMACPP_NO_WARMUP=${LLAMACPP_NO_WARMUP:-1} + - LLAMACPP_TEMPERATURE=${LLAMACPP_TEMPERATURE:-} + ports: + - "8080:8080" + volumes: + - ./models:/models:ro + entrypoint: ["/bin/sh","-lc"] + command: + - | + set -e + ARGS="--model /models/model.gguf --host ${LLAMACPP_HOST:-0.0.0.0} --port ${LLAMACPP_PORT:-8080} --ctx-size ${LLAMACPP_CTX_SIZE:-8192}" + if [ "${LLAMACPP_USE_GPU:-0}" = "1" ]; then + LAYERS="${LLAMACPP_GPU_LAYERS:--1}" + else + LAYERS="${LLAMACPP_GPU_LAYERS:-0}" + fi + ARGS="$$ARGS --n-gpu-layers $${LAYERS}" + if [ -n "${LLAMACPP_GPU_SPLIT:-}" ]; then + ARGS="$$ARGS --tensor-split ${LLAMACPP_GPU_SPLIT}" + fi + if [ "${LLAMACPP_NO_WARMUP:-1}" != "0" ]; then + ARGS="$$ARGS --no-warmup" + fi + if [ -n "${LLAMACPP_THREADS:-}" ]; then + ARGS="$$ARGS --threads ${LLAMACPP_THREADS}" + fi + if [ -n "${LLAMACPP_EXTRA_ARGS:-}" ]; then + ARGS="$$ARGS ${LLAMACPP_EXTRA_ARGS}" + fi + exec /app/llama-server $$ARGS + + indexer: + build: + context: . + dockerfile: Dockerfile.indexer + depends_on: + - qdrant + env_file: + - .env + environment: + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - HUGGINGFACE_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - HF_HUB_DISABLE_XET=1 + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + working_dir: /work + volumes: + - ${HOST_INDEX_PATH:-.}:/work:ro + - ${HOST_INDEX_PATH:-.}/.codebase:/work/.codebase:rw + + entrypoint: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/ingest_code.py"] + + watcher: + build: + context: . + dockerfile: Dockerfile.indexer + depends_on: + - qdrant + env_file: + - .env + environment: + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - HUGGINGFACE_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - HF_HUB_DISABLE_XET=1 + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - WATCH_ROOT=/work + # Watcher-specific backpressure & timeouts (safer defaults) + - QDRANT_TIMEOUT=60 + - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200} + - INDEX_UPSERT_BATCH=128 + - INDEX_UPSERT_RETRIES=5 + - WATCH_DEBOUNCE_SECS=${WATCH_DEBOUNCE_SECS:-1.5} + working_dir: /work + volumes: + - ${HOST_INDEX_PATH:-.}:/work:ro + - ${HOST_INDEX_PATH:-.}/.codebase:/work/.codebase:rw + entrypoint: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/watch_index.py"] + + + upload_service: + build: + context: . + dockerfile: Dockerfile.upload-service + container_name: upload-service + depends_on: + - qdrant + env_file: + - .env + environment: + - UPLOAD_SERVICE_HOST=0.0.0.0 + - UPLOAD_SERVICE_PORT=8002 + - QDRANT_URL=${QDRANT_URL} + - WORK_DIR=/work + - CTXCE_ADMIN_COLLECTION_DELETE_ENABLED=0 + - CTXCE_COLLECTION_REGISTRY_UNDELETE_ON_DISCOVERY=${CTXCE_COLLECTION_REGISTRY_UNDELETE_ON_DISCOVERY:-0} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + ports: + - "8004:8002" + volumes: + - ${HOST_INDEX_PATH:-.}:/work:rw + - ${HOST_INDEX_PATH:-.}/.codebase:/work/.codebase:rw + user: "0:0" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8002/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + restart: unless-stopped + + init_payload: + build: + context: . + dockerfile: Dockerfile.indexer + depends_on: + - qdrant + env_file: + - .env + environment: + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME:-codebase} + working_dir: /work + volumes: + - ${HOST_INDEX_PATH:-.}:/work:ro + - ${HOST_INDEX_PATH:-.}/.codebase:/work/.codebase:rw + + entrypoint: ["python", "/app/scripts/create_indexes.py"] + +volumes: + qdrant_storage: + driver: local diff --git a/docker-compose.openlit.yml b/docker-compose.openlit.yml new file mode 100644 index 00000000..a8403efb --- /dev/null +++ b/docker-compose.openlit.yml @@ -0,0 +1,79 @@ +# OpenLit Observability Stack - POC +# Usage: docker compose -f docker-compose.yml -f docker-compose.openlit.yml up -d +# +# This adds OpenLit observability to your existing Context-Engine stack. +# Dashboard: http://localhost:3000 (login: user@openlit.io / openlituser) + +services: + # ClickHouse - storage backend for OpenLit + clickhouse: + image: clickhouse/clickhouse-server:24.4.1 + container_name: openlit-clickhouse + ports: + - "9000:9000" # Native protocol (for OTEL exporter) + - "8123:8123" # HTTP interface (for dashboard queries) + volumes: + - clickhouse_data:/var/lib/clickhouse + - ./config/clickhouse-config.xml:/etc/clickhouse-server/config.d/custom-config.xml:ro + - ./config/docker_related_config.xml:/etc/clickhouse-server/config.d/docker_related_config.xml:ro + environment: + - CLICKHOUSE_PASSWORD=OPENLIT + - CLICKHOUSE_USER=default + healthcheck: + test: ["CMD", "clickhouse-client", "--query", "SELECT 1"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - dev-remote-network + + # OpenLit Dashboard (has built-in OTEL collector) + openlit: + image: ghcr.io/openlit/openlit:latest + container_name: openlit-dashboard + ports: + - "3000:3000" # Dashboard UI + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver + environment: + - INIT_DB_HOST=clickhouse + - INIT_DB_PORT=8123 + - INIT_DB_DATABASE=openlit + - INIT_DB_USERNAME=default + - INIT_DB_PASSWORD=OPENLIT + - SQLITE_DATABASE_URL=file:/app/client/data/data.db + volumes: + - openlit_data:/app/client/data + - ./config/otel-collector-config.yaml:/etc/otel/otel-collector-config.yaml:ro + depends_on: + clickhouse: + condition: service_healthy + networks: + - dev-remote-network + + # Enable OpenLit observability for MCP services + mcp: + environment: + - OPENLIT_ENABLED=1 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 + + mcp_http: + environment: + - OPENLIT_ENABLED=1 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 + + mcp_indexer: + environment: + - OPENLIT_ENABLED=1 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 + + mcp_indexer_http: + environment: + - OPENLIT_ENABLED=1 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 + +volumes: + clickhouse_data: + driver: local + openlit_data: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..cb5403a2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,677 @@ +# Development Docker Compose for Remote Upload System Testing +# This file simulates Kubernetes environment with shared volumes that simulate the Kubernetes CephFS RWX PVC behavior. +# Repos stored in /work/ (which is project root - avoiding docker volumes) and metadata are stored in /work/.codebase/repos (project root/.codebase) +# Updated to use separate PVCs for workspace and codebase to eliminate circular dependencies + +version: '3.8' + +services: + # Qdrant vector database - same as base compose + qdrant: + image: qdrant/qdrant:latest + container_name: qdrant-db-dev-remote + ports: + - "6333:6333" + - "6334:6334" + volumes: + - qdrant_storage_dev_remote:/qdrant/storage + networks: + - dev-remote-network + + # MCP search service - same as base compose + mcp: + build: + context: . + dockerfile: Dockerfile.mcp + container_name: mcp-search-dev-remote + user: "1000:1000" + depends_on: + - qdrant + env_file: + - .env + environment: + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_PORT=${FASTMCP_PORT} + - QDRANT_URL=${QDRANT_URL} + # OpenLit observability (optional - enable via OPENLIT_ENABLED=1) + - OPENLIT_ENABLED=${OPENLIT_ENABLED:-0} + - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-http://openlit:4318} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} + - COLLECTION_NAME=${COLLECTION_NAME} + - PATH_EMIT_MODE=auto + # Use /tmp for HF caches to avoid root-owned docker volume permissions + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - HUGGINGFACE_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - HF_HUB_DISABLE_XET=1 + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - TOOL_STORE_DESCRIPTION=${TOOL_STORE_DESCRIPTION} + - TOOL_FIND_DESCRIPTION=${TOOL_FIND_DESCRIPTION} + - FASTMCP_HEALTH_PORT=18000 + # Cross-encoder reranker configuration + - RERANKER_MODEL=${RERANKER_MODEL:-} + - RERANKER_ONNX_PATH=${RERANKER_ONNX_PATH:-} + - RERANKER_TOKENIZER_PATH=${RERANKER_TOKENIZER_PATH:-} + # Learning reranker configuration + - RERANK_LEARNING=${RERANK_LEARNING:-1} + - RERANKER_WEIGHTS_DIR=/tmp/rerank_weights + - RERANK_EVENTS_DIR=/tmp/rerank_events + - RERANK_EVENTS_ENABLED=${RERANK_EVENTS_ENABLED:-1} + ports: + - "18000:18000" + - "8000:8000" + volumes: + - workspace_pvc:/work:ro + - rerank_data:/tmp/rerank_weights:rw + - rerank_events:/tmp/rerank_events:rw + networks: + - dev-remote-network + + # MCP indexer service - same as base compose + mcp_indexer: + build: + context: . + dockerfile: Dockerfile.mcp-indexer + container_name: mcp-indexer-dev-remote + user: "1000:1000" + # In K8s, scripts would be accessed directly at /app/scripts/ or via proper initContainer + # For Docker Compose dev-remote simulation, create symlink so /work/scripts/ works + # Use /tmp/huggingface for cache to avoid permission issues (universally writable) + # Set CORRECT environment variables for HuggingFace and FastEmbed + command: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/mcp_indexer_server.py"] + depends_on: + - qdrant + env_file: + - .env + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - FASTMCP_HEALTH_PORT=18001 + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_INDEXER_PORT=${FASTMCP_INDEXER_PORT} + - QDRANT_URL=${QDRANT_URL} + # OpenLit observability (optional) + - OPENLIT_ENABLED=${OPENLIT_ENABLED:-0} + - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} + - REFRAG_DECODER=${REFRAG_DECODER:-1} + - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} + - GLM_API_KEY=${GLM_API_KEY} + - GLM_API_BASE=${GLM_API_BASE:-https://api.z.ai/api/paas/v4/} + - GLM_MODEL=${GLM_MODEL:-glm-4.6} + - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080} + - COLLECTION_NAME=${COLLECTION_NAME} + - PATH_EMIT_MODE=auto + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60} + # Chunking config - use ${VAR:-} to properly inherit from .env (not host shell) + - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-} + - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-} + - MICRO_CHUNK_TOKENS=${MICRO_CHUNK_TOKENS:-} + - MICRO_CHUNK_STRIDE=${MICRO_CHUNK_STRIDE:-} + - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512} + - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5} + - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200} + # Lexical vector config - use ${VAR:-} to properly inherit from .env (not host shell) + - LEX_VECTOR_DIM=${LEX_VECTOR_DIM:-} + - LEX_MULTI_HASH=${LEX_MULTI_HASH:-} + - LEX_BIGRAMS=${LEX_BIGRAMS:-} + - LEX_BIGRAM_WEIGHT=${LEX_BIGRAM_WEIGHT:-} + - LEX_SPARSE_MODE=${LEX_SPARSE_MODE:-} + - LEX_SPARSE_NAME=${LEX_SPARSE_NAME:-} + # Pattern vectors for structural code similarity + - PATTERN_VECTORS=${PATTERN_VECTORS:-} + # Cross-encoder reranker configuration + - RERANKER_MODEL=${RERANKER_MODEL:-} + - RERANKER_ONNX_PATH=${RERANKER_ONNX_PATH:-} + - RERANKER_TOKENIZER_PATH=${RERANKER_TOKENIZER_PATH:-} + # Learning reranker configuration + - RERANK_LEARNING=${RERANK_LEARNING:-1} + - RERANKER_WEIGHTS_DIR=/tmp/rerank_weights + - RERANK_EVENTS_DIR=/tmp/rerank_events + - RERANK_EVENTS_ENABLED=${RERANK_EVENTS_ENABLED:-1} + ports: + - "${FASTMCP_INDEXER_PORT:-8001}:8001" + - "18001:18001" + volumes: + - workspace_pvc:/work:rw + - codebase_pvc:/work/.codebase:rw + - rerank_data:/tmp/rerank_weights:rw + - rerank_events:/tmp/rerank_events:rw + networks: + - dev-remote-network + + # Learning reranker worker - processes training events in background + learning_worker: + build: + context: . + dockerfile: Dockerfile.mcp-indexer + container_name: learning-worker-dev-remote + user: "1000:1000" + command: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/learning_reranker_worker.py --daemon"] + depends_on: + - qdrant + - mcp_indexer + env_file: + - .env + environment: + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME} + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + # Cross-encoder reranker (used as teacher for learning) + - RERANKER_MODEL=${RERANKER_MODEL:-} + - RERANKER_ONNX_PATH=${RERANKER_ONNX_PATH:-} + - RERANKER_TOKENIZER_PATH=${RERANKER_TOKENIZER_PATH:-} + - RERANK_EVENTS_DIR=/tmp/rerank_events + - RERANKER_WEIGHTS_DIR=/tmp/rerank_weights + - RERANK_LEARNING_BATCH_SIZE=${RERANK_LEARNING_BATCH_SIZE:-32} + - RERANK_LEARNING_POLL_INTERVAL=${RERANK_LEARNING_POLL_INTERVAL:-30} + - RERANK_LEARNING_RATE=${RERANK_LEARNING_RATE:-0.001} + - RERANK_LLM_TEACHER=${RERANK_LLM_TEACHER:-0} + - RERANK_LLM_SAMPLE_RATE=${RERANK_LLM_SAMPLE_RATE:-1.0} + - GLM_API_KEY=${GLM_API_KEY:-} + - GLM_API_BASE=${GLM_API_BASE:-} + - GLM_MODEL=${GLM_MODEL:-} + - GLM_MODEL_FAST=${GLM_MODEL_FAST:-} + - REFRAG_RUNTIME=${REFRAG_RUNTIME:-glm} + - LLAMACPP_URL=${LLAMACPP_URL:-} + - RERANK_VICREG_WEIGHT=${RERANK_VICREG_WEIGHT:-0.1} + volumes: + - workspace_pvc:/work:rw + - rerank_data:/tmp/rerank_weights:rw + - rerank_events:/tmp/rerank_events:rw + networks: + - dev-remote-network + restart: unless-stopped + + # MCP HTTP search service - same as base compose + mcp_http: + build: + context: . + dockerfile: Dockerfile.mcp + container_name: mcp-search-http-dev-remote + user: "1000:1000" + depends_on: + - qdrant + env_file: + - .env + environment: + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_PORT=8000 + - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} + - QDRANT_URL=${QDRANT_URL} + # OpenLit observability (optional - enable via OPENLIT_ENABLED=1) + - OPENLIT_ENABLED=${OPENLIT_ENABLED:-0} + - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-http://openlit:4318} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} + - COLLECTION_NAME=${COLLECTION_NAME} + - PATH_EMIT_MODE=auto + # Use /tmp for HF caches to avoid root-owned docker volume permissions + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - HUGGINGFACE_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - HF_HUB_DISABLE_XET=1 + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - TOOL_STORE_DESCRIPTION=${TOOL_STORE_DESCRIPTION} + - TOOL_FIND_DESCRIPTION=${TOOL_FIND_DESCRIPTION} + - FASTMCP_HEALTH_PORT=18000 + # Cross-encoder reranker configuration + - RERANKER_MODEL=${RERANKER_MODEL:-} + - RERANKER_ONNX_PATH=${RERANKER_ONNX_PATH:-} + - RERANKER_TOKENIZER_PATH=${RERANKER_TOKENIZER_PATH:-} + # Learning reranker configuration + - RERANK_LEARNING=${RERANK_LEARNING:-1} + - RERANKER_WEIGHTS_DIR=/tmp/rerank_weights + - RERANK_EVENTS_DIR=/tmp/rerank_events + - RERANK_EVENTS_ENABLED=${RERANK_EVENTS_ENABLED:-1} + ports: + - "${FASTMCP_HTTP_HEALTH_PORT:-18002}:18000" + - "${FASTMCP_HTTP_PORT:-8002}:8000" + volumes: + - workspace_pvc:/work:ro + - rerank_data:/tmp/rerank_weights:rw + - rerank_events:/tmp/rerank_events:rw + networks: + - dev-remote-network + + # MCP HTTP indexer service - same as base compose + mcp_indexer_http: + build: + context: . + dockerfile: Dockerfile.mcp-indexer + container_name: mcp-indexer-http-dev-remote + user: "1000:1000" + # In K8s, scripts would be accessed directly at /app/scripts/ or via proper initContainer + # For Docker Compose dev-remote simulation, create symlink so /work/scripts/ works + # Use /tmp/huggingface for cache to avoid permission issues (universally writable) + # Set CORRECT environment variables for HuggingFace and FastEmbed + command: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/mcp_indexer_server.py"] + depends_on: + - qdrant + env_file: + - .env + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - FASTMCP_HOST=${FASTMCP_HOST} + - FASTMCP_INDEXER_PORT=8001 + - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} + - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} + - REFRAG_DECODER=${REFRAG_DECODER:-1} + - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} + - GLM_API_KEY=${GLM_API_KEY} + - GLM_API_BASE=${GLM_API_BASE:-https://api.z.ai/api/paas/v4/} + - GLM_MODEL=${GLM_MODEL:-glm-4.6} + - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080} + - FASTMCP_HEALTH_PORT=18001 + - COLLECTION_NAME=${COLLECTION_NAME} + - PATH_EMIT_MODE=auto + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60} + # Chunking config - use ${VAR:-} to properly inherit from .env (not host shell) + - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-} + - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-} + - MICRO_CHUNK_TOKENS=${MICRO_CHUNK_TOKENS:-} + - MICRO_CHUNK_STRIDE=${MICRO_CHUNK_STRIDE:-} + - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512} + - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5} + - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200} + # Lexical vector config - use ${VAR:-} to properly inherit from .env (not host shell) + - LEX_VECTOR_DIM=${LEX_VECTOR_DIM:-} + - LEX_MULTI_HASH=${LEX_MULTI_HASH:-} + - LEX_BIGRAMS=${LEX_BIGRAMS:-} + - LEX_BIGRAM_WEIGHT=${LEX_BIGRAM_WEIGHT:-} + - LEX_SPARSE_MODE=${LEX_SPARSE_MODE:-} + - LEX_SPARSE_NAME=${LEX_SPARSE_NAME:-} + # Pattern vectors for structural code similarity + - PATTERN_VECTORS=${PATTERN_VECTORS:-} + # Cross-encoder reranker configuration + - RERANKER_MODEL=${RERANKER_MODEL:-} + - RERANKER_ONNX_PATH=${RERANKER_ONNX_PATH:-} + - RERANKER_TOKENIZER_PATH=${RERANKER_TOKENIZER_PATH:-} + # Learning reranker configuration + - RERANK_LEARNING=${RERANK_LEARNING:-1} + - RERANKER_WEIGHTS_DIR=/tmp/rerank_weights + - RERANK_EVENTS_DIR=/tmp/rerank_events + - RERANK_EVENTS_ENABLED=${RERANK_EVENTS_ENABLED:-1} + ports: + - "${FASTMCP_INDEXER_HTTP_PORT:-8003}:8001" + - "${FASTMCP_INDEXER_HTTP_HEALTH_PORT:-18003}:18001" + volumes: + - workspace_pvc:/work:rw + - codebase_pvc:/work/.codebase:rw + - rerank_data:/tmp/rerank_weights:rw + - rerank_events:/tmp/rerank_events:rw + networks: + - dev-remote-network + + # Llama.cpp decoder service - same as base compose + llamacpp: + image: ghcr.io/ggerganov/llama.cpp:server + container_name: llama-decoder-dev-remote + environment: + - LLAMA_ARG_MODEL=/models/model.gguf + - LLAMA_ARG_CTX_SIZE=8192 + - LLAMA_ARG_HOST=0.0.0.0 + - LLAMA_ARG_PORT=8080 + ports: + - "8080:8080" + volumes: + - ./models:/models:ro + command: ["--model", "/models/model.gguf", "--host", "0.0.0.0", "--port", "8080", "--no-warmup"] + networks: + - dev-remote-network + + # Indexer service - modified for PVC volumes + indexer: + build: + context: . + dockerfile: Dockerfile.indexer + container_name: indexer-dev-remote + user: "1000:1000" + depends_on: + - qdrant + env_file: + - .env + environment: + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME} + - GIT_HISTORY_PRUNE=${GIT_HISTORY_PRUNE:-1} + - GIT_HISTORY_DELETE_MANIFEST=${GIT_HISTORY_DELETE_MANIFEST:-0} + - GIT_HISTORY_MANIFEST_MAX_FILES=${GIT_HISTORY_MANIFEST_MAX_FILES:-0} + # Use /tmp for HF caches to avoid permission issues during model download + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - HUGGINGFACE_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - HF_HUB_DISABLE_XET=1 + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - HOST_INDEX_PATH=/work + - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60} + # Chunking config - use ${VAR:-} to properly inherit from .env (not host shell) + - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-} + - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-} + - MICRO_CHUNK_TOKENS=${MICRO_CHUNK_TOKENS:-} + - MICRO_CHUNK_STRIDE=${MICRO_CHUNK_STRIDE:-} + - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512} + - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5} + - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200} + # Lexical vector config - use ${VAR:-} to properly inherit from .env (not host shell) + - LEX_VECTOR_DIM=${LEX_VECTOR_DIM:-} + - LEX_MULTI_HASH=${LEX_MULTI_HASH:-} + - LEX_BIGRAMS=${LEX_BIGRAMS:-} + - LEX_BIGRAM_WEIGHT=${LEX_BIGRAM_WEIGHT:-} + - LEX_SPARSE_MODE=${LEX_SPARSE_MODE:-} + - LEX_SPARSE_NAME=${LEX_SPARSE_NAME:-} + # Pattern vectors for structural code similarity + - PATTERN_VECTORS=${PATTERN_VECTORS:-} + volumes: + - workspace_pvc:/work:rw + - codebase_pvc:/work/.codebase:rw + entrypoint: ["sh", "-c", "mkdir -p /tmp/logs /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && /app/scripts/wait-for-qdrant.sh && cd /app && python /app/scripts/ingest_code.py --root /work"] + restart: "no" # Run once on startup, do not restart after completion + cpus: 2.0 + networks: + - dev-remote-network + + # Watcher service - modified for PVC volumes + watcher: + build: + context: . + dockerfile: Dockerfile.indexer + container_name: watcher-dev-remote + user: "1000:1000" + depends_on: + - qdrant + env_file: + - .env + environment: + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME} + - GIT_HISTORY_PRUNE=${GIT_HISTORY_PRUNE:-1} + - GIT_HISTORY_DELETE_MANIFEST=${GIT_HISTORY_DELETE_MANIFEST:-0} + - GIT_HISTORY_MANIFEST_MAX_FILES=${GIT_HISTORY_MANIFEST_MAX_FILES:-0} + - HF_HOME=/tmp/huggingface + - HF_HUB_CACHE=/tmp/huggingface/hub + - TRANSFORMERS_CACHE=/tmp/huggingface/transformers + - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT} + - WATCH_ROOT=${WATCH_ROOT:-/work} + - HOST_INDEX_PATH=/work + - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60} + # Chunking config - use ${VAR:-} to properly inherit from .env (not host shell) + - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-} + - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-} + - MICRO_CHUNK_TOKENS=${MICRO_CHUNK_TOKENS:-} + - MICRO_CHUNK_STRIDE=${MICRO_CHUNK_STRIDE:-} + - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512} + - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5} + - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200} + - WATCH_DEBOUNCE_SECS=${WATCH_DEBOUNCE_SECS:-1.5} + - REMOTE_UPLOAD_ENABLED=${REMOTE_UPLOAD_ENABLED:-0} + # Lexical vector config - use ${VAR:-} to properly inherit from .env (not host shell) + - LEX_VECTOR_DIM=${LEX_VECTOR_DIM:-} + - LEX_MULTI_HASH=${LEX_MULTI_HASH:-} + - LEX_BIGRAMS=${LEX_BIGRAMS:-} + - LEX_BIGRAM_WEIGHT=${LEX_BIGRAM_WEIGHT:-} + - LEX_SPARSE_MODE=${LEX_SPARSE_MODE:-} + - LEX_SPARSE_NAME=${LEX_SPARSE_NAME:-} + # Pattern vectors for structural code similarity + - PATTERN_VECTORS=${PATTERN_VECTORS:-} + volumes: + - workspace_pvc:/work:rw + - codebase_pvc:/work/.codebase:rw + command: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/watch_index.py"] + cpus: 2 + networks: + - dev-remote-network + + # Init payload service - modified for PVC volumes with complete bootstrap + init_payload: + build: + context: . + dockerfile: Dockerfile.indexer + container_name: init-payload-dev-remote + user: "0:0" + depends_on: + - qdrant + env_file: + - .env + environment: + - QDRANT_URL=${QDRANT_URL} + - COLLECTION_NAME=${COLLECTION_NAME} + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} + - HF_HOME=/work/.cache/huggingface + - TRANSFORMERS_CACHE=/work/.cache/huggingface + - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface + - WORKDIR=/work + - TOKENIZER_URL=${TOKENIZER_URL:-https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json} + - TOKENIZER_PATH=${TOKENIZER_PATH:-/work/models/tokenizer.json} + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + volumes: + - workspace_pvc:/work:rw + - codebase_pvc:/work/.codebase:rw + command: [ + "sh", + "-c", + "mkdir -p /tmp/logs /work/.codebase && (chgrp -R 1000 /work/.codebase 2>/dev/null || true) && (chmod -R g+rwX /work/.codebase 2>/dev/null || true) && (find /work/.codebase -type d -exec chmod g+s {} + 2>/dev/null || true) && echo 'Starting initialization sequence...' && /app/scripts/wait-for-qdrant.sh && PYTHONPATH=/app python /app/scripts/create_indexes.py && echo 'Collections and metadata created' && python /app/scripts/warm_all_collections.py && echo 'Search caches warmed for all collections' && python /app/scripts/health_check.py && echo 'Initialization completed successfully!'" + ] + restart: "no" # Run once on startup + networks: + - dev-remote-network + + # NEW: Upload Service for Remote Upload System + upload_service: + build: + context: . + dockerfile: Dockerfile.upload-service + container_name: upload-service-dev-remote + user: "0:0" # Windows bind-mount to /work requires root to create workspace dirs + depends_on: + - qdrant + env_file: + - .env + environment: + # Upload service configuration + - UPLOAD_SERVICE_HOST=0.0.0.0 + - UPLOAD_SERVICE_PORT=8002 + - QDRANT_URL=${QDRANT_URL} + - WORKDIR=/work + - MAX_BUNDLE_SIZE_MB=100 + - UPLOAD_TIMEOUT_SECS=300 + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} + - CTXCE_ADMIN_COLLECTION_DELETE_ENABLED=${CTXCE_ADMIN_COLLECTION_DELETE_ENABLED:-0} + - CTXCE_COLLECTION_REGISTRY_UNDELETE_ON_DISCOVERY=${CTXCE_COLLECTION_REGISTRY_UNDELETE_ON_DISCOVERY:-0} + + # Indexing configuration + - COLLECTION_NAME=${COLLECTION_NAME} + - HF_HOME=/work/.cache/huggingface + - TRANSFORMERS_CACHE=/work/.cache/huggingface + - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface + - EMBEDDING_MODEL=${EMBEDDING_MODEL} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER} + - QWEN3_EMBEDDING_ENABLED=${QWEN3_EMBEDDING_ENABLED:-0} + - QWEN3_QUERY_INSTRUCTION=${QWEN3_QUERY_INSTRUCTION:-1} + - QWEN3_INSTRUCTION_TEXT=${QWEN3_INSTRUCTION_TEXT:-} + - USE_TREE_SITTER=${USE_TREE_SITTER:-} + - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-} + - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-} + + # Remote upload mode configuration + - REMOTE_UPLOAD_ENABLED=1 + - REMOTE_UPLOAD_MODE=development + - REMOTE_UPLOAD_DEBUG=1 + - REMOTE_UPLOAD_TIMEOUT=300 + - REMOTE_UPLOAD_MAX_RETRIES=5 + - MAX_BUNDLE_SIZE_MB=256 + + # Qdrant configuration - use ${VAR:-} to properly inherit from .env + - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-} + - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-} + - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-} + - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-} + # Lexical vector config - use ${VAR:-} to properly inherit from .env + - LEX_VECTOR_DIM=${LEX_VECTOR_DIM:-} + - LEX_MULTI_HASH=${LEX_MULTI_HASH:-} + - LEX_BIGRAMS=${LEX_BIGRAMS:-} + - LEX_BIGRAM_WEIGHT=${LEX_BIGRAM_WEIGHT:-} + - LEX_SPARSE_MODE=${LEX_SPARSE_MODE:-} + - LEX_SPARSE_NAME=${LEX_SPARSE_NAME:-} + ports: + - "8004:8002" # Map to different host port to avoid conflicts + - "18004:18000" # Health check port + volumes: + - workspace_pvc:/work:rw + - codebase_pvc:/work/.codebase:rw + - upload_temp:/tmp/uploads + command: [ + "sh", + "-c", + "mkdir -p /work/.codebase && (chgrp -R 1000 /work/.codebase 2>/dev/null || true) && (chmod -R g+rwX /work/.codebase 2>/dev/null || true) && (find /work/.codebase -type d -exec chmod g+s {} + 2>/dev/null || true) && exec python scripts/upload_service.py" + ] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8002/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + restart: unless-stopped + networks: + - dev-remote-network + + +# PVCs to simulate CephFS RWX behavior (production-like) +volumes: + # Main workspace volume - simulates CephFS RWX for repository storage + workspace_pvc: + driver: local + driver_opts: + type: none + o: bind + device: ${HOST_INDEX_PATH:-./dev-workspace} + + # Codebase metadata volume - simulates CephFS RWX for indexing metadata + codebase_pvc: + driver: local + driver_opts: + type: none + o: bind + device: ./.codebase + + # Temporary upload storage + upload_temp: + driver: local + + # HuggingFace cache for model downloads + huggingface_cache: + driver: local + + # Indexer cache for model downloads + indexer_cache: + driver: local + + # Qdrant storage - separate from base compose to avoid conflicts + qdrant_storage_dev_remote: + driver: local + + # Learning reranker weights storage (shared between indexer and worker) + rerank_data: + driver: local + + # Learning reranker events storage (shared between indexer and worker) + rerank_events: + driver: local + +# Custom network for service discovery +networks: + dev-remote-network: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/16 \ No newline at end of file diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..00798685 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,347 @@ +# Context Engine Architecture + +**Documentation:** [README](../README.md) · [Getting Started](GETTING_STARTED.md) · [Configuration](CONFIGURATION.md) · [IDE Clients](IDE_CLIENTS.md) · [MCP API](MCP_API.md) · [ctx CLI](CTX_CLI.md) · [Memory Guide](MEMORY_GUIDE.md) · [Architecture](ARCHITECTURE.md) · [Multi-Repo](MULTI_REPO_COLLECTIONS.md) · [Observability](OBSERVABILITY.md) · [Kubernetes](../deploy/kubernetes/README.md) · [VS Code Extension](vscode-extension.md) · [Troubleshooting](TROUBLESHOOTING.md) · [Development](DEVELOPMENT.md) + +--- + +**On this page:** +- [Overview](#overview) +- [Core Principles](#core-principles) +- [System Architecture](#system-architecture) +- [Learning Reranker System](#5-learning-reranker-system) +- [Data Flow](#data-flow) +- [ReFRAG Pipeline](#refrag-pipeline) + +--- + +## Overview + +Production-ready MCP (Model Context Protocol) retrieval stack unifying code indexing, hybrid search, and optional LLM decoding. Enables teams to ship context-aware AI agents with semantic and lexical search capabilities and dual-transport compatibility. + +## Core Principles + +- **Research-Grade Retrieval**: ReFRAG-inspired micro-chunking and span budgeting +- **Dual-Transport Support**: SSE (legacy) and HTTP RMCP (modern) protocols +- **Performance-First**: Intelligent caching, connection pooling, and async I/O +- **Production-Ready**: Comprehensive health checks, monitoring, and operational tooling + +## System Architecture + +### Component Diagram + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Client Apps │◄──►│ MCP Servers │◄──►│ Qdrant DB │ +│ (IDE, CLI, Web) │ │ (SSE + HTTP) │ │ (Vector Store) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ + ▼ + ┌─────────────────┐ + │ LLM Decoder │ + │ (llama.cpp) │ + │ (Optional) │ + └─────────────────┘ +``` + +## Core Components + +### 1. MCP Servers + +#### Memory Server (`scripts/mcp_memory_server.py`) +- **Purpose**: Knowledge base storage and retrieval +- **Transport**: SSE (port 8000) + HTTP RMCP (port 8002) +- **Key Features**: + - Structured memory storage with rich metadata + - Hybrid search (dense + lexical) + - Dual vector support for embedding and lexical hashes + - Automatic collection management + +#### Indexer Server (`scripts/mcp_indexer_server.py`) +- **Purpose**: Code search, indexing, and management +- **Transport**: SSE (port 8001) + HTTP RMCP (port 8003) +- **Key Features**: + - Hybrid code search with multiple filtering options + - ReFRAG-inspired micro-chunking (16-token windows) + - Context-aware Q&A with local LLM integration + - Workspace and collection management + - Live indexing and pruning capabilities + +### 2. Search Pipeline + +#### Hybrid Search Engine (`scripts/hybrid_search.py`) +- **Multi-Vector Architecture**: + - **Dense Vectors**: Semantic embeddings (BAAI/bge-base-en-v1.5) + - **Lexical Vectors**: BM25-style hashing (4096 dimensions) + - **Mini Vectors**: ReFRAG gating (64 dimensions, optional) + +- **Retrieval Process**: + 1. **Query Expansion**: Generate multiple query variations + 2. **Parallel Search**: Dense + lexical search with RRF fusion + 3. **Optional Reranking**: Cross-encoder neural reranking + 4. **Result Assembly**: Format with citations and metadata + +- **Advanced Features**: + - Request deduplication + - Intelligent caching (multi-policy: LRU, LFU, TTL, FIFO) + - Connection pooling to Qdrant + - Batch processing support + +#### ReFRAG Implementation +- **Micro-chunking**: Token-level windows (16 tokens, 8 stride) +- **Span Budgeting**: Global token budget management +- **Gate-First Filtering**: Mini-vector pre-filtering for efficiency + +### 3. Storage Layer + +#### Qdrant Vector Database +- **Primary Storage**: Embeddings and metadata +- **Collection Management**: Automatic creation and configuration +- **Named Vectors**: Separate storage for different embedding types +- **Performance**: HNSW indexing for fast approximate nearest neighbor search + +#### Unified Cache System (`scripts/cache_manager.py`) +- **Eviction Policies**: LRU, LFU, TTL, FIFO +- **Memory Management**: Configurable size limits and monitoring +- **Thread Safety**: Proper locking for concurrent access +- **Statistics Tracking**: Hit rates, memory usage, eviction counts + +### 4. Supporting Infrastructure + +#### Async Subprocess Manager (`scripts/async_subprocess_manager.py`) +- **Process Management**: Async subprocess execution with resource cleanup +- **Connection Pooling**: Reused HTTP connections +- **Timeout Handling**: Configurable timeouts with graceful degradation +- **Resource Tracking**: Active process monitoring and statistics + +#### Deduplication System (`scripts/deduplication.py`) +- **Request Deduplication**: Prevent redundant processing +- **Cache Integration**: Works with unified cache system +- **Performance Impact**: Significant reduction in duplicate work + +#### Semantic Expansion (`scripts/semantic_expansion.py`) +- **Query Enhancement**: LLM-assisted query variation generation +- **Local LLM Integration**: llama.cpp for offline expansion +- **Caching**: Expanded query results cached for reuse + +#### Pattern Detection (`scripts/pattern_detection/`) +- **Structural Search**: Find similar code patterns across languages via AST analysis +- **64-dim Pattern Vector**: WL graph kernel, CFG fingerprint, SimHash, spectral features +- **Auto-Detection**: Identifies retry patterns, resource cleanup, filter loops +- **Requires**: `PATTERN_VECTORS=1` to enable + +### 5. Learning Reranker System (Optional) + +The Learning Reranker is an **optional** self-improving ranking system that learns from search patterns to provide increasingly relevant results over time. It is enabled by default but can be disabled via `RERANK_LEARNING=0` and `RERANK_EVENTS_ENABLED=0` environment variables. See [Configuration](CONFIGURATION.md#learning-reranker) for all options. + +#### Architecture Overview + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Search Query │────►│ Hybrid Search │────►│ TinyScorer │ +│ │ │ (initial rank) │ │ (learned rank) │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ + │ + ┌──────────────────┐ │ + │ Event Logger │◄────────────┘ + │ (NDJSON files) │ + └────────┬─────────┘ + │ + ┌────────▼─────────┐ + │ Learning Worker │ + │ (background) │ + └────────┬─────────┘ + │ + ┌────────▼─────────┐ + │ ONNX Teacher │ + │ (cross-encoder) │ + └────────┬─────────┘ + │ + ┌────────▼─────────┐ + │ Weight Updates │ + │ (.npz files) │ + └──────────────────┘ +``` + +#### Components + +**TinyScorer** (`scripts/rerank_recursive.py`) +- 2-layer MLP neural network (~3MB per collection) +- Scores query-document pairs based on learned patterns +- Hot-reloads weights every 60 seconds from disk +- Per-collection weights (each repo learns independently) + +**Event Logger** (`scripts/rerank_events.py`) +- Logs every search to NDJSON files at `/tmp/rerank_events/` +- Records: query, candidates, initial scores, timestamps +- Hourly file rotation with configurable retention + +**Learning Worker** (`scripts/learning_reranker_worker.py`) +- Background daemon that processes logged events +- Uses ONNX cross-encoder as "teacher" model +- Trains TinyScorer via knowledge distillation +- Saves versioned weight checkpoints atomically + +#### Learning Flow + +1. **Event Capture**: Every search logs query + candidates to NDJSON +2. **Teacher Scoring**: ONNX cross-encoder scores the candidates +3. **Student Training**: TinyScorer learns to match teacher rankings +4. **Weight Update**: New weights saved atomically with versioning +5. **Hot Reload**: Serving path picks up new weights within 60s +6. **Score Integration**: `learning_score` blends with other signals + +#### Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `RERANKER_WEIGHTS_DIR` | Directory for weight files | `/tmp/rerank_weights` | +| `RERANKER_WEIGHTS_RELOAD_INTERVAL` | Hot-reload check interval (seconds) | 60 | +| `RERANKER_MAX_CHECKPOINTS` | Number of weight versions to keep | 5 | +| `RERANKER_LR_DECAY_STEPS` | Steps between learning rate decay | 1000 | +| `RERANKER_LR_DECAY_RATE` | Learning rate decay multiplier | 0.95 | +| `RERANKER_MIN_LR` | Minimum learning rate | 0.0001 | +| `RERANK_EVENTS_DIR` | Directory for event logs | `/tmp/rerank_events` | +| `RERANK_EVENTS_RETENTION_DAYS` | Days to keep event files | 7 | +| `RERANK_LEARNING_BATCH_SIZE` | Events per training batch | 32 | +| `RERANK_LEARNING_POLL_INTERVAL` | Worker poll interval (seconds) | 30 | +| `RERANK_LEARNING_RATE` | Initial learning rate | 0.001 | + +#### Observability + +Search results include learning metrics in the `why` field: +```json +{ + "score": 3.2, + "why": ["lexical:1.0", "dense_rrf:0.05", "learning:3", "score:3.2"], + "components": { + "learning_score": 3.2, + "learning_iterations": 3 + } +} +``` + +Worker logs show training progress: +``` +[codebase] Processed 5 events | v12 | lr=0.001 | avg_loss=1.8 | converged=False +``` + +#### Benefits + +- **Zero Manual Training**: Learns automatically from usage +- **Per-Collection Specialization**: Each codebase gets tuned rankings +- **Fast Inference**: TinyScorer adds <1ms to search latency +- **Continuous Improvement**: Rankings improve over time +- **Offline Capable**: Teacher runs locally, no external API calls + +#### MCP Router (`scripts/mcp_router.py`) +- **Intent Classification**: Determines which MCP tool to call based on query +- **Tool Orchestration**: Routes to search, answer, memory, or index tools +- **HTTP Execution**: Executes tools via RMCP/HTTP without extra dependencies +- **Plan Mode**: Preview tool selection without execution + +## Data Flow Architecture + +### Search Request Flow +``` +1. Client Query → MCP Server +2. Query Expansion (optional) → Multiple Query Variations +3. Parallel Execution → Dense Search + Lexical Search +4. RRF Fusion → Combined Results +5. Reranking (optional) → Enhanced Relevance +6. Result Formatting → Structured Response with Citations +7. Return to Client → MCP Protocol Response +``` + +### Indexing Flow +``` +1. File Change Detection → File System Watcher +2. Content Processing → Tokenization + Chunking +3. Embedding Generation → Model Inference +4. Vector Creation → Dense + Lexical + Mini +5. Metadata Assembly → Path, symbols, language, etc. +6. Batch Upsert → Qdrant Storage +7. Cache Updates → Local Cache Refresh +``` + +## Configuration Architecture + +### Environment-Based Configuration +- **Docker-Native**: All configuration via environment variables +- **Development Support**: Local .env file configuration +- **Production Ready**: External secret management integration + +### Key Configuration Areas +- **Service Configuration**: Ports, hosts, transport protocols +- **Model Configuration**: Embedding models, reranker settings +- **Performance Tuning**: Cache sizes, batch sizes, timeouts +- **Feature Flags**: Experimental features, debug modes + +## Transport Layer Architecture + +### Dual-Transport Design +- **SSE (Server-Sent Events)**: Legacy client compatibility +- **HTTP RMCP**: Modern JSON-RPC over HTTP +- **Simultaneous Operation**: Both protocols can run together +- **Automatic Fallback**: Graceful degradation when transport fails + +### MCP Protocol Implementation +- **FastMCP Framework**: Modern MCP server implementation +- **Tool Registry**: Automatic tool discovery and registration +- **Health Endpoints**: `/readyz` and `/tools` endpoints +- **Error Handling**: Structured error responses and logging + +## Performance Architecture + +### Caching Strategy +- **Multi-Level Caching**: Embedding cache, search cache, expansion cache +- **Intelligent Invalidation**: TTL-based and LRU eviction +- **Memory Management**: Configurable limits and monitoring +- **Performance Monitoring**: Hit rates, response times, memory usage + +### Concurrency Model +- **Async I/O**: Non-blocking operations throughout +- **Connection Pooling**: Reused connections to external services +- **Batch Processing**: Efficient bulk operations +- **Resource Management**: Proper cleanup and resource limits + +## Security Architecture + +### Isolation and Safety +- **Container-Based**: Docker isolation for all services +- **Network Segmentation**: Internal service communication +- **Input Validation**: Comprehensive parameter validation +- **Resource Limits**: Configurable timeouts and memory limits + +### Data Protection +- **No Hardcoded Secrets**: Environment-based configuration +- **API Key Management**: External secret manager integration +- **Audit Logging**: Structured logging for security events + +## Operational Architecture + +### Health Monitoring +- **Service Health**: `/readyz` endpoints for all services +- **Tool Availability**: Dynamic tool listing and status +- **Performance Metrics**: Response times, cache statistics +- **Error Tracking**: Structured error logging and alerting + +### Deployment Patterns +- **Docker Compose**: Multi-service orchestration +- **Environment Parity**: Development ↔ Production consistency +- **Graceful Shutdown**: Proper resource cleanup on termination +- **Rolling Updates**: Zero-downtime deployment support + +## Extensibility Architecture + +### Plugin System +- **MCP Tool Extension**: Easy addition of new tools +- **Transport Flexibility**: Support for future MCP transports +- **Model Pluggability**: Support for different embedding models +- **Storage Abstraction**: Potential for alternative vector stores + +### Configuration Extension +- **Environment-Driven**: Easy configuration via environment variables +- **Feature Flags**: Experimental feature toggling +- **A/B Testing**: Multiple configuration variants support + +This architecture enables Context Engine to serve as a production-ready, scalable context layer for AI applications while maintaining the flexibility to evolve with changing requirements and technologies. \ No newline at end of file diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 00000000..152348ea --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,530 @@ +# Context Engine Retrieval Benchmarks + +This document describes the retrieval evaluation methodology for Context Engine across three complementary benchmarks: **CoSQA**, **CoIR**, and **SWE-bench Retrieval**. + +## Overview + +| Benchmark | Task | Granularity | Query Type | Corpus | +|-----------|------|-------------|------------|--------| +| CoSQA | Text→Code | Function | Natural language | 20K Python functions | +| CoIR | Multi-task | Mixed | NL + Code | Multiple datasets | +| SWE-bench | Issue→Files | File/Function | GitHub issues | Real repositories | + +## 1. CoSQA (Code Search Question Answering) + +### Description + +CoSQA evaluates natural language to code retrieval using web search queries paired with Python code answers. Unlike synthetic benchmarks, queries come from real Bing search logs. + +### Dataset + +- **Source**: `mteb/cosqa` on HuggingFace (MTEB-formatted version) +- **Queries**: 500 web search questions (test split) +- **Corpus**: 20,604 Python functions (full corpus, `split="test"`) +- **Labels**: Binary relevance (1 correct answer per query) +- **Collection**: `cosqa-corpus` (auto-created on first run) + +**Note**: This is the **full mteb/cosqa corpus** (~20K snippets), not an in-repo subset. The corpus is downloaded once and cached in `~/.cache/cosqa/`. + +### Methodology + +``` +1. Index corpus into Qdrant collection +2. For each query: + a. Run hybrid search (dense + lexical + rerank) + b. Retrieve top-K results + c. Compare against ground truth +3. Compute metrics +``` + +### Metrics + +| Metric | Description | +|--------|-------------| +| MRR | Mean Reciprocal Rank - average of 1/rank of first correct result | +| NDCG@K | Normalized Discounted Cumulative Gain at K | +| Recall@K | Fraction of relevant items found in top K | +| Hit@K | Binary: was the correct answer in top K? | + +### Published Baselines + +| Model | MRR | +|-------|-----| +| BoW | 0.065 | +| BM25 (Lucene) | 0.167 | +| CodeT5+ embedding | 0.266 | +| UniXcoder | 0.319 | +| CodeBERT | 0.392 | +| text-embedding-3-large | 0.393 | + +### Usage + +```bash +# Full benchmark (comparable to paper baselines) +python -m scripts.benchmarks.cosqa.runner --limit 500 + +# Dense-only mode (pure vector search, no hybrid scoring) +python -m scripts.benchmarks.cosqa.runner --limit 500 --dense-mode + +# Enable LLM-based pseudo/tags generation during indexing +python -m scripts.benchmarks.cosqa.runner --limit 500 --enable-llm +``` + +#### CLI Flags + +| Flag | Description | +|------|-------------| +| `--limit N` | Max results per query (default: 10) | +| `--corpus-limit N` | Index only N docs (0 = full corpus) | +| `--query-limit N` | Evaluate only N queries (0 = all) | +| `--no-rerank` | Disable cross-encoder reranking | +| `--dense-mode` | Pure vector search (disables lexical, symbol boosts, heuristics) | +| `--enable-llm` | Enable LLM pseudo/tags generation during indexing | +| `--debug` | Print detailed per-query debug output | +| `--output FILE` | Output JSON report path | + +#### CoSQA ablation runs (refrag/mini, rerank, learning) + +Use the helper script to run a consistent matrix across: +- rerank vs no rerank +- ReFRAG/mini vectors vs no ReFRAG +- learning vs no learning + +```bash +# Default: 50/50 subset per run, new collection per variant +bash scripts/benchmarks/cosqa/run_ablation.sh + +# Full corpus/queries (set 0 to disable limits) +CORPUS_LIMIT=0 QUERY_LIMIT=0 RUN_TAG=full \ + bash scripts/benchmarks/cosqa/run_ablation.sh +``` + +Outputs: +- JSON reports: `bench_results/cosqa//cosqa_