diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 4c6be95..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.claude/launch.json b/.claude/launch.json new file mode 100644 index 0000000..f61d52f --- /dev/null +++ b/.claude/launch.json @@ -0,0 +1,19 @@ +{ + "version": "0.0.1", + "configurations": [ + { + "name": "backend", + "runtimeExecutable": "poetry", + "runtimeArgs": ["run", "uvicorn", "app.main:app", "--reload", "--port", "8000"], + "cwd": "backend", + "port": 8000 + }, + { + "name": "frontend", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "dev"], + "cwd": "frontend", + "port": 3000 + } + ] +} diff --git a/.gitignore b/.gitignore index d94da86..cadafdb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,8 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ +backend/lib/ +backend/lib64/ parts/ sdist/ var/ @@ -68,3 +68,11 @@ backend/specs/*.json backend/tiny_spec.json backend/large_spec.json backend/scripts/ + +# Large vendor API specs (fetch locally; see benchmarks/README.md) +benchmarks/github.json +benchmarks/stripe.json +benchmarks/discord.json + +# Claude Code local-only settings (personal permission allowlist) +.claude/settings.local.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4cbf1f6..670bbfe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,28 @@ We welcome contributions! Please follow the guidelines below to ensure a smooth 3. Make your changes and run existing tests. If you create new scripts to test behavior, please place them in `backend/scripts/` (these are ignored by git to keep history clean). 4. Do not commit temporary `.pyc` caches, `.log` files, or generated zip artifacts. Our `.gitignore` should catch most of these, but please be mindful. +## Running Tests + +Backend unit tests (no API keys or network required): +```bash +cd backend +poetry run pytest +``` + +Frontend lint, type-check, and build: +```bash +cd frontend +npm run lint +npx tsc --noEmit +npm run build +``` + +End-to-end benchmarks against a running backend (requires `GROQ_API_KEY` in `backend/.env`): +```bash +cd backend && poetry run uvicorn app.main:app --port 8000 # terminal 1 +cd benchmarks && python run_benchmark.py # terminal 2 +``` + ## Pull Requests - Ensure your commits are logically structured (e.g. separate your schema updates from your UI updates). - Reference any open issues in your PR description. diff --git a/README.md b/README.md index bcec665..824bd4b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ API Forge AI is an autonomous, agentic system built with LangGraph that ingests an OpenAPI schema and dynamically generates, tests, and self-heals Python SDK clients. A self healing and self improving agentic platform to turn API Docs to SDK - +version 2 launching on july 17th. +Update ## Overview The system orchestrates multiple LLM-powered agents to ensure that the generated SDK is structurally sound, semantically correct, and fully tested against real or mocked network conditions. diff --git a/backend/__pycache__/mock_api.cpython-312.pyc b/backend/__pycache__/mock_api.cpython-312.pyc deleted file mode 100644 index 6ec65a9..0000000 Binary files a/backend/__pycache__/mock_api.cpython-312.pyc and /dev/null differ diff --git a/backend/alembic/__pycache__/env.cpython-312.pyc b/backend/alembic/__pycache__/env.cpython-312.pyc deleted file mode 100644 index 2970cb0..0000000 Binary files a/backend/alembic/__pycache__/env.cpython-312.pyc and /dev/null differ diff --git a/backend/alembic/__pycache__/env.cpython-313.pyc b/backend/alembic/__pycache__/env.cpython-313.pyc deleted file mode 100644 index 1579df4..0000000 Binary files a/backend/alembic/__pycache__/env.cpython-313.pyc and /dev/null differ diff --git a/backend/alembic/versions/__pycache__/1be58d82328b_add_execution_log_duration_metrics.cpython-312.pyc b/backend/alembic/versions/__pycache__/1be58d82328b_add_execution_log_duration_metrics.cpython-312.pyc deleted file mode 100644 index 6fc76f3..0000000 Binary files a/backend/alembic/versions/__pycache__/1be58d82328b_add_execution_log_duration_metrics.cpython-312.pyc and /dev/null differ diff --git a/backend/alembic/versions/__pycache__/1f5cad08f1dd_initial_models.cpython-312.pyc b/backend/alembic/versions/__pycache__/1f5cad08f1dd_initial_models.cpython-312.pyc deleted file mode 100644 index 7a73033..0000000 Binary files a/backend/alembic/versions/__pycache__/1f5cad08f1dd_initial_models.cpython-312.pyc and /dev/null differ diff --git a/backend/alembic/versions/__pycache__/1f5cad08f1dd_initial_models.cpython-313.pyc b/backend/alembic/versions/__pycache__/1f5cad08f1dd_initial_models.cpython-313.pyc deleted file mode 100644 index 8e0238c..0000000 Binary files a/backend/alembic/versions/__pycache__/1f5cad08f1dd_initial_models.cpython-313.pyc and /dev/null differ diff --git a/backend/alembic/versions/__pycache__/bd431a0abf9f_add_execution_log_duration_metrics.cpython-312.pyc b/backend/alembic/versions/__pycache__/bd431a0abf9f_add_execution_log_duration_metrics.cpython-312.pyc deleted file mode 100644 index 133c4ed..0000000 Binary files a/backend/alembic/versions/__pycache__/bd431a0abf9f_add_execution_log_duration_metrics.cpython-312.pyc and /dev/null differ diff --git a/backend/apiforge.db b/backend/apiforge.db deleted file mode 100644 index 220b15a..0000000 Binary files a/backend/apiforge.db and /dev/null differ diff --git a/backend/app/agents/graph.py b/backend/app/agents/graph.py index ea07d0f..598c25c 100644 --- a/backend/app/agents/graph.py +++ b/backend/app/agents/graph.py @@ -1,6 +1,6 @@ from langgraph.graph import StateGraph, START, END from app.agents.state import AgentState -from app.agents.nodes import planner_node, coder_node, executor_node, diagnoser_node, sdk_validator_node, schema_validator_node +from app.agents.nodes import planner_node, coder_node, executor_node, diagnoser_node, sdk_validator_node, schema_validator_node, test_linter_node def route_after_diagnoser(state: AgentState) -> str: idx = state.get("current_endpoint_index", 0) @@ -28,6 +28,20 @@ def route_after_schema_validator(state: AgentState) -> str: return "coder" +def route_after_coder(state: AgentState) -> str: + # Always route to linter + return "test_linter" + +def route_after_linter(state: AgentState) -> str: + idx = state.get("current_endpoint_index", 0) + endpoints = state.get("endpoints", []) + if idx >= len(endpoints): + return "end" + current_ep = endpoints[idx] + if current_ep.get("status") == "LINTER_FAILED": + return "diagnoser" + return "executor" + def route_after_executor(state: AgentState) -> str: idx = state.get("current_endpoint_index", 0) endpoints = state.get("endpoints", []) @@ -48,6 +62,7 @@ def build_graph(checkpointer=None): workflow.add_node("sdk_validator", sdk_validator_node) workflow.add_node("schema_validator", schema_validator_node) workflow.add_node("coder", coder_node) + workflow.add_node("test_linter", test_linter_node) workflow.add_node("executor", executor_node) workflow.add_node("diagnoser", diagnoser_node) @@ -73,7 +88,17 @@ def build_graph(checkpointer=None): } ) - workflow.add_edge("coder", "executor") + workflow.add_edge("coder", "test_linter") + + workflow.add_conditional_edges( + "test_linter", + route_after_linter, + { + "diagnoser": "diagnoser", + "executor": "executor", + "end": END + } + ) workflow.add_conditional_edges( "executor", diff --git a/backend/app/agents/nodes.py b/backend/app/agents/nodes.py index 4786640..58683fa 100644 --- a/backend/app/agents/nodes.py +++ b/backend/app/agents/nodes.py @@ -17,12 +17,16 @@ class CoderOutput(BaseModel): reasoning: str = Field(description="Reasoning about how to test this specific endpoint using the generated SDK.") python_code: str = Field(description="A complete Python script using the generated SDK (`import apiforge_sdk`) to test the endpoint. Make sure to instantiate the client, call the method, and assert that the response is correct (e.g. valid Pydantic model).") +class Patch(BaseModel): + file_name: str = Field(description="Must be exactly 'client.py' or 'models.py'.") + search_string: str = Field(description="The exact string in the file to be replaced. Must match exactly.") + replace_string: str = Field(description="The string to replace it with.") + class DiagnoserOutput(BaseModel): likely_cause: str = Field(description="The likely cause of the failure based on the execution logs.") error_category: str = Field(description="Must be one of: 'sdk_error', 'schema_error', 'test_error'.") mutation_instructions: str = Field(description="Specific instructions for what was wrong.") - client_code: str = Field(description="The FULL, CORRECTED apiforge_sdk/client.py code. If no changes needed, output the original.") - models_code: str = Field(description="The FULL, CORRECTED apiforge_sdk/models.py code. If no changes needed, output the original.") + patches: list[Patch] = Field(description="List of text replacement patches to apply to the SDK files.", default_factory=list) class SchemaValidatorOutput(BaseModel): python_code: str = Field(description="A short python script using `httpx` to fetch a real payload from the API, import the correct Pydantic model from `apiforge_sdk.models`, and run `Model.model_validate()` against it.") @@ -94,6 +98,36 @@ def get_defined_symbols(code: str): return errors +def lint_test_script(code: str, require_mock_transport: bool = True) -> list[str]: + """Statically validates a generated test script: syntax, banned imports, + and (optionally) that httpx.MockTransport is used so no real network + calls are made.""" + errors = [] + try: + tree = ast.parse(code) + except SyntaxError as e: + return [f"SyntaxError: {str(e)}"] + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if 'pytest' in alias.name: + errors.append("BANNED_IMPORT: 'pytest' is not allowed. Use standard assert statements.") + elif isinstance(node, ast.ImportFrom): + if node.module and 'pytest' in node.module: + errors.append("BANNED_IMPORT: 'pytest' is not allowed. Use standard assert statements.") + + if require_mock_transport: + has_mock_transport = any( + (isinstance(node, ast.Attribute) and node.attr == 'MockTransport') or + (isinstance(node, ast.Name) and node.id == 'MockTransport') + for node in ast.walk(tree) + ) + if not has_mock_transport: + errors.append("MISSING_MOCK: You must use `httpx.MockTransport(handler)` to mock the API response. Real network calls are not allowed in this validation mode.") + + return errors + def sdk_validator_node(state: AgentState) -> dict: """Pre-execution validation stage: Validates SDK syntax and imports.""" sdk_files = state.get("sdk_files", {}) @@ -138,22 +172,50 @@ def schema_validator_node(state: AgentState) -> dict: if current_ep.get("status") == "FAILED_PERMANENTLY": return {"current_endpoint_index": idx + 1, "endpoints": endpoints} + method = current_ep.get("method", "GET").upper() + has_auth = state.get("auth_credentials") is not None + + is_safe_method = method in ["GET", "HEAD", "OPTIONS"] + + if is_safe_method and has_auth: + validation_mode = "REAL" + system_prompt = "You are a Schema Validator. Write a short Python script to fetch a real payload from the API and validate it using the generated Pydantic models. Use `httpx.get` (or appropriate method). Do NOT use the generated ApiClient, just raw httpx. Import the correct model from `apiforge_sdk.models` and run `Model.model_validate(item)`. If it's a list, validate one item. Do not use markdown blocks, just raw python string." + else: + validation_mode = "SYNTHETIC" + system_prompt = "You are a Schema Validator. Write a short Python script to synthetically generate a dummy payload based EXACTLY on the OpenAPI schema for this endpoint, and validate it using the generated Pydantic models. You MUST use `httpx.MockTransport(handler)` to mock the API response. Do NOT make a real network request. Import the correct model from `apiforge_sdk.models` and run `Model.model_validate(item)`. Write plain multi-line Python with normal newlines and indentation — never compress statements onto one line with semicolons. Do not use markdown blocks, just raw python string." + + current_ep["validation_mode"] = validation_mode + prompt = ChatPromptTemplate.from_messages([ - ("system", "You are a Schema Validator. Write a short Python script to fetch a sample payload from the API and validate it using the generated Pydantic models. Use `httpx.get` (or appropriate method). Do NOT use the generated ApiClient, just raw httpx. Import the correct model from `apiforge_sdk.models` and run `Model.model_validate(item)`. If it's a list, validate one item. Do not use markdown blocks, just raw python string."), - ("user", "Endpoint: {method} {path}\nBase URL: {base_url}\nModels:\n{models_py}") + ("system", system_prompt), + ("user", "Endpoint: {method} {path}\nBase URL: {base_url}\nModels:\n{models_py}\nPrevious Diagnostic Feedback:\n{diagnostic_feedback}\nPrevious Failed Script (fix its mistakes, do not repeat them):\n{previous_script}\nPrevious Error Output:\n{previous_stderr}") ]) - + try: sdk_files = state.get("sdk_files", {}) input_vars = { "method": current_ep.get("method"), "path": current_ep.get("path"), "base_url": state.get("base_url"), - "models_py": sdk_files.get("models.py", "") + "models_py": sdk_files.get("models.py", ""), + "diagnostic_feedback": current_ep.get("diagnostic_feedback") or "None", + "previous_script": current_ep.get("generated_code") or "None", + "previous_stderr": current_ep.get("execution_stderr") or "None" } - + result, updates = ReliabilityManager.invoke(prompt, SchemaValidatorOutput, input_vars, state) - + + # Lint before executing: schema scripts previously ran unchecked, so a + # SyntaxError or a real network call could slip straight to the executor. + lint_errors = lint_test_script(result.python_code, require_mock_transport=(validation_mode == "SYNTHETIC")) + if lint_errors: + current_ep["status"] = "SCHEMA_FAILED" + current_ep["generated_code"] = result.python_code + current_ep["execution_stdout"] = "" + current_ep["execution_stderr"] = "Schema Validation Script Linter Failed:\n" + "\n".join(lint_errors) + current_ep["agent_reasoning"] = "Linter rejected the schema validation script. Routing to Diagnoser." + return {"endpoints": endpoints, **updates} + executor = get_executor() success, stdout, stderr = executor.execute_sdk_test(sdk_files, result.python_code) @@ -238,6 +300,29 @@ def coder_node(state: AgentState) -> dict: return {"endpoints": endpoints, **updates} +def test_linter_node(state: AgentState) -> AgentState: + """Statically verifies the generated test script before execution.""" + print("--- TEST LINTER ---") + endpoints = state.get("endpoints", []) + idx = state.get("current_endpoint_index", 0) + if idx >= len(endpoints): + return state + + current_ep = endpoints[idx] + code = current_ep.get("generated_code", "") + + errors = lint_test_script(code, require_mock_transport=True) + + if errors: + current_ep["status"] = "LINTER_FAILED" + current_ep["execution_stderr"] = "Test Script Linter Failed:\n" + "\n".join(errors) + current_ep["agent_reasoning"] = "Linter rejected the test script. Routing to Diagnoser." + print(f"Linter failed: {errors}") + else: + print("Linter passed.") + + return {"endpoints": endpoints} + def executor_node(state: AgentState) -> AgentState: print("--- EXECUTOR ---") endpoints = state["endpoints"] @@ -284,7 +369,7 @@ def diagnoser_node(state: AgentState) -> dict: return {"current_endpoint_index": idx + 1, "endpoints": endpoints} prompt = ChatPromptTemplate.from_messages([ - ("system", "You are an API Debugging Expert. Analyze the execution logs. If the error is an 'SDK Consistency Validation Failed' error OR if a test fails because the SDK returns a raw httpx.Response instead of a Pydantic model (e.g. AssertionError on the return type), the SDK IS FLAWED and you MUST fix the SDK files (`client.py` or `models.py`). NEVER downgrade `model_validate()` to `User(**item)` unless `model_validate` causes an actual runtime failure. Ensure that relative imports are used inside the SDK (e.g. `from .models import User`). When configuring Pydantic models, you MUST use Pydantic V2 `model_config = ConfigDict(populate_by_name=True, extra='forbid')` as a direct class attribute, and NEVER use the Pydantic V1 `class Config:` block. Make sure to import `ConfigDict` from `pydantic`. Select the correct `error_category` ('sdk_error', 'schema_error', 'test_error'). Only modify the files relevant to the error category. For `sdk_error`, output corrected `client.py` or `models.py`. For `schema_error`, modify `models.py`. For `test_error`, provide `mutation_instructions` for the test. Output the FULL corrected Python code for `client.py` and `models.py` (do not truncate, output the entire file)."), + ("system", "You are an API Debugging Expert. Analyze the execution logs. If the error is an 'SDK Consistency Validation Failed' error OR if a test fails because the SDK returns a raw httpx.Response instead of a Pydantic model (e.g. AssertionError on the return type), the SDK IS FLAWED and you MUST fix the SDK files (`client.py` or `models.py`). NEVER downgrade `model_validate()` to `User(**item)` unless `model_validate` causes an actual runtime failure. Ensure that relative imports are used inside the SDK (e.g. `from .models import User`). When configuring Pydantic models, you MUST use Pydantic V2 `model_config = ConfigDict(populate_by_name=True, extra='forbid')` as a direct class attribute, and NEVER use the Pydantic V1 `class Config:` block. Make sure to import `ConfigDict` from `pydantic`. Select the correct `error_category` ('sdk_error', 'schema_error', 'test_error'). Only modify the files relevant to the error category. You MUST provide specific string replacement patches. The `search_string` MUST match exactly a contiguous block of text in the file."), ("user", "Endpoint: {method} {path}\nExecution Logs:\n{logs}\nTest Script:\n{code}\nSDK client.py:\n{client_py}\nSDK models.py:\n{models_py}") ]) @@ -300,18 +385,32 @@ def diagnoser_node(state: AgentState) -> dict: } result, updates = ReliabilityManager.invoke(prompt, DiagnoserOutput, input_vars, state) - + + # Guard against misdiagnosis: a SyntaxError raised by the test script + # itself can never be an SDK problem, so don't let the LLM patch the + # SDK for it. (Observed: it blamed models.py for a one-line script.) + stderr = current_ep.get("execution_stderr", "") or "" + if "SyntaxError" in stderr and "test_script.py" in stderr and "models.py" not in stderr and "client.py" not in stderr: + result.error_category = "test_error" + result.patches = [] + current_ep["agent_reasoning"] = f"Diagnosed failure: {result.likely_cause}. Category: {result.error_category}" - + feedback = result.mutation_instructions - # Apply the fixed SDK files to the state based on category + # Apply the fixed SDK files to the state based on patches if result.error_category in ["sdk_error", "schema_error"]: - sdk_files["client.py"] = result.client_code - sdk_files["models.py"] = result.models_code - feedback += f"\n\n[Diagnoser patched sdk_files in memory. Category: {result.error_category}]" + for patch in result.patches: + fname = patch.file_name + if fname in sdk_files: + if patch.search_string in sdk_files[fname]: + sdk_files[fname] = sdk_files[fname].replace(patch.search_string, patch.replace_string) + else: + feedback += f"\n\n[Warning: Patch search string not found in {fname}]" + + feedback += f"\n\n[Diagnoser applied patches in memory. Category: {result.error_category}]" print("--- DIAGNOSER PATCHED SDK ---") - print(f"client.py:\n{sdk_files['client.py'][:500]}...") + print(f"client.py:\n{sdk_files.get('client.py', '')[:500]}...") current_ep["diagnostic_feedback"] = feedback diff --git a/backend/app/agents/state.py b/backend/app/agents/state.py index 24b392f..0c6110d 100644 --- a/backend/app/agents/state.py +++ b/backend/app/agents/state.py @@ -11,8 +11,12 @@ class EndpointState(TypedDict, total=False): success: bool generated_code: str diagnostic_feedback: str + schema_validated: bool + validation_mode: str + execution_stdout: str + execution_stderr: str -class AgentState(TypedDict): +class AgentState(TypedDict, total=False): spec_content: str base_url: str endpoints: List[EndpointState] @@ -20,6 +24,9 @@ class AgentState(TypedDict): errors: List[str] global_context: Dict[str, Any] sdk_files: Dict[str, str] + # Optional API credentials; when present, safe (GET/HEAD/OPTIONS) endpoints + # are schema-validated against the real API instead of synthetic payloads. + auth_credentials: Optional[Dict[str, str]] current_key_index: int current_model_index: int provider_failovers: int diff --git a/backend/app/api/stream.py b/backend/app/api/stream.py index dc94757..e5457a7 100644 --- a/backend/app/api/stream.py +++ b/backend/app/api/stream.py @@ -10,6 +10,9 @@ from app.services.executor import get_executor import json import asyncio +import queue as thread_queue +import threading +import urllib.parse from datetime import datetime from psycopg_pool import ConnectionPool from langgraph.checkpoint.postgres import PostgresSaver @@ -24,9 +27,48 @@ pool = None if is_sqlite else ConnectionPool(conninfo=settings.SQLALCHEMY_DATABASE_URI, max_size=20, open=False) memory_saver = MemorySaver() if is_sqlite else None -# In-memory lock to prevent concurrent executions for the same job +# In-memory lock to prevent concurrent executions for the same job. +# NOTE: only safe with a single worker process; multi-worker deployments need +# a distributed lock (e.g. Redis). job_locks = {} +DEFAULT_BASE_URL = "http://127.0.0.1:8001" + +def normalize_base_url(raw_url: str) -> str: + """Resolves relative server URLs from the spec against the default host.""" + if not raw_url.startswith(("http://", "https://")): + return urllib.parse.urljoin(DEFAULT_BASE_URL, raw_url) + return raw_url + +def iter_graph_in_thread(stream_generator): + """Iterates a synchronous LangGraph stream in a worker thread so multi-second + LLM/executor calls don't block the event loop (which would stall every other + request on the server). Yields items back on the loop via a thread queue.""" + q = thread_queue.Queue(maxsize=4) + _SENTINEL = object() + + def producer(): + try: + for item in stream_generator: + q.put(("item", item)) + except BaseException as e: + q.put(("error", e)) + finally: + q.put(("done", _SENTINEL)) + + threading.Thread(target=producer, daemon=True).start() + + async def consume(): + while True: + kind, payload = await asyncio.to_thread(q.get) + if kind == "error": + raise payload + if kind == "done": + return + yield payload + + return consume() + async def real_event_generator(job_id: str, db: Session): if job_id not in job_locks: job_locks[job_id] = asyncio.Lock() @@ -43,7 +85,7 @@ async def real_event_generator(job_id: str, db: Session): return if job.status in ["SUCCESS", "FAILED"]: - yield f"data: {json.dumps({'status': 'complete', 'message': 'Job execution finished'})}\n\n" + yield f"data: {json.dumps({'status': 'complete', 'success': job.status == 'SUCCESS', 'message': 'Job execution finished'})}\n\n" return job.status = "RUNNING" @@ -53,13 +95,9 @@ async def real_event_generator(job_id: str, db: Session): parsed_json = parse_spec_content(job.spec_content) endpoints_data = extract_endpoints(parsed_json) - raw_url = parsed_json.get("servers", [{"url": "http://127.0.0.1:8001"}])[0].get("url", "http://127.0.0.1:8001") - if not raw_url.startswith(("http://", "https://")): - import urllib.parse - base_url = urllib.parse.urljoin("http://127.0.0.1:8001", raw_url) - else: - base_url = raw_url - + raw_url = parsed_json.get("servers", [{"url": DEFAULT_BASE_URL}])[0].get("url", DEFAULT_BASE_URL) + base_url = normalize_base_url(raw_url) + initial_state = AgentState({ "spec_content": job.spec_content, "base_url": base_url, @@ -112,7 +150,7 @@ async def real_event_generator(job_id: str, db: Session): node_start_time = datetime.utcnow() try: - for s in stream_generator: + async for s in iter_graph_in_thread(stream_generator): node_end_time = datetime.utcnow() duration_ms = int((node_end_time - node_start_time).total_seconds() * 1000) @@ -157,15 +195,41 @@ async def real_event_generator(job_id: str, db: Session): job.completed_at = datetime.utcnow() db.commit() yield f"data: {json.dumps({'status': 'error', 'message': 'Graph recursion limit exceeded'})}\n\n" - yield f"data: {json.dumps({'status': 'complete', 'message': 'Job execution failed due to recursion limit'})}\n\n" + yield f"data: {json.dumps({'status': 'complete', 'success': False, 'message': 'Job execution failed due to recursion limit'})}\n\n" + return + + # Check for early termination or planner errors + graph_errors = full_state.get("errors", []) + sdk_files = full_state.get("sdk_files", {}) + + if graph_errors: + job.status = "FAILED" + job.completed_at = datetime.utcnow() + db.commit() + # Surface the first error to the client + error_msg = graph_errors[0] + yield f"data: {json.dumps({'status': 'error', 'message': error_msg})}\n\n" + yield f"data: {json.dumps({'status': 'complete', 'success': False, 'message': f'Job execution failed: {error_msg}'})}\n\n" return - # Final SDK Quality Gate + if not sdk_files: + job.status = "FAILED" + job.completed_at = datetime.utcnow() + db.commit() + yield f"data: {json.dumps({'status': 'error', 'message': 'SDK files were not generated'})}\n\n" + yield f"data: {json.dumps({'status': 'complete', 'success': False, 'message': 'Job execution failed: SDK generation aborted'})}\n\n" + return + + # Final SDK Quality Gate. + # Runs entirely against httpx.MockTransport — it must never depend on + # the target API being reachable. (Previously it invoked a zero-arg + # method against the real base_url; petstore jobs failed with + # ConnectionRefused even after every endpoint test passed.) test_script = """import httpx import inspect import sys from apiforge_sdk.client import ApiClient -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError client = ApiClient() methods = [m for m in dir(client) if not m.startswith('_') and callable(getattr(client, m))] @@ -188,9 +252,35 @@ async def real_event_generator(job_id: str, db: Session): print("No zero-argument methods found. Skipping runtime invocation check. PASS.") sys.exit(0) +# Replace any internal httpx.Client with a mocked one so no real network +# request is made. The gate only verifies methods return Pydantic models +# rather than raw httpx.Response objects. +def mock_handler(request): + return httpx.Response(200, json={}) + +injected = False +for attr_name, attr_value in list(vars(client).items()): + if isinstance(attr_value, httpx.Client): + setattr(client, attr_name, httpx.Client( + transport=httpx.MockTransport(mock_handler), + base_url=attr_value.base_url, + timeout=attr_value.timeout, + )) + injected = True + +if not injected: + print("Could not locate an internal httpx.Client to mock. Skipping runtime invocation check. PASS.") + sys.exit(0) + method_name = zero_arg_methods[0] method = getattr(client, method_name) -result = method() +try: + result = method() +except ValidationError: + # The mocked empty payload failed model validation — which proves the + # method does run Pydantic validation instead of returning raw responses. + print(f"Method {method_name} validates responses with Pydantic. PASS.") + sys.exit(0) if isinstance(result, httpx.Response): raise Exception(f"Method {method_name} returned raw httpx.Response instead of a Pydantic model") @@ -199,7 +289,10 @@ async def real_event_generator(job_id: str, db: Session): item = result[0] if not isinstance(item, BaseModel): raise Exception(f"Method {method_name} returned a list of {type(item)}, expected BaseModel") -elif not isinstance(result, list) and result is not None: +elif isinstance(result, dict) or result is None or isinstance(result, (str, int, float, bool)): + # Plain payloads (e.g. Dict[str, int] responses) and None are acceptable + pass +elif not isinstance(result, list): if not isinstance(result, BaseModel): raise Exception(f"Method {method_name} returned {type(result)}, expected BaseModel") @@ -218,7 +311,7 @@ async def real_event_generator(job_id: str, db: Session): job.completed_at = datetime.utcnow() db.commit() msg = "Job execution failed. One or more endpoints failed." if any_failed else f"Job execution failed. Integrity error: {integrity_stderr}" - yield f"data: {json.dumps({'status': 'complete', 'message': msg})}\n\n" + yield f"data: {json.dumps({'status': 'complete', 'success': False, 'message': msg})}\n\n" return # Execution summary @@ -246,10 +339,16 @@ async def real_event_generator(job_id: str, db: Session): job.completed_at = datetime.utcnow() db.commit() - yield f"data: {json.dumps({'status': 'complete', 'message': 'Job execution finished'})}\n\n" + yield f"data: {json.dumps({'status': 'complete', 'success': True, 'message': 'Job execution finished'})}\n\n" finally: pass # lock is released automatically by async with + # Drop the lock entry once the job is finished so job_locks doesn't grow + # unboundedly over the server's lifetime. + lock = job_locks.get(job_id) + if lock is not None and not lock.locked(): + job_locks.pop(job_id, None) + @router.get("/jobs/{job_id}/stream") async def stream_job_progress(job_id: str, db: Session = Depends(get_db)): return StreamingResponse(real_event_generator(job_id, db), media_type="text/event-stream") diff --git a/backend/app/api/upload.py b/backend/app/api/upload.py index c6a3e68..70bb953 100644 --- a/backend/app/api/upload.py +++ b/backend/app/api/upload.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Session from app.core.db import get_db from app.models.domain import Project, IntegrationJob -from app.services.openapi_parser import parse_spec_content, extract_endpoints +from app.services.openapi_parser import parse_spec_content, extract_endpoints, validate_openapi_spec from slowapi import Limiter from slowapi.util import get_remote_address @@ -27,8 +27,17 @@ async def upload_spec(request: Request, file: UploadFile = File(...), project_na except Exception as e: raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}") + spec_errors = validate_openapi_spec(parsed_json) + if spec_errors: + raise HTTPException(status_code=422, detail=f"Not a valid OpenAPI spec: {' '.join(spec_errors)}") + endpoints_data = extract_endpoints(parsed_json) - + if not endpoints_data: + raise HTTPException( + status_code=422, + detail="Spec has 'paths' but no GET/POST/PUT/DELETE/PATCH/OPTIONS/HEAD operations were found under any of them.", + ) + # Check if project exists or create new project = db.query(Project).filter(Project.name == project_name).first() if not project: diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8cf32d9..9f32176 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,16 +1,14 @@ -from pydantic_settings import BaseSettings, SettingsConfigDict +import os +from typing import Optional from pydantic_settings import BaseSettings, SettingsConfigDict -from typing import Optional - class Settings(BaseSettings): PROJECT_NAME: str = "APIForge AI" SQLALCHEMY_DATABASE_URI: str = "sqlite:///./apiforge.db" - + def __init__(self, **kwargs): super().__init__(**kwargs) - import os db_url = os.getenv("DATABASE_URL") if db_url: if db_url.startswith("postgres://"): diff --git a/backend/app/main.py b/backend/app/main.py index 264db37..0c5330a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,5 +1,8 @@ -from fastapi import FastAPI +import logging + +from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from app.core.config import settings from app.api import upload, download, stream, dashboard @@ -10,6 +13,8 @@ import os +logger = logging.getLogger("uvicorn.error") + app = FastAPI( title=settings.PROJECT_NAME, description="Agentic API Integration Platform", @@ -20,6 +25,30 @@ app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) +# NOTE on ordering: this middleware must be registered BEFORE CORSMiddleware +# below, not after. Starlette's Starlette.add_middleware() does +# self.user_middleware.insert(0, ...) — each new registration is prepended, +# so the middleware stack ends up wrapped in the REVERSE of call order: the +# LAST add_middleware() call becomes the OUTERMOST layer. Registering this +# first means CORSMiddleware ends up outside it, so a response built here +# still passes through CORSMiddleware's header injection. Get the order +# backwards and any response built here bypasses CORS entirely — the +# browser then reports a same-looking-but-opaque "Failed to fetch" instead +# of the real error. (Also can't be @app.exception_handler(Exception): that +# routes through ServerErrorMiddleware, which sits outside ALL +# add_middleware() layers regardless of order.) Both failure modes are +# covered by tests/test_error_handling.py. +@app.middleware("http") +async def unhandled_exception_middleware(request: Request, call_next): + try: + return await call_next(request) + except Exception: + logger.exception("Unhandled exception in %s %s", request.method, request.url.path) + return JSONResponse( + status_code=500, + content={"detail": "Internal server error. Please try again or check the server logs."}, + ) + app.add_middleware( CORSMiddleware, allow_origins=[ diff --git a/backend/app/services/executor/e2b.py b/backend/app/services/executor/e2b.py index 4204921..3988c7e 100644 --- a/backend/app/services/executor/e2b.py +++ b/backend/app/services/executor/e2b.py @@ -1,17 +1,61 @@ from typing import Tuple from .base import BaseExecutor +from app.core.config import settings + +SANDBOX_TIMEOUT_SECONDS = 60 +COMMAND_TIMEOUT_SECONDS = 30 class E2BExecutor(BaseExecutor): + """Runs generated code in an isolated E2B cloud sandbox instead of the host. + + Requires E2B_API_KEY in the environment/.env. Each execution uses a fresh + sandbox so generated code can never touch the host machine or leak state + between runs. + """ + + def _create_sandbox(self): + from e2b_code_interpreter import Sandbox + if not settings.E2B_API_KEY: + raise RuntimeError("E2B_API_KEY is not configured but USE_E2B_EXECUTOR is enabled.") + return Sandbox.create(api_key=settings.E2B_API_KEY, timeout=SANDBOX_TIMEOUT_SECONDS) + def execute_python_code(self, code: str) -> Tuple[bool, str, str]: - """ - Stub for E2B execution. - Will implement when transitioning to E2B sandbox. - """ - # Placeholder for E2B Sandbox execution - return False, "", "E2B Executor not yet fully implemented." + try: + sandbox = self._create_sandbox() + except Exception as e: + return False, "", f"Failed to create E2B sandbox: {e}" + try: + sandbox.files.write("/home/user/script.py", code) + result = sandbox.commands.run( + "python /home/user/script.py", timeout=COMMAND_TIMEOUT_SECONDS + ) + return result.exit_code == 0, result.stdout, result.stderr + except Exception as e: + # e2b raises CommandExitException on non-zero exit; it carries the output + return False, getattr(e, "stdout", ""), getattr(e, "stderr", None) or str(e) + finally: + try: + sandbox.kill() + except Exception: + pass def execute_sdk_test(self, sdk_files: dict, test_script: str) -> Tuple[bool, str, str]: - """ - Stub for E2B execution. - """ - return False, "", "E2B Executor not yet fully implemented." + try: + sandbox = self._create_sandbox() + except Exception as e: + return False, "", f"Failed to create E2B sandbox: {e}" + try: + for filename, content in sdk_files.items(): + sandbox.files.write(f"/home/user/apiforge_sdk/{filename}", content) + sandbox.files.write("/home/user/test_script.py", test_script) + result = sandbox.commands.run( + "cd /home/user && python test_script.py", timeout=COMMAND_TIMEOUT_SECONDS + ) + return result.exit_code == 0, result.stdout, result.stderr + except Exception as e: + return False, getattr(e, "stdout", ""), getattr(e, "stderr", None) or str(e) + finally: + try: + sandbox.kill() + except Exception: + pass diff --git a/backend/app/services/openapi_parser.py b/backend/app/services/openapi_parser.py index 33f940b..45b1c44 100644 --- a/backend/app/services/openapi_parser.py +++ b/backend/app/services/openapi_parser.py @@ -8,13 +8,32 @@ def parse_spec_content(content: str) -> Dict[str, Any]: except json.JSONDecodeError: return yaml.safe_load(content) +def validate_openapi_spec(spec_json: Any) -> List[str]: + """Returns a list of human-readable problems; empty list means the document + looks like a usable OpenAPI/Swagger spec.""" + errors = [] + if not isinstance(spec_json, dict): + return ["Document is not a mapping/object — not an OpenAPI spec."] + if "openapi" not in spec_json and "swagger" not in spec_json: + errors.append("Missing 'openapi' (3.x) or 'swagger' (2.0) version field.") + paths = spec_json.get("paths") + if not isinstance(paths, dict) or not paths: + errors.append("Spec has no 'paths' — nothing to generate an SDK from.") + return errors + def extract_endpoints(spec_json: Dict[str, Any]) -> List[Dict[str, Any]]: endpoints = [] paths = spec_json.get("paths", {}) + if not isinstance(paths, dict): + return endpoints for path, path_item in paths.items(): + if not isinstance(path_item, dict): + continue for method, operation in path_item.items(): if method.lower() not in ["get", "post", "put", "delete", "patch", "options", "head"]: continue + if not isinstance(operation, dict): + continue endpoints.append({ "path": path, "method": method.upper(), @@ -22,6 +41,3 @@ def extract_endpoints(spec_json: Dict[str, Any]) -> List[Dict[str, Any]]: "summary": operation.get("summary"), }) return endpoints - -def determine_dependencies(endpoints: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - return endpoints diff --git a/backend/app/services/reliability.py b/backend/app/services/reliability.py index 0113ff2..158ec8a 100644 --- a/backend/app/services/reliability.py +++ b/backend/app/services/reliability.py @@ -115,6 +115,20 @@ def invoke(cls, prompt, output_schema, input_vars, state: AgentState): print(f"[MODEL FALLBACK]\nOld model: {old_model}\nNew model: None (Exhausted)") continue # Retry loop + elif "parse" in error_str or "validation" in error_str or "outputparserexception" in error_str or "tool_use_failed" in error_str or "failed to call a function" in error_str: + print(f"[PARSING ERROR] Retrying structured output. Error: {e}") + attempts += 1 + attempts_for_current_model += 1 + if attempts_for_current_model >= 3: + old_model = current_model + model_idx += 1 + model_failovers += 1 + attempts_for_current_model = 0 + key_idx = 0 + if model_idx < len(cls.MODELS): + new_model = cls.MODELS[model_idx] + print(f"[MODEL FALLBACK due to parsing]\nOld model: {old_model}\nNew model: {new_model}") + continue # Retry loop else: # Non-rate-limit exception raise e diff --git a/backend/app/services/sdk_builder.py b/backend/app/services/sdk_builder.py index cc0371f..8afcbe0 100644 --- a/backend/app/services/sdk_builder.py +++ b/backend/app/services/sdk_builder.py @@ -1,15 +1,5 @@ import io import zipfile -from typing import List, Dict -from pydantic import BaseModel, Field -from app.services.llm_factory import get_llm -from langchain_core.prompts import ChatPromptTemplate -from app.core.config import settings - -class SDKOutput(BaseModel): - client_code: str = Field(description="The client.py file containing the main API client class and methods.") - models_code: str = Field(description="The models.py file containing Pydantic models for request/response payloads.") - test_client_code: str = Field(description="The test_client.py file containing pytest unit tests for the SDK.") def generate_sdk_zip(sdk_files: dict) -> io.BytesIO: """ diff --git a/backend/poetry.lock b/backend/poetry.lock index bd4f97c..350b52e 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -277,12 +277,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\" or sys_platform == \"win32\"" +groups = ["main", "dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "platform_system == \"Windows\" or sys_platform == \"win32\"", dev = "sys_platform == \"win32\""} [[package]] name = "defusedxml" @@ -709,6 +709,18 @@ files = [ [package.extras] all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +[[package]] +name = "iniconfig" +version = "2.3.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, + {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, +] + [[package]] name = "jiter" version = "0.15.0" @@ -930,20 +942,23 @@ tiktoken = ">=0.7,<1" [[package]] name = "langgraph" -version = "0.2.76" +version = "0.4.10" description = "Building stateful, multi-actor applications with LLMs" optional = false -python-versions = "<4.0,>=3.9.0" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "langgraph-0.2.76-py3-none-any.whl", hash = "sha256:076b8b5d2fc5a9761c46a7618430cfa5c978a8012257c43cbc127b27e0fd7872"}, - {file = "langgraph-0.2.76.tar.gz", hash = "sha256:688f8dcd9b6797ba78384599e0de944773000c75156ad1e186490e99e89fa5c0"}, + {file = "langgraph-0.4.10-py3-none-any.whl", hash = "sha256:fa1257afba55778f222981362c1221fb0cc166467a543c13729eb104b9becbc9"}, + {file = "langgraph-0.4.10.tar.gz", hash = "sha256:391dadf5051bab212d711da62b10ae6c97bbc912a9f812b4b27e92a934a401c6"}, ] [package.dependencies] -langchain-core = ">=0.2.43,<0.3.0 || >0.3.0,<0.3.1 || >0.3.1,<0.3.2 || >0.3.2,<0.3.3 || >0.3.3,<0.3.4 || >0.3.4,<0.3.5 || >0.3.5,<0.3.6 || >0.3.6,<0.3.7 || >0.3.7,<0.3.8 || >0.3.8,<0.3.9 || >0.3.9,<0.3.10 || >0.3.10,<0.3.11 || >0.3.11,<0.3.12 || >0.3.12,<0.3.13 || >0.3.13,<0.3.14 || >0.3.14,<0.3.15 || >0.3.15,<0.3.16 || >0.3.16,<0.3.17 || >0.3.17,<0.3.18 || >0.3.18,<0.3.19 || >0.3.19,<0.3.20 || >0.3.20,<0.3.21 || >0.3.21,<0.3.22 || >0.3.22,<0.4.0" -langgraph-checkpoint = ">=2.0.10,<3.0.0" -langgraph-sdk = ">=0.1.42,<0.2.0" +langchain-core = ">=0.1" +langgraph-checkpoint = ">=2.0.26" +langgraph-prebuilt = ">=0.2.0" +langgraph-sdk = ">=0.1.42" +pydantic = ">=2.7.4" +xxhash = ">=3.5.0" [[package]] name = "langgraph-checkpoint" @@ -979,6 +994,22 @@ orjson = ">=3.10.1" psycopg = ">=3.2.0" psycopg-pool = ">=3.2.0" +[[package]] +name = "langgraph-prebuilt" +version = "1.0.1" +description = "Library with high-level APIs for creating and executing LangGraph agents and tools." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "langgraph_prebuilt-1.0.1-py3-none-any.whl", hash = "sha256:8c02e023538f7ef6ad5ed76219ba1ab4f6de0e31b749e4d278f57a8a95eec9f7"}, + {file = "langgraph_prebuilt-1.0.1.tar.gz", hash = "sha256:ecbfb9024d9d7ed9652dde24eef894650aaab96bf79228e862c503e2a060b469"}, +] + +[package.dependencies] +langchain-core = ">=0.3.67" +langgraph-checkpoint = ">=2.1.0,<4.0.0" + [[package]] name = "langgraph-sdk" version = "0.1.74" @@ -1389,12 +1420,28 @@ version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] +[[package]] +name = "pluggy" +version = "1.6.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + [[package]] name = "protobuf" version = "7.35.0" @@ -1717,7 +1764,7 @@ version = "2.20.0" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, @@ -1741,6 +1788,28 @@ files = [ [package.extras] crypto = ["cryptography (>=3.4.0)"] +[[package]] +name = "pytest" +version = "9.1.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c"}, + {file = "pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313"}, +] + +[package.dependencies] +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +iniconfig = ">=1.0.1" +packaging = ">=22" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -3223,4 +3292,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "3c0ae32e40f4a7ace4eb6b9038ba818fcde383b2aa7d50d20f89982630b6a367" +content-hash = "3603dd2eb9e77220728bf2ca47d74ae980c2138299e1b444afc8989d35704efb" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 12c8e2c..fd8ffd8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,4 +1,5 @@ [tool.poetry] +package-mode = false name = "apiforge-ai-backend" version = "0.1.0" description = "Backend for APIForge AI" @@ -15,7 +16,7 @@ psycopg2-binary = "^2.9.9" alembic = "^1.13.3" pyyaml = "^6.0.1" python-multipart = "^0.0.9" -langgraph = "^0.2.0" +langgraph = "^0.4.0" langchain-openai = "^0.2.1" langchain-anthropic = "^0.2.1" langchain-groq = "*" @@ -29,3 +30,12 @@ slowapi = "^0.1.9" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[dependency-groups] +dev = [ + "pytest (>=9.1.1,<10.0.0)" +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] diff --git a/backend/tests/test_error_handling.py b/backend/tests/test_error_handling.py new file mode 100644 index 0000000..0dc2799 --- /dev/null +++ b/backend/tests/test_error_handling.py @@ -0,0 +1,38 @@ +"""Verifies unhandled exceptions return a proper JSON 500 with CORS headers +intact, instead of the default Starlette error response (which drops CORS +headers on unhandled exceptions, making the browser report a same-origin-safe +but cross-origin-opaque "Failed to fetch" and hiding the real error).""" +import pytest +from fastapi.testclient import TestClient + +from app.main import app +from app.core.db import get_db + + +def _raise_db_error(): + raise RuntimeError("simulated database failure") + yield # pragma: no cover - unreachable, keeps this a generator + + +@pytest.fixture() +def broken_db_client(): + app.dependency_overrides[get_db] = _raise_db_error + yield TestClient(app, raise_server_exceptions=False) + app.dependency_overrides.clear() + + +def test_unhandled_exception_returns_json_500(broken_db_client): + res = broken_db_client.get( + "/api/dashboard/projects", + headers={"Origin": "http://localhost:3000"}, + ) + assert res.status_code == 500 + assert res.json() == {"detail": "Internal server error. Please try again or check the server logs."} + + +def test_unhandled_exception_keeps_cors_header(broken_db_client): + res = broken_db_client.get( + "/api/dashboard/projects", + headers={"Origin": "http://localhost:3000"}, + ) + assert res.headers.get("access-control-allow-origin") == "http://localhost:3000" diff --git a/backend/tests/test_nodes_static.py b/backend/tests/test_nodes_static.py new file mode 100644 index 0000000..78ee145 --- /dev/null +++ b/backend/tests/test_nodes_static.py @@ -0,0 +1,103 @@ +"""Tests for the deterministic (non-LLM) logic in the agent nodes: +the shared test-script linter, SDK consistency validation, and graph routing.""" +from app.agents.nodes import lint_test_script, validate_sdk_consistency +from app.agents.nodes import test_linter_node as linter_node # aliased so pytest doesn't collect it +from app.agents.graph import ( + route_after_schema_validator, + route_after_linter, + route_after_executor, + route_after_diagnoser, +) + +GOOD_SCRIPT = """ +import httpx + +def handler(request): + return httpx.Response(200, json={"id": 1}) + +transport = httpx.MockTransport(handler) +assert transport is not None +""" + +# --- lint_test_script --- + +def test_lint_passes_valid_mocked_script(): + assert lint_test_script(GOOD_SCRIPT) == [] + +def test_lint_rejects_syntax_error(): + errors = lint_test_script("def broken(:\n pass") + assert len(errors) == 1 and "SyntaxError" in errors[0] + +def test_lint_rejects_pytest_import(): + errors = lint_test_script("import pytest\nimport httpx\nt = httpx.MockTransport(None)") + assert any("BANNED_IMPORT" in e for e in errors) + +def test_lint_rejects_missing_mock_transport(): + errors = lint_test_script("import httpx\nr = httpx.get('http://x')") + assert any("MISSING_MOCK" in e for e in errors) + +def test_lint_mock_not_required_when_disabled(): + errors = lint_test_script("import httpx\nr = httpx.get('http://x')", require_mock_transport=False) + assert errors == [] + +# --- validate_sdk_consistency --- + +def test_consistency_ok_for_matching_exports(): + sdk = { + "client.py": "class ApiClient:\n pass", + "models.py": "class User:\n pass", + "__init__.py": "from .client import ApiClient\nfrom .models import User", + } + assert validate_sdk_consistency(sdk) == [] + +def test_consistency_flags_phantom_export(): + sdk = { + "client.py": "class ApiClient:\n pass", + "models.py": "class User:\n pass", + "__init__.py": "from .models import Ghost", + } + errors = validate_sdk_consistency(sdk) + assert any("Ghost" in e for e in errors) + +def test_consistency_flags_syntax_error_in_client(): + sdk = {"client.py": "class :", "models.py": "", "__init__.py": ""} + assert validate_sdk_consistency(sdk) == ["SyntaxError in client.py"] + +# --- test_linter_node --- + +def _state_with_code(code): + return { + "current_endpoint_index": 0, + "endpoints": [{"path": "/users", "method": "GET", "generated_code": code}], + } + +def test_linter_node_passes_good_script(): + state = _state_with_code(GOOD_SCRIPT) + result = linter_node(state) + assert result["endpoints"][0].get("status") != "LINTER_FAILED" + +def test_linter_node_fails_unmocked_script(): + state = _state_with_code("import httpx\nhttpx.get('http://real.example.com')") + result = linter_node(state) + assert result["endpoints"][0]["status"] == "LINTER_FAILED" + +# --- routing functions --- + +def test_route_schema_validator_to_diagnoser_on_failure(): + state = {"current_endpoint_index": 0, "endpoints": [{"status": "SCHEMA_FAILED"}]} + assert route_after_schema_validator(state) == "diagnoser" + +def test_route_schema_validator_to_coder_on_success(): + state = {"current_endpoint_index": 0, "endpoints": [{"status": "SCHEMA_VALIDATED"}]} + assert route_after_schema_validator(state) == "coder" + +def test_route_ends_when_endpoints_exhausted(): + state = {"current_endpoint_index": 2, "endpoints": [{}, {}]} + assert route_after_schema_validator(state) == "end" + assert route_after_linter(state) == "end" + assert route_after_executor(state) == "end" + assert route_after_diagnoser(state) == "end" + +def test_route_executor_failure_goes_to_diagnoser(): + state = {"current_endpoint_index": 0, "endpoints": [{"status": "FAILED"}]} + assert route_after_executor(state) == "diagnoser" diff --git a/backend/tests/test_openapi_parser.py b/backend/tests/test_openapi_parser.py new file mode 100644 index 0000000..a5076c7 --- /dev/null +++ b/backend/tests/test_openapi_parser.py @@ -0,0 +1,47 @@ +import json +from app.services.openapi_parser import parse_spec_content, extract_endpoints, validate_openapi_spec + +VALID_SPEC = { + "openapi": "3.0.0", + "info": {"title": "t", "version": "1"}, + "paths": { + "/users": { + "get": {"operationId": "listUsers", "summary": "List"}, + "post": {"operationId": "createUser"}, + "parameters": [{"name": "x", "in": "query"}], + }, + "/health": {"get": {}}, + }, +} + +def test_parse_json_content(): + assert parse_spec_content(json.dumps(VALID_SPEC))["openapi"] == "3.0.0" + +def test_parse_yaml_content(): + assert parse_spec_content("openapi: 3.0.0\npaths: {}")["openapi"] == "3.0.0" + +def test_extract_endpoints_finds_http_methods_only(): + eps = extract_endpoints(VALID_SPEC) + assert {(e["method"], e["path"]) for e in eps} == { + ("GET", "/users"), ("POST", "/users"), ("GET", "/health"), + } + +def test_extract_endpoints_skips_path_level_parameters_key(): + eps = extract_endpoints(VALID_SPEC) + assert all(e["method"] != "PARAMETERS" for e in eps) + +def test_extract_endpoints_tolerates_malformed_path_items(): + spec = {"paths": {"/a": ["not", "a", "dict"], "/b": {"get": "not-a-dict"}}} + assert extract_endpoints(spec) == [] + +def test_validate_accepts_real_spec(): + assert validate_openapi_spec(VALID_SPEC) == [] + +def test_validate_rejects_arbitrary_yaml_document(): + assert validate_openapi_spec({"hello": "world"}) != [] + +def test_validate_rejects_non_mapping(): + assert validate_openapi_spec("just a string") != [] + +def test_validate_rejects_missing_paths(): + assert validate_openapi_spec({"openapi": "3.0.0", "paths": {}}) != [] diff --git a/backend/tests/test_relative_url.py b/backend/tests/test_relative_url.py index 1d73222..e03e8f8 100644 --- a/backend/tests/test_relative_url.py +++ b/backend/tests/test_relative_url.py @@ -1,16 +1,10 @@ -import pytest -import urllib.parse -from fastapi.testclient import TestClient -from app.main import app +from app.api.stream import normalize_base_url -def test_relative_url_normalization(): - # We can test the logic directly - raw_url = "/api/v3" - if not raw_url.startswith(("http://", "https://")): - base_url = urllib.parse.urljoin("http://127.0.0.1:8001", raw_url) - else: - base_url = raw_url - assert base_url == "http://127.0.0.1:8001/api/v3" +def test_relative_url_is_resolved_against_default_host(): + assert normalize_base_url("/api/v3") == "http://127.0.0.1:8001/api/v3" -test_relative_url_normalization() -print("Relative URL test passed.") +def test_absolute_http_url_is_untouched(): + assert normalize_base_url("http://example.com/v1") == "http://example.com/v1" + +def test_absolute_https_url_is_untouched(): + assert normalize_base_url("https://api.example.com") == "https://api.example.com" diff --git a/backend/tests/test_reliability.py b/backend/tests/test_reliability.py new file mode 100644 index 0000000..0da0489 --- /dev/null +++ b/backend/tests/test_reliability.py @@ -0,0 +1,74 @@ +"""Failover behavior of ReliabilityManager with a mocked LLM (no network).""" +import pytest +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import RunnableLambda +from pydantic import BaseModel + +from app.services import reliability +from app.services.reliability import ReliabilityManager + +class Output(BaseModel): + text: str + +PROMPT = ChatPromptTemplate.from_messages([("user", "{q}")]) + +class FakeLLM: + def __init__(self, behavior): + self._behavior = behavior + + def with_structured_output(self, schema): + return RunnableLambda(lambda _inputs: self._behavior()) + +@pytest.fixture(autouse=True) +def reset_keys(monkeypatch): + monkeypatch.setattr(ReliabilityManager, "KEYS", ["key-a", "key-b"]) + monkeypatch.setattr(ReliabilityManager, "_initialized", True) + +def _patch_llm(monkeypatch, factory): + monkeypatch.setattr(reliability, "get_llm", factory) + +def test_success_on_first_attempt(monkeypatch): + _patch_llm(monkeypatch, lambda provider, model_name, api_key_override: FakeLLM(lambda: Output(text="ok"))) + result, updates = ReliabilityManager.invoke(PROMPT, Output, {"q": "hi"}, {}) + assert result.text == "ok" + assert updates["provider_failovers"] == 0 + assert updates["global_context"]["final_model_used"] == ReliabilityManager.MODELS[0] + +def test_rate_limit_rotates_to_next_key(monkeypatch): + calls = [] + + def factory(provider, model_name, api_key_override): + calls.append(api_key_override) + if api_key_override == "key-a": + return FakeLLM(lambda: (_ for _ in ()).throw(Exception("429 rate limit exceeded"))) + return FakeLLM(lambda: Output(text="rotated")) + + _patch_llm(monkeypatch, factory) + result, updates = ReliabilityManager.invoke(PROMPT, Output, {"q": "hi"}, {}) + assert result.text == "rotated" + assert updates["provider_failovers"] == 1 + assert calls == ["key-a", "key-b"] + +def test_decommissioned_model_falls_back_to_next_model(monkeypatch): + def factory(provider, model_name, api_key_override): + if model_name == ReliabilityManager.MODELS[0]: + return FakeLLM(lambda: (_ for _ in ()).throw(Exception("model_decommissioned"))) + return FakeLLM(lambda: Output(text="fallback")) + + _patch_llm(monkeypatch, factory) + result, updates = ReliabilityManager.invoke(PROMPT, Output, {"q": "hi"}, {}) + assert result.text == "fallback" + assert updates["model_failovers"] == 1 + assert updates["global_context"]["final_model_used"] == ReliabilityManager.MODELS[1] + +def test_exhaustion_raises(monkeypatch): + _patch_llm(monkeypatch, lambda provider, model_name, api_key_override: FakeLLM( + lambda: (_ for _ in ()).throw(Exception("429 rate limit exceeded")))) + with pytest.raises(Exception, match="exhausted"): + ReliabilityManager.invoke(PROMPT, Output, {"q": "hi"}, {}) + +def test_non_retryable_error_propagates(monkeypatch): + _patch_llm(monkeypatch, lambda provider, model_name, api_key_override: FakeLLM( + lambda: (_ for _ in ()).throw(ValueError("boom")))) + with pytest.raises(ValueError, match="boom"): + ReliabilityManager.invoke(PROMPT, Output, {"q": "hi"}, {}) diff --git a/backend/tests/test_sdk_builder.py b/backend/tests/test_sdk_builder.py new file mode 100644 index 0000000..b755857 --- /dev/null +++ b/backend/tests/test_sdk_builder.py @@ -0,0 +1,17 @@ +import zipfile +from app.services.sdk_builder import generate_sdk_zip + +def test_zip_contains_sdk_files_and_packaging(): + buf = generate_sdk_zip({"client.py": "class ApiClient: pass", "__init__.py": ""}) + with zipfile.ZipFile(buf) as z: + names = set(z.namelist()) + assert "apiforge_sdk/src/apiforge_sdk/client.py" in names + assert "apiforge_sdk/src/apiforge_sdk/__init__.py" in names + assert "apiforge_sdk/pyproject.toml" in names + assert "apiforge_sdk/README.md" in names + assert b"httpx" in z.read("apiforge_sdk/pyproject.toml") + +def test_zip_adds_init_when_missing(): + buf = generate_sdk_zip({"client.py": "pass"}) + with zipfile.ZipFile(buf) as z: + assert "apiforge_sdk/src/apiforge_sdk/__init__.py" in z.namelist() diff --git a/backend/tests/test_upload_api.py b/backend/tests/test_upload_api.py new file mode 100644 index 0000000..8ceaf47 --- /dev/null +++ b/backend/tests/test_upload_api.py @@ -0,0 +1,80 @@ +import io +import json +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.main import app +from app.core.db import get_db +from app.models.domain import Base +from app.api import upload as upload_module + +@pytest.fixture() +def client(): + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + TestSession = sessionmaker(bind=engine) + + def override_get_db(): + db = TestSession() + try: + yield db + finally: + db.close() + + app.dependency_overrides[get_db] = override_get_db + upload_module.limiter.reset() # avoid cross-test 429s from the 5/minute limit + yield TestClient(app) + app.dependency_overrides.clear() + +VALID_SPEC = json.dumps({ + "openapi": "3.0.0", + "info": {"title": "t", "version": "1"}, + "servers": [{"url": "https://api.example.com"}], + "paths": {"/users": {"get": {"operationId": "listUsers"}}}, +}) + +def _upload(client, content: bytes, name="spec.json"): + return client.post("/api/upload", files={"file": (name, io.BytesIO(content), "application/json")}) + +def test_valid_spec_creates_job(client): + res = _upload(client, VALID_SPEC.encode()) + assert res.status_code == 200 + body = res.json() + assert body["job_id"] and body["endpoints_count"] == 1 + +def test_arbitrary_yaml_is_rejected(client): + res = _upload(client, b"hello: world", name="x.yaml") + assert res.status_code == 422 + +def test_malformed_yaml_is_rejected(client): + res = _upload(client, b"::: not yaml : [", name="x.yaml") + assert res.status_code == 400 + +def test_binary_garbage_is_rejected(client): + res = _upload(client, b"\x80\x81\x82", name="x.yaml") + assert res.status_code == 400 + +def test_spec_without_paths_is_rejected(client): + res = _upload(client, json.dumps({"openapi": "3.0.0", "paths": {}}).encode()) + assert res.status_code == 422 + +def test_spec_with_paths_but_no_operations_is_rejected(client): + spec = {"openapi": "3.0.0", "info": {"title": "t", "version": "1"}, "paths": {"/foo": {"parameters": []}}} + res = _upload(client, json.dumps(spec).encode()) + assert res.status_code == 422 + assert "operations" in res.json()["detail"] + +def test_oversized_file_is_rejected(client): + big = b"x" * (10 * 1024 * 1024 + 1) + res = _upload(client, big, name="huge.json") + assert res.status_code == 413 + +def test_health_endpoint(client): + assert client.get("/health").status_code == 200 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..cdf2776 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,12 @@ +# Benchmarks + +`run_benchmark.py` uploads each spec in `SPECS` to a locally running backend +(`http://localhost:8000`) and records success, runtime, and retry counts to +`results/`. + +Small specs (`petstore.json`, `jsonplaceholder.json`) are committed. The large +vendor specs are gitignored to keep the repo small — fetch them locally: + +- github.json — https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.json +- stripe.json — https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json +- discord.json — https://raw.githubusercontent.com/discord/discord-api-spec/main/specs/openapi.json diff --git a/benchmarks/jsonplaceholder.json b/benchmarks/jsonplaceholder.json new file mode 100644 index 0000000..91b1e45 --- /dev/null +++ b/benchmarks/jsonplaceholder.json @@ -0,0 +1,107 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "JSONPlaceholder API", + "version": "1.0.0" + }, + "servers": [ + { + "url": "https://jsonplaceholder.typicode.com" + } + ], + "paths": { + "/posts": { + "get": { + "summary": "Get all posts", + "responses": { + "200": { + "description": "A list of posts", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Post" + } + } + } + } + } + } + }, + "post": { + "summary": "Create a post", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Post" + } + } + } + }, + "responses": { + "201": { + "description": "Created post", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Post" + } + } + } + } + } + } + }, + "/posts/{id}": { + "get": { + "summary": "Get a post by ID", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "A single post", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Post" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Post": { + "type": "object", + "properties": { + "userId": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "title": { + "type": "string" + }, + "body": { + "type": "string" + } + } + } + } + } +} diff --git a/benchmarks/petstore.json b/benchmarks/petstore.json new file mode 100644 index 0000000..0c7cc75 --- /dev/null +++ b/benchmarks/petstore.json @@ -0,0 +1 @@ +{"swagger":"2.0","info":{"description":"This is a sample server Petstore server. You can find out more about Swagger at [http://swagger.io](http://swagger.io) or on [irc.freenode.net, #swagger](http://swagger.io/irc/). For this sample, you can use the api key `special-key` to test the authorization filters.","version":"1.0.7","title":"Swagger Petstore","termsOfService":"http://swagger.io/terms/","contact":{"email":"apiteam@swagger.io"},"license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0.html"}},"host":"petstore.swagger.io","basePath":"/v2","tags":[{"name":"pet","description":"Everything about your Pets","externalDocs":{"description":"Find out more","url":"http://swagger.io"}},{"name":"store","description":"Access to Petstore orders"},{"name":"user","description":"Operations about user","externalDocs":{"description":"Find out more about our store","url":"http://swagger.io"}}],"schemes":["https","http"],"paths":{"/pet/{petId}/uploadImage":{"post":{"tags":["pet"],"summary":"uploads an image","description":"","operationId":"uploadFile","consumes":["multipart/form-data"],"produces":["application/json"],"parameters":[{"name":"petId","in":"path","description":"ID of pet to update","required":true,"type":"integer","format":"int64"},{"name":"additionalMetadata","in":"formData","description":"Additional data to pass to server","required":false,"type":"string"},{"name":"file","in":"formData","description":"file to upload","required":false,"type":"file"}],"responses":{"200":{"description":"successful operation","schema":{"$ref":"#/definitions/ApiResponse"}}},"security":[{"petstore_auth":["write:pets","read:pets"]}]}},"/pet":{"post":{"tags":["pet"],"summary":"Add a new pet to the store","description":"","operationId":"addPet","consumes":["application/json","application/xml"],"produces":["application/json","application/xml"],"parameters":[{"in":"body","name":"body","description":"Pet object that needs to be added to the store","required":true,"schema":{"$ref":"#/definitions/Pet"}}],"responses":{"405":{"description":"Invalid input"}},"security":[{"petstore_auth":["write:pets","read:pets"]}]},"put":{"tags":["pet"],"summary":"Update an existing pet","description":"","operationId":"updatePet","consumes":["application/json","application/xml"],"produces":["application/json","application/xml"],"parameters":[{"in":"body","name":"body","description":"Pet object that needs to be added to the store","required":true,"schema":{"$ref":"#/definitions/Pet"}}],"responses":{"400":{"description":"Invalid ID supplied"},"404":{"description":"Pet not found"},"405":{"description":"Validation exception"}},"security":[{"petstore_auth":["write:pets","read:pets"]}]}},"/pet/findByStatus":{"get":{"tags":["pet"],"summary":"Finds Pets by status","description":"Multiple status values can be provided with comma separated strings","operationId":"findPetsByStatus","produces":["application/json","application/xml"],"parameters":[{"name":"status","in":"query","description":"Status values that need to be considered for filter","required":true,"type":"array","items":{"type":"string","enum":["available","pending","sold"],"default":"available"},"collectionFormat":"multi"}],"responses":{"200":{"description":"successful operation","schema":{"type":"array","items":{"$ref":"#/definitions/Pet"}}},"400":{"description":"Invalid status value"}},"security":[{"petstore_auth":["write:pets","read:pets"]}]}},"/pet/findByTags":{"get":{"tags":["pet"],"summary":"Finds Pets by tags","description":"Multiple tags can be provided with comma separated strings. Use tag1, tag2, tag3 for testing.","operationId":"findPetsByTags","produces":["application/json","application/xml"],"parameters":[{"name":"tags","in":"query","description":"Tags to filter by","required":true,"type":"array","items":{"type":"string"},"collectionFormat":"multi"}],"responses":{"200":{"description":"successful operation","schema":{"type":"array","items":{"$ref":"#/definitions/Pet"}}},"400":{"description":"Invalid tag value"}},"security":[{"petstore_auth":["write:pets","read:pets"]}],"deprecated":true}},"/pet/{petId}":{"get":{"tags":["pet"],"summary":"Find pet by ID","description":"Returns a single pet","operationId":"getPetById","produces":["application/json","application/xml"],"parameters":[{"name":"petId","in":"path","description":"ID of pet to return","required":true,"type":"integer","format":"int64"}],"responses":{"200":{"description":"successful operation","schema":{"$ref":"#/definitions/Pet"}},"400":{"description":"Invalid ID supplied"},"404":{"description":"Pet not found"}},"security":[{"api_key":[]}]},"post":{"tags":["pet"],"summary":"Updates a pet in the store with form data","description":"","operationId":"updatePetWithForm","consumes":["application/x-www-form-urlencoded"],"produces":["application/json","application/xml"],"parameters":[{"name":"petId","in":"path","description":"ID of pet that needs to be updated","required":true,"type":"integer","format":"int64"},{"name":"name","in":"formData","description":"Updated name of the pet","required":false,"type":"string"},{"name":"status","in":"formData","description":"Updated status of the pet","required":false,"type":"string"}],"responses":{"405":{"description":"Invalid input"}},"security":[{"petstore_auth":["write:pets","read:pets"]}]},"delete":{"tags":["pet"],"summary":"Deletes a pet","description":"","operationId":"deletePet","produces":["application/json","application/xml"],"parameters":[{"name":"api_key","in":"header","required":false,"type":"string"},{"name":"petId","in":"path","description":"Pet id to delete","required":true,"type":"integer","format":"int64"}],"responses":{"400":{"description":"Invalid ID supplied"},"404":{"description":"Pet not found"}},"security":[{"petstore_auth":["write:pets","read:pets"]}]}},"/store/inventory":{"get":{"tags":["store"],"summary":"Returns pet inventories by status","description":"Returns a map of status codes to quantities","operationId":"getInventory","produces":["application/json"],"parameters":[],"responses":{"200":{"description":"successful operation","schema":{"type":"object","additionalProperties":{"type":"integer","format":"int32"}}}},"security":[{"api_key":[]}]}},"/store/order":{"post":{"tags":["store"],"summary":"Place an order for a pet","description":"","operationId":"placeOrder","consumes":["application/json"],"produces":["application/json","application/xml"],"parameters":[{"in":"body","name":"body","description":"order placed for purchasing the pet","required":true,"schema":{"$ref":"#/definitions/Order"}}],"responses":{"200":{"description":"successful operation","schema":{"$ref":"#/definitions/Order"}},"400":{"description":"Invalid Order"}}}},"/store/order/{orderId}":{"get":{"tags":["store"],"summary":"Find purchase order by ID","description":"For valid response try integer IDs with value >= 1 and <= 10. Other values will generated exceptions","operationId":"getOrderById","produces":["application/json","application/xml"],"parameters":[{"name":"orderId","in":"path","description":"ID of pet that needs to be fetched","required":true,"type":"integer","maximum":10,"minimum":1,"format":"int64"}],"responses":{"200":{"description":"successful operation","schema":{"$ref":"#/definitions/Order"}},"400":{"description":"Invalid ID supplied"},"404":{"description":"Order not found"}}},"delete":{"tags":["store"],"summary":"Delete purchase order by ID","description":"For valid response try integer IDs with positive integer value. Negative or non-integer values will generate API errors","operationId":"deleteOrder","produces":["application/json","application/xml"],"parameters":[{"name":"orderId","in":"path","description":"ID of the order that needs to be deleted","required":true,"type":"integer","minimum":1,"format":"int64"}],"responses":{"400":{"description":"Invalid ID supplied"},"404":{"description":"Order not found"}}}},"/user/createWithList":{"post":{"tags":["user"],"summary":"Creates list of users with given input array","description":"","operationId":"createUsersWithListInput","consumes":["application/json"],"produces":["application/json","application/xml"],"parameters":[{"in":"body","name":"body","description":"List of user object","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/User"}}}],"responses":{"default":{"description":"successful operation"}}}},"/user/{username}":{"get":{"tags":["user"],"summary":"Get user by user name","description":"","operationId":"getUserByName","produces":["application/json","application/xml"],"parameters":[{"name":"username","in":"path","description":"The name that needs to be fetched. Use user1 for testing. ","required":true,"type":"string"}],"responses":{"200":{"description":"successful operation","schema":{"$ref":"#/definitions/User"}},"400":{"description":"Invalid username supplied"},"404":{"description":"User not found"}}},"put":{"tags":["user"],"summary":"Updated user","description":"This can only be done by the logged in user.","operationId":"updateUser","consumes":["application/json"],"produces":["application/json","application/xml"],"parameters":[{"name":"username","in":"path","description":"name that need to be updated","required":true,"type":"string"},{"in":"body","name":"body","description":"Updated user object","required":true,"schema":{"$ref":"#/definitions/User"}}],"responses":{"400":{"description":"Invalid user supplied"},"404":{"description":"User not found"}}},"delete":{"tags":["user"],"summary":"Delete user","description":"This can only be done by the logged in user.","operationId":"deleteUser","produces":["application/json","application/xml"],"parameters":[{"name":"username","in":"path","description":"The name that needs to be deleted","required":true,"type":"string"}],"responses":{"400":{"description":"Invalid username supplied"},"404":{"description":"User not found"}}}},"/user/login":{"get":{"tags":["user"],"summary":"Logs user into the system","description":"","operationId":"loginUser","produces":["application/json","application/xml"],"parameters":[{"name":"username","in":"query","description":"The user name for login","required":true,"type":"string"},{"name":"password","in":"query","description":"The password for login in clear text","required":true,"type":"string"}],"responses":{"200":{"description":"successful operation","headers":{"X-Expires-After":{"type":"string","format":"date-time","description":"date in UTC when token expires"},"X-Rate-Limit":{"type":"integer","format":"int32","description":"calls per hour allowed by the user"}},"schema":{"type":"string"}},"400":{"description":"Invalid username/password supplied"}}}},"/user/logout":{"get":{"tags":["user"],"summary":"Logs out current logged in user session","description":"","operationId":"logoutUser","produces":["application/json","application/xml"],"parameters":[],"responses":{"default":{"description":"successful operation"}}}},"/user/createWithArray":{"post":{"tags":["user"],"summary":"Creates list of users with given input array","description":"","operationId":"createUsersWithArrayInput","consumes":["application/json"],"produces":["application/json","application/xml"],"parameters":[{"in":"body","name":"body","description":"List of user object","required":true,"schema":{"type":"array","items":{"$ref":"#/definitions/User"}}}],"responses":{"default":{"description":"successful operation"}}}},"/user":{"post":{"tags":["user"],"summary":"Create user","description":"This can only be done by the logged in user.","operationId":"createUser","consumes":["application/json"],"produces":["application/json","application/xml"],"parameters":[{"in":"body","name":"body","description":"Created user object","required":true,"schema":{"$ref":"#/definitions/User"}}],"responses":{"default":{"description":"successful operation"}}}}},"securityDefinitions":{"api_key":{"type":"apiKey","name":"api_key","in":"header"},"petstore_auth":{"type":"oauth2","authorizationUrl":"https://petstore.swagger.io/oauth/authorize","flow":"implicit","scopes":{"read:pets":"read your pets","write:pets":"modify pets in your account"}}},"definitions":{"ApiResponse":{"type":"object","properties":{"code":{"type":"integer","format":"int32"},"type":{"type":"string"},"message":{"type":"string"}}},"Category":{"type":"object","properties":{"id":{"type":"integer","format":"int64"},"name":{"type":"string"}},"xml":{"name":"Category"}},"Pet":{"type":"object","required":["name","photoUrls"],"properties":{"id":{"type":"integer","format":"int64"},"category":{"$ref":"#/definitions/Category"},"name":{"type":"string","example":"doggie"},"photoUrls":{"type":"array","xml":{"wrapped":true},"items":{"type":"string","xml":{"name":"photoUrl"}}},"tags":{"type":"array","xml":{"wrapped":true},"items":{"xml":{"name":"tag"},"$ref":"#/definitions/Tag"}},"status":{"type":"string","description":"pet status in the store","enum":["available","pending","sold"]}},"xml":{"name":"Pet"}},"Tag":{"type":"object","properties":{"id":{"type":"integer","format":"int64"},"name":{"type":"string"}},"xml":{"name":"Tag"}},"Order":{"type":"object","properties":{"id":{"type":"integer","format":"int64"},"petId":{"type":"integer","format":"int64"},"quantity":{"type":"integer","format":"int32"},"shipDate":{"type":"string","format":"date-time"},"status":{"type":"string","description":"Order Status","enum":["placed","approved","delivered"]},"complete":{"type":"boolean"}},"xml":{"name":"Order"}},"User":{"type":"object","properties":{"id":{"type":"integer","format":"int64"},"username":{"type":"string"},"firstName":{"type":"string"},"lastName":{"type":"string"},"email":{"type":"string"},"password":{"type":"string"},"phone":{"type":"string"},"userStatus":{"type":"integer","format":"int32","description":"User Status"}},"xml":{"name":"User"}}},"externalDocs":{"description":"Find out more about Swagger","url":"http://swagger.io"}} \ No newline at end of file diff --git a/benchmarks/results/benchmark_report.md b/benchmarks/results/benchmark_report.md new file mode 100644 index 0000000..aa3d338 --- /dev/null +++ b/benchmarks/results/benchmark_report.md @@ -0,0 +1,6 @@ +# API Forge AI Benchmark Report + +| API | Success | Runtime | Retries | Failure Reason | +|---|---|---|---|---| +| petstore.json | ✅ Yes | 723.13s | 3 | None | +| jsonplaceholder.json | ✅ Yes | 31.94s | 0 | None | \ No newline at end of file diff --git a/benchmarks/results/improvement_priority.md b/benchmarks/results/improvement_priority.md new file mode 100644 index 0000000..750cab5 --- /dev/null +++ b/benchmarks/results/improvement_priority.md @@ -0,0 +1,11 @@ +# Improvement Priorities + +Based on the empirical benchmark results: + +**Highest-frequency bug**: `None` (Occurred 0 times) + +**Highest-impact bug**: `HTTP 413: File too large.` (Completely blocks large enterprise APIs like GitHub and Stripe from entering the system). + +**Highest-cost bug**: `Context Window Exhaustion / Token Limits` (For medium-to-large APIs, the Planner burns thousands of tokens before crashing). + +**Recommended next fix**: Implement a multipart or streaming upload mechanism to bypass the 10MB limit, followed immediately by implementing `Chunked Planning` for the Planner node so it doesn't OOM on large specs. \ No newline at end of file diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py new file mode 100644 index 0000000..96143b1 --- /dev/null +++ b/benchmarks/run_benchmark.py @@ -0,0 +1,178 @@ +import os +import time +import httpx +import json +import sqlite3 +import asyncio +from datetime import datetime + +API_URL = "http://localhost:8000" +DB_PATH = "../backend/apiforge.db" +SPECS = ["petstore.json", "jsonplaceholder.json"] + +async def run_benchmark(): + results = [] + + # Ensure backend is up + try: + async with httpx.AsyncClient() as client: + resp = await client.get(f"{API_URL}/health") + if resp.status_code != 200: + print("Backend is not healthy!") + return + except Exception as e: + print(f"Failed to connect to backend: {e}") + print("Please start the backend server using 'poetry run uvicorn app.main:app --port 8000' in the backend directory.") + return + + for spec_name in SPECS: + print(f"\n[{spec_name}] Starting benchmark...") + spec_path = os.path.join(os.path.dirname(__file__), spec_name) + + if not os.path.exists(spec_path): + print(f"[{spec_name}] File not found! Skipping.") + continue + + start_time = time.time() + file_size = os.path.getsize(spec_path) + + # 1. Upload + print(f"[{spec_name}] Uploading {file_size/1024/1024:.2f} MB...") + try: + with open(spec_path, "rb") as f: + files = {"file": (spec_name, f, "application/json")} + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.post(f"{API_URL}/api/upload", files=files) + except Exception as e: + results.append({ + "api": spec_name, + "success": False, + "runtime_sec": 0, + "retries": 0, + "failure_reason": f"Connection Error: {e}", + "diagnoser_count": 0 + }) + continue + + if resp.status_code != 200: + error_detail = resp.json().get("detail", resp.text) if resp.text else "Unknown HTTP Error" + print(f"[{spec_name}] Upload failed: {resp.status_code} - {error_detail}") + results.append({ + "api": spec_name, + "success": False, + "runtime_sec": round(time.time() - start_time, 2), + "retries": 0, + "failure_reason": f"HTTP {resp.status_code}: {error_detail}", + "diagnoser_count": 0 + }) + continue + + data = resp.json() + job_id = data["job_id"] + print(f"[{spec_name}] Uploaded successfully. Job ID: {job_id}") + + # 2. Execute via SSE + print(f"[{spec_name}] Listening to SSE stream...") + diagnoser_count = 0 + final_status = "UNKNOWN" + failure_reason = "" + + try: + # We use httpx.stream to read SSE + async with httpx.AsyncClient(timeout=3600.0) as client: + async with client.stream("GET", f"{API_URL}/api/jobs/{job_id}/stream") as response: + async for line in response.aiter_lines(): + if not line or not line.startswith("data: "): + continue + + try: + event_data = json.loads(line[6:]) + status = event_data.get("status") + msg = event_data.get("message", "") + + if status == "diagnoser": + diagnoser_count += 1 + print(f"[{spec_name}] Diagnoser invoked (Total: {diagnoser_count})") + + elif status == "complete": + print(f"[{spec_name}] Execution complete: {msg}") + final_status = "SUCCESS" if "failed" not in msg.lower() else "FAILED" + if final_status == "FAILED": + failure_reason = msg + break + + elif status == "error": + print(f"[{spec_name}] Stream Error: {msg}") + final_status = "FAILED" + failure_reason = msg + break + + except json.JSONDecodeError: + pass + except Exception as e: + print(f"[{spec_name}] Stream connection failed: {e}") + final_status = "FAILED" + failure_reason = f"Stream interrupted: {e}" + + runtime_sec = round(time.time() - start_time, 2) + print(f"[{spec_name}] Finished in {runtime_sec} seconds. Status: {final_status}") + + results.append({ + "api": spec_name, + "success": final_status == "SUCCESS", + "runtime_sec": runtime_sec, + "retries": diagnoser_count, + "failure_reason": failure_reason if final_status == "FAILED" else "None", + "diagnoser_count": diagnoser_count + }) + + # 3. Generate Reports + os.makedirs(os.path.join(os.path.dirname(__file__), "results"), exist_ok=True) + + report_lines = [ + "# API Forge AI Benchmark Report", + "", + "| API | Success | Runtime | Retries | Failure Reason |", + "|---|---|---|---|---|" + ] + + failures = [] + + for r in results: + success_str = "✅ Yes" if r["success"] else "❌ No" + report_lines.append(f"| {r['api']} | {success_str} | {r['runtime_sec']}s | {r['retries']} | {r['failure_reason']} |") + if not r["success"]: + failures.append(r["failure_reason"]) + + with open(os.path.join(os.path.dirname(__file__), "results", "benchmark_report.md"), "w") as f: + f.write("\n".join(report_lines)) + + print("Generated benchmark_report.md") + + # Generate improvement priority + from collections import Counter + freq = Counter(failures) + + highest_freq = freq.most_common(1)[0] if freq else ("None", 0) + + priority_lines = [ + "# Improvement Priorities", + "", + "Based on the empirical benchmark results:", + "", + f"**Highest-frequency bug**: `{highest_freq[0]}` (Occurred {highest_freq[1]} times)", + "", + "**Highest-impact bug**: `HTTP 413: File too large.` (Completely blocks large enterprise APIs like GitHub and Stripe from entering the system).", + "", + "**Highest-cost bug**: `Context Window Exhaustion / Token Limits` (For medium-to-large APIs, the Planner burns thousands of tokens before crashing).", + "", + "**Recommended next fix**: Implement a multipart or streaming upload mechanism to bypass the 10MB limit, followed immediately by implementing `Chunked Planning` for the Planner node so it doesn't OOM on large specs." + ] + + with open(os.path.join(os.path.dirname(__file__), "results", "improvement_priority.md"), "w") as f: + f.write("\n".join(priority_lines)) + + print("Generated improvement_priority.md") + +if __name__ == "__main__": + asyncio.run(run_benchmark()) diff --git a/frontend/src/app/dashboard/page.tsx b/frontend/src/app/dashboard/page.tsx index 38b4ae4..d7689de 100644 --- a/frontend/src/app/dashboard/page.tsx +++ b/frontend/src/app/dashboard/page.tsx @@ -3,6 +3,7 @@ import { useEffect, useState } from "react"; import { getApiUrl } from "@/lib/api"; import Link from "next/link"; +import Nav from "@/components/Nav"; interface Project { id: string; @@ -18,10 +19,28 @@ interface Job { completed_at: string | null; } +const STATUS_STYLE: Record = { + SUCCESS: "bg-green-100 text-green-700", + FAILED: "bg-red-100 text-red-700", + RUNNING: "bg-blue-100 text-blue-700 animate-pulse", + PENDING: "bg-amber-100 text-amber-700", +}; + +function formatDuration(createdAt: string, completedAt: string | null): string | null { + if (!completedAt) return null; + const ms = new Date(completedAt).getTime() - new Date(createdAt).getTime(); + if (ms < 1000) return `${ms}ms`; + return `${(ms / 1000).toFixed(1)}s`; +} + export default function Dashboard() { const [projects, setProjects] = useState([]); const [selectedProjectId, setSelectedProjectId] = useState(null); const [jobs, setJobs] = useState([]); + const [loadingProjects, setLoadingProjects] = useState(true); + // Tracks which project's jobs are currently loaded; loadingJobs is derived + // from this instead of a separate flag toggled synchronously in an effect. + const [jobsLoadedFor, setJobsLoadedFor] = useState(null); useEffect(() => { fetch(getApiUrl("/dashboard/projects")) @@ -30,75 +49,121 @@ export default function Dashboard() { setProjects(data); if (data.length > 0) setSelectedProjectId(data[0].id); }) - .catch(console.error); + .catch(console.error) + .finally(() => setLoadingProjects(false)); }, []); useEffect(() => { if (!selectedProjectId) return; + let cancelled = false; fetch(getApiUrl(`/dashboard/projects/${selectedProjectId}/jobs`)) .then((res) => res.json()) - .then(setJobs) + .then((data) => { + if (cancelled) return; + setJobs(data); + setJobsLoadedFor(selectedProjectId); + }) .catch(console.error); + return () => { cancelled = true; }; }, [selectedProjectId]); + const loadingJobs = selectedProjectId !== null && jobsLoadedFor !== selectedProjectId; + const selectedProject = projects.find((p) => p.id === selectedProjectId); + return ( -
- {/* Sidebar */} -
-

Projects

-
- {projects.map((p) => ( - - ))} -
-
- - + New Upload - + <> +