feat: merge claude modifications and fixes

.claude/launch.json

@@ -0,0 +1,19 @@
+{
+  "version": "0.0.1",
+  "configurations": [
+    {
+      "name": "backend",
+      "runtimeExecutable": "poetry",
+      "runtimeArgs": ["run", "uvicorn", "app.main:app", "--reload", "--port", "8000"],
+      "cwd": "backend",
+      "port": 8000
+    },
+    {
+      "name": "frontend",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": ["run", "dev"],
+      "cwd": "frontend",
+      "port": 3000
+    }
+  ]
+}

.gitignore

@@ -10,8 +10,8 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
-lib64/
+backend/lib/
+backend/lib64/
 parts/
 sdist/
 var/
@@ -68,3 +68,11 @@ backend/specs/*.json
 backend/tiny_spec.json
 backend/large_spec.json
 backend/scripts/
+
+# Large vendor API specs (fetch locally; see benchmarks/README.md)
+benchmarks/github.json
+benchmarks/stripe.json
+benchmarks/discord.json
+
+# Claude Code local-only settings (personal permission allowlist)
+.claude/settings.local.json

CONTRIBUTING.md

@@ -13,6 +13,28 @@ We welcome contributions! Please follow the guidelines below to ensure a smooth
 3. Make your changes and run existing tests. If you create new scripts to test behavior, please place them in `backend/scripts/` (these are ignored by git to keep history clean).
 4. Do not commit temporary `.pyc` caches, `.log` files, or generated zip artifacts. Our `.gitignore` should catch most of these, but please be mindful.
 
+## Running Tests
+
+Backend unit tests (no API keys or network required):
+```bash
+cd backend
+poetry run pytest
+```
+
+Frontend lint, type-check, and build:
+```bash
+cd frontend
+npm run lint
+npx tsc --noEmit
+npm run build
+```
+
+End-to-end benchmarks against a running backend (requires `GROQ_API_KEY` in `backend/.env`):
+```bash
+cd backend && poetry run uvicorn app.main:app --port 8000  # terminal 1
+cd benchmarks && python run_benchmark.py                   # terminal 2
+```
+
 ## Pull Requests
 - Ensure your commits are logically structured (e.g. separate your schema updates from your UI updates).
 - Reference any open issues in your PR description.

README.md

@@ -2,7 +2,8 @@
 
 API Forge AI is an autonomous, agentic system built with LangGraph that ingests an OpenAPI schema and dynamically generates, tests, and self-heals Python SDK clients.
 A self healing and self improving agentic platform to turn API Docs to SDK
-
+version 2 launching on july 17th. 
+Update
 ## Overview
 
 The system orchestrates multiple LLM-powered agents to ensure that the generated SDK is structurally sound, semantically correct, and fully tested against real or mocked network conditions.

backend/app/agents/graph.py

@@ -1,6 +1,6 @@
 from langgraph.graph import StateGraph, START, END
 from app.agents.state import AgentState
-from app.agents.nodes import planner_node, coder_node, executor_node, diagnoser_node, sdk_validator_node, schema_validator_node
+from app.agents.nodes import planner_node, coder_node, executor_node, diagnoser_node, sdk_validator_node, schema_validator_node, test_linter_node
 
 def route_after_diagnoser(state: AgentState) -> str:
     idx = state.get("current_endpoint_index", 0)
@@ -28,6 +28,20 @@ def route_after_schema_validator(state: AgentState) -> str:
 
     return "coder"
 
+def route_after_coder(state: AgentState) -> str:
+    # Always route to linter
+    return "test_linter"
+
+def route_after_linter(state: AgentState) -> str:
+    idx = state.get("current_endpoint_index", 0)
+    endpoints = state.get("endpoints", [])
+    if idx >= len(endpoints):
+        return "end"
+    current_ep = endpoints[idx]
+    if current_ep.get("status") == "LINTER_FAILED":
+        return "diagnoser"
+    return "executor"
+
 def route_after_executor(state: AgentState) -> str:
     idx = state.get("current_endpoint_index", 0)
     endpoints = state.get("endpoints", [])
@@ -48,6 +62,7 @@ def build_graph(checkpointer=None):
     workflow.add_node("sdk_validator", sdk_validator_node)
     workflow.add_node("schema_validator", schema_validator_node)
     workflow.add_node("coder", coder_node)
+    workflow.add_node("test_linter", test_linter_node)
     workflow.add_node("executor", executor_node)
     workflow.add_node("diagnoser", diagnoser_node)
 
@@ -73,7 +88,17 @@ def build_graph(checkpointer=None):
         }
     )
 
-    workflow.add_edge("coder", "executor")
+    workflow.add_edge("coder", "test_linter")
+
+    workflow.add_conditional_edges(
+        "test_linter",
+        route_after_linter,
+        {
+            "diagnoser": "diagnoser",
+            "executor": "executor",
+            "end": END
+        }
+    )
 
     workflow.add_conditional_edges(
         "executor",

backend/app/agents/nodes.py

@@ -17,12 +17,16 @@ class CoderOutput(BaseModel):
     reasoning: str = Field(description="Reasoning about how to test this specific endpoint using the generated SDK.")
     python_code: str = Field(description="A complete Python script using the generated SDK (`import apiforge_sdk`) to test the endpoint. Make sure to instantiate the client, call the method, and assert that the response is correct (e.g. valid Pydantic model).")
 
+class Patch(BaseModel):
+    file_name: str = Field(description="Must be exactly 'client.py' or 'models.py'.")
+    search_string: str = Field(description="The exact string in the file to be replaced. Must match exactly.")
+    replace_string: str = Field(description="The string to replace it with.")
+
 class DiagnoserOutput(BaseModel):
     likely_cause: str = Field(description="The likely cause of the failure based on the execution logs.")
     error_category: str = Field(description="Must be one of: 'sdk_error', 'schema_error', 'test_error'.")
     mutation_instructions: str = Field(description="Specific instructions for what was wrong.")
-    client_code: str = Field(description="The FULL, CORRECTED apiforge_sdk/client.py code. If no changes needed, output the original.")
-    models_code: str = Field(description="The FULL, CORRECTED apiforge_sdk/models.py code. If no changes needed, output the original.")
+    patches: list[Patch] = Field(description="List of text replacement patches to apply to the SDK files.", default_factory=list)
 
 class SchemaValidatorOutput(BaseModel):
     python_code: str = Field(description="A short python script using `httpx` to fetch a real payload from the API, import the correct Pydantic model from `apiforge_sdk.models`, and run `Model.model_validate()` against it.")
@@ -94,6 +98,36 @@ def get_defined_symbols(code: str):
 
     return errors
 
+def lint_test_script(code: str, require_mock_transport: bool = True) -> list[str]:
+    """Statically validates a generated test script: syntax, banned imports,
+    and (optionally) that httpx.MockTransport is used so no real network
+    calls are made."""
+    errors = []
+    try:
+        tree = ast.parse(code)
+    except SyntaxError as e:
+        return [f"SyntaxError: {str(e)}"]
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                if 'pytest' in alias.name:
+                    errors.append("BANNED_IMPORT: 'pytest' is not allowed. Use standard assert statements.")
+        elif isinstance(node, ast.ImportFrom):
+            if node.module and 'pytest' in node.module:
+                errors.append("BANNED_IMPORT: 'pytest' is not allowed. Use standard assert statements.")
+
+    if require_mock_transport:
+        has_mock_transport = any(
+            (isinstance(node, ast.Attribute) and node.attr == 'MockTransport') or
+            (isinstance(node, ast.Name) and node.id == 'MockTransport')
+            for node in ast.walk(tree)
+        )
+        if not has_mock_transport:
+            errors.append("MISSING_MOCK: You must use `httpx.MockTransport(handler)` to mock the API response. Real network calls are not allowed in this validation mode.")
+
+    return errors
+
 def sdk_validator_node(state: AgentState) -> dict:
     """Pre-execution validation stage: Validates SDK syntax and imports."""
     sdk_files = state.get("sdk_files", {})
@@ -138,22 +172,50 @@ def schema_validator_node(state: AgentState) -> dict:
     if current_ep.get("status") == "FAILED_PERMANENTLY":
         return {"current_endpoint_index": idx + 1, "endpoints": endpoints}
 
+    method = current_ep.get("method", "GET").upper()
+    has_auth = state.get("auth_credentials") is not None
+
+    is_safe_method = method in ["GET", "HEAD", "OPTIONS"]
+
+    if is_safe_method and has_auth:
+        validation_mode = "REAL"
+        system_prompt = "You are a Schema Validator. Write a short Python script to fetch a real payload from the API and validate it using the generated Pydantic models. Use `httpx.get` (or appropriate method). Do NOT use the generated ApiClient, just raw httpx. Import the correct model from `apiforge_sdk.models` and run `Model.model_validate(item)`. If it's a list, validate one item. Do not use markdown blocks, just raw python string."
+    else:
+        validation_mode = "SYNTHETIC"
+        system_prompt = "You are a Schema Validator. Write a short Python script to synthetically generate a dummy payload based EXACTLY on the OpenAPI schema for this endpoint, and validate it using the generated Pydantic models. You MUST use `httpx.MockTransport(handler)` to mock the API response. Do NOT make a real network request. Import the correct model from `apiforge_sdk.models` and run `Model.model_validate(item)`. Write plain multi-line Python with normal newlines and indentation — never compress statements onto one line with semicolons. Do not use markdown blocks, just raw python string."
+
+    current_ep["validation_mode"] = validation_mode
+
     prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are a Schema Validator. Write a short Python script to fetch a sample payload from the API and validate it using the generated Pydantic models. Use `httpx.get` (or appropriate method). Do NOT use the generated ApiClient, just raw httpx. Import the correct model from `apiforge_sdk.models` and run `Model.model_validate(item)`. If it's a list, validate one item. Do not use markdown blocks, just raw python string."),
-        ("user", "Endpoint: {method} {path}\nBase URL: {base_url}\nModels:\n{models_py}")
+        ("system", system_prompt),
+        ("user", "Endpoint: {method} {path}\nBase URL: {base_url}\nModels:\n{models_py}\nPrevious Diagnostic Feedback:\n{diagnostic_feedback}\nPrevious Failed Script (fix its mistakes, do not repeat them):\n{previous_script}\nPrevious Error Output:\n{previous_stderr}")
     ])
-    
+
     try:
         sdk_files = state.get("sdk_files", {})
         input_vars = {
             "method": current_ep.get("method"),
             "path": current_ep.get("path"),
             "base_url": state.get("base_url"),
-            "models_py": sdk_files.get("models.py", "")
+            "models_py": sdk_files.get("models.py", ""),
+            "diagnostic_feedback": current_ep.get("diagnostic_feedback") or "None",
+            "previous_script": current_ep.get("generated_code") or "None",
+            "previous_stderr": current_ep.get("execution_stderr") or "None"
         }
-        
+
         result, updates = ReliabilityManager.invoke(prompt, SchemaValidatorOutput, input_vars, state)
-
+
+        # Lint before executing: schema scripts previously ran unchecked, so a
+        # SyntaxError or a real network call could slip straight to the executor.
+        lint_errors = lint_test_script(result.python_code, require_mock_transport=(validation_mode == "SYNTHETIC"))
+        if lint_errors:
+            current_ep["status"] = "SCHEMA_FAILED"
+            current_ep["generated_code"] = result.python_code
+            current_ep["execution_stdout"] = ""
+            current_ep["execution_stderr"] = "Schema Validation Script Linter Failed:\n" + "\n".join(lint_errors)
+            current_ep["agent_reasoning"] = "Linter rejected the schema validation script. Routing to Diagnoser."
+            return {"endpoints": endpoints, **updates}
+
         executor = get_executor()
         success, stdout, stderr = executor.execute_sdk_test(sdk_files, result.python_code)
 
@@ -238,6 +300,29 @@ def coder_node(state: AgentState) -> dict:
 
     return {"endpoints": endpoints, **updates}
 
+def test_linter_node(state: AgentState) -> AgentState:
+    """Statically verifies the generated test script before execution."""
+    print("--- TEST LINTER ---")
+    endpoints = state.get("endpoints", [])
+    idx = state.get("current_endpoint_index", 0)
+    if idx >= len(endpoints):
+        return state
+
+    current_ep = endpoints[idx]
+    code = current_ep.get("generated_code", "")
+
+    errors = lint_test_script(code, require_mock_transport=True)
+
+    if errors:
+        current_ep["status"] = "LINTER_FAILED"
+        current_ep["execution_stderr"] = "Test Script Linter Failed:\n" + "\n".join(errors)
+        current_ep["agent_reasoning"] = "Linter rejected the test script. Routing to Diagnoser."
+        print(f"Linter failed: {errors}")
+    else:
+        print("Linter passed.")
+
+    return {"endpoints": endpoints}
+
 def executor_node(state: AgentState) -> AgentState:
     print("--- EXECUTOR ---")
     endpoints = state["endpoints"]
@@ -284,7 +369,7 @@ def diagnoser_node(state: AgentState) -> dict:
         return {"current_endpoint_index": idx + 1, "endpoints": endpoints}
 
     prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are an API Debugging Expert. Analyze the execution logs. If the error is an 'SDK Consistency Validation Failed' error OR if a test fails because the SDK returns a raw httpx.Response instead of a Pydantic model (e.g. AssertionError on the return type), the SDK IS FLAWED and you MUST fix the SDK files (`client.py` or `models.py`). NEVER downgrade `model_validate()` to `User(**item)` unless `model_validate` causes an actual runtime failure. Ensure that relative imports are used inside the SDK (e.g. `from .models import User`). When configuring Pydantic models, you MUST use Pydantic V2 `model_config = ConfigDict(populate_by_name=True, extra='forbid')` as a direct class attribute, and NEVER use the Pydantic V1 `class Config:` block. Make sure to import `ConfigDict` from `pydantic`. Select the correct `error_category` ('sdk_error', 'schema_error', 'test_error'). Only modify the files relevant to the error category. For `sdk_error`, output corrected `client.py` or `models.py`. For `schema_error`, modify `models.py`. For `test_error`, provide `mutation_instructions` for the test. Output the FULL corrected Python code for `client.py` and `models.py` (do not truncate, output the entire file)."),
+        ("system", "You are an API Debugging Expert. Analyze the execution logs. If the error is an 'SDK Consistency Validation Failed' error OR if a test fails because the SDK returns a raw httpx.Response instead of a Pydantic model (e.g. AssertionError on the return type), the SDK IS FLAWED and you MUST fix the SDK files (`client.py` or `models.py`). NEVER downgrade `model_validate()` to `User(**item)` unless `model_validate` causes an actual runtime failure. Ensure that relative imports are used inside the SDK (e.g. `from .models import User`). When configuring Pydantic models, you MUST use Pydantic V2 `model_config = ConfigDict(populate_by_name=True, extra='forbid')` as a direct class attribute, and NEVER use the Pydantic V1 `class Config:` block. Make sure to import `ConfigDict` from `pydantic`. Select the correct `error_category` ('sdk_error', 'schema_error', 'test_error'). Only modify the files relevant to the error category. You MUST provide specific string replacement patches. The `search_string` MUST match exactly a contiguous block of text in the file."),
         ("user", "Endpoint: {method} {path}\nExecution Logs:\n{logs}\nTest Script:\n{code}\nSDK client.py:\n{client_py}\nSDK models.py:\n{models_py}")
     ])
 
@@ -300,18 +385,32 @@ def diagnoser_node(state: AgentState) -> dict:
         }
 
         result, updates = ReliabilityManager.invoke(prompt, DiagnoserOutput, input_vars, state)
-
+
+        # Guard against misdiagnosis: a SyntaxError raised by the test script
+        # itself can never be an SDK problem, so don't let the LLM patch the
+        # SDK for it. (Observed: it blamed models.py for a one-line script.)
+        stderr = current_ep.get("execution_stderr", "") or ""
+        if "SyntaxError" in stderr and "test_script.py" in stderr and "models.py" not in stderr and "client.py" not in stderr:
+            result.error_category = "test_error"
+            result.patches = []
+
         current_ep["agent_reasoning"] = f"Diagnosed failure: {result.likely_cause}. Category: {result.error_category}"
-        
+
         feedback = result.mutation_instructions
 
-        # Apply the fixed SDK files to the state based on category
+        # Apply the fixed SDK files to the state based on patches
         if result.error_category in ["sdk_error", "schema_error"]:
-            sdk_files["client.py"] = result.client_code
-            sdk_files["models.py"] = result.models_code
-            feedback += f"\n\n[Diagnoser patched sdk_files in memory. Category: {result.error_category}]"
+            for patch in result.patches:
+                fname = patch.file_name
+                if fname in sdk_files:
+                    if patch.search_string in sdk_files[fname]:
+                        sdk_files[fname] = sdk_files[fname].replace(patch.search_string, patch.replace_string)
+                    else:
+                        feedback += f"\n\n[Warning: Patch search string not found in {fname}]"
+
+            feedback += f"\n\n[Diagnoser applied patches in memory. Category: {result.error_category}]"
             print("--- DIAGNOSER PATCHED SDK ---")
-            print(f"client.py:\n{sdk_files['client.py'][:500]}...")
+            print(f"client.py:\n{sdk_files.get('client.py', '')[:500]}...")
 
         current_ep["diagnostic_feedback"] = feedback
 

backend/app/agents/state.py

@@ -11,15 +11,22 @@ class EndpointState(TypedDict, total=False):
     success: bool
     generated_code: str
     diagnostic_feedback: str
+    schema_validated: bool
+    validation_mode: str
+    execution_stdout: str
+    execution_stderr: str
 
-class AgentState(TypedDict):
+class AgentState(TypedDict, total=False):
     spec_content: str
     base_url: str
     endpoints: List[EndpointState]
     current_endpoint_index: int
     errors: List[str]
     global_context: Dict[str, Any]
     sdk_files: Dict[str, str]
+    # Optional API credentials; when present, safe (GET/HEAD/OPTIONS) endpoints
+    # are schema-validated against the real API instead of synthetic payloads.
+    auth_credentials: Optional[Dict[str, str]]
     current_key_index: int
     current_model_index: int
     provider_failovers: int