Add maze benchmarks and linked artifacts

README.md

@@ -115,7 +115,7 @@ The `/tasks` endpoints expose an asynchronous `task_status_v1` lifecycle with `q
 
 Course PDFs can be ingested with Docling and indexed into a local Chroma store. During assignment creation, the app will search both indexed course documents and live Canvas module content, then attach the most relevant excerpts to generated outputs.
 
-For explicit maze-search assignments that require `maze_solvers.py`, the generator now emits a working Python maze project with BFS, DFS, A* implementations, a sample maze file, and generated tests instead of only stub functions.
+For explicit maze-search assignments that require `maze_solvers.py`, the generator now emits a working Python maze project with BFS, DFS, A* implementations, a benchmark script, a sample maze file, generated tests, and downloaded linked maze artifacts when the assignment brief exposes maze text files.
 
 ## MCP Server
 

app/agent.py

@@ -178,6 +178,7 @@ async def create_repository_for_assignment(
         assignment: dict,
         language: str = "python",
         course_context: Optional[Sequence[dict]] = None,
+        assignment_artifacts: Optional[Sequence[dict]] = None,
     ) -> Optional[dict]:
         """Create a GitHub repository with assignment starter files."""
         assignment_name = assignment.get("name", "Assignment")
@@ -212,6 +213,7 @@ async def create_repository_for_assignment(
             due_date=due_at,
             language=language,
             course_context=list(course_context or []),
+            assignment_artifacts=list(assignment_artifacts or []),
         )
 
         owner = self.github_org if self.github_org else self.github_username
@@ -230,10 +232,27 @@ async def create_repository_for_assignment(
             "repository": repo,
             "assignment": assignment,
             "course_context": list(course_context or []),
+            "assignment_artifacts": list(assignment_artifacts or []),
             "files_created": list(starter_files.keys()),
             "files_uploaded": files_ok,
         }
 
+    async def fetch_assignment_artifacts(self, course_id: int, assignment: dict) -> list[dict[str, Any]]:
+        """Download linked text-maze artifacts from the assignment brief when available."""
+        try:
+            artifacts = await asyncio.to_thread(
+                self.canvas_tools.download_assignment_maze_artifacts,
+                course_id,
+                assignment,
+            )
+        except Exception as error:
+            print(f"\n⚠️  Assignment artifact download failed: {error}")
+            return []
+
+        if artifacts:
+            print(f"\n🧪 Downloaded {len(artifacts)} linked maze artifact(s) from the assignment brief.")
+        return artifacts
+
     async def create_notion_page_for_assignment(self, assignment: dict) -> Optional[dict]:
         """Create a Notion page for a writing assignment."""
         return await self.create_notion_page_for_assignment_with_mode(assignment, content_mode="structured")
@@ -348,11 +367,13 @@ async def run(
         print(f"\n🧭 Assignment type selected: {assignment_type}")
 
         if assignment_type == "coding":
+            assignment_artifacts = await self.fetch_assignment_artifacts(course_id, assignment)
             print(f"\n🚀 Creating GitHub repository with {language} starter code...")
             result = await self.create_repository_for_assignment(
                 assignment,
                 language,
                 course_context=course_context,
+                assignment_artifacts=assignment_artifacts,
             )
 
             if not result or "repository" not in result:

scaffolding/templates.py

@@ -1226,6 +1226,126 @@ def _append_maze_readme_notes(existing_readme: str) -> str:
     )
 
 
+def _build_maze_benchmark_file(function_names: List[str], artifact_paths: List[str]) -> str:
+    solver_names = list(function_names[:3])
+    defaults = ["maze_solver_one", "maze_solver_two", "maze_solver_three"]
+    for default_name in defaults:
+        if len(solver_names) >= 3:
+            break
+        if default_name not in solver_names:
+            solver_names.append(default_name)
+
+    default_candidates = [*artifact_paths, "maze.txt"]
+    candidate_lines = "\n".join(f'    "{path}",' for path in default_candidates)
+
+    return (
+        '"""Benchmark helper for generated maze assignments."""\n\n'
+        "from __future__ import annotations\n\n"
+        "import json\n"
+        "import sys\n"
+        "import tracemalloc\n"
+        "from pathlib import Path\n"
+        "from time import perf_counter\n\n"
+        f"from maze_solvers import {solver_names[0]}, {solver_names[1]}, {solver_names[2]}\n\n"
+        "DEFAULT_MAZE_CANDIDATES = [\n"
+        f"{candidate_lines}\n"
+        "]\n\n"
+        "def _pick_maze_path(explicit_path: str | None = None) -> str:\n"
+        "    if explicit_path:\n"
+        "        return explicit_path\n"
+        "    for candidate in DEFAULT_MAZE_CANDIDATES:\n"
+        "        if Path(candidate).exists():\n"
+        "            return candidate\n"
+        '    return "maze.txt"\n\n'
+        "def _path_cell_count(solved_maze: str) -> int:\n"
+        '    body = "\\n".join(solved_maze.splitlines()[1:])\n'
+        '    return body.count("*") + 2\n\n'
+        "def _run_single(name: str, solver, maze_path: str) -> dict:\n"
+        "    tracemalloc.start()\n"
+        "    started_at = perf_counter()\n"
+        "    solved_maze = solver(maze_path)\n"
+        "    runtime_seconds = perf_counter() - started_at\n"
+        "    _, peak_bytes = tracemalloc.get_traced_memory()\n"
+        "    tracemalloc.stop()\n"
+        "    return {\n"
+        '        "algorithm": name,\n'
+        '        "runtime_seconds": round(runtime_seconds, 6),\n'
+        '        "peak_memory_bytes": peak_bytes,\n'
+        '        "path_cell_count": _path_cell_count(solved_maze),\n'
+        '        "solved_maze": solved_maze,\n'
+        "    }\n\n"
+        "def benchmark_solvers(maze_path: str | None = None) -> dict:\n"
+        "    selected_maze = _pick_maze_path(maze_path)\n"
+        "    results = [\n"
+        f'        _run_single("{solver_names[0]}", {solver_names[0]}, selected_maze),\n'
+        f'        _run_single("{solver_names[1]}", {solver_names[1]}, selected_maze),\n'
+        f'        _run_single("{solver_names[2]}", {solver_names[2]}, selected_maze),\n'
+        "    ]\n"
+        "    return {\n"
+        '        "maze_path": selected_maze,\n'
+        '        "results": results,\n'
+        "    }\n\n"
+        "def _build_markdown_report(payload: dict) -> str:\n"
+        "    lines = [\n"
+        '        "# Maze Benchmark Results",\n'
+        '        "",\n'
+        '        f"- Maze: {payload[\"maze_path\"]}",\n'
+        '        "",\n'
+        '        "| Algorithm | Runtime (s) | Peak Memory (bytes) | Path Cells |",\n'
+        '        "| --- | ---: | ---: | ---: |",\n'
+        "    ]\n"
+        "    for result in payload[\"results\"]:\n"
+        "        lines.append(\n"
+        '            f"| {result[\"algorithm\"]} | {result[\"runtime_seconds\"]:.6f} | {result[\"peak_memory_bytes\"]} | {result[\"path_cell_count\"]} |"\n'
+        "        )\n"
+        '    return "\\n".join(lines) + "\\n"\n\n'
+        "def main() -> None:\n"
+        "    maze_path = sys.argv[1] if len(sys.argv) > 1 else None\n"
+        "    payload = benchmark_solvers(maze_path)\n"
+        '    Path("benchmark_results.json").write_text(json.dumps(payload, indent=2) + "\\n", encoding="utf-8")\n'
+        '    Path("BENCHMARK_RESULTS.md").write_text(_build_markdown_report(payload), encoding="utf-8")\n'
+        '    print(json.dumps(payload, indent=2))\n\n'
+        'if __name__ == "__main__":\n'
+        "    main()\n"
+    )
+
+
+def _build_artifact_readme(assignment_artifacts: List[Dict[str, Any]]) -> str:
+    lines = [
+        "# Assignment Artifacts",
+        "",
+        "Downloaded maze files from assignment links are stored here.",
+        "",
+    ]
+    for artifact in assignment_artifacts:
+        lines.append(
+            f"- {artifact.get('path', '').split('/')[-1]}: {artifact.get('source_url', 'unknown source')}"
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _append_maze_benchmark_notes(existing_readme: str, artifact_paths: List[str]) -> str:
+    lines = [
+        "",
+        "## Maze Benchmarking",
+        "- Run `python benchmark_maze.py` to benchmark the generated BFS, DFS, and A* solvers.",
+        "- The script writes machine-readable results to `benchmark_results.json` and a report-ready table to `BENCHMARK_RESULTS.md`.",
+    ]
+    if artifact_paths:
+        lines.append(f"- Benchmarking will prefer the downloaded maze artifact files: {', '.join(artifact_paths)}.")
+    else:
+        lines.append("- If no linked maze artifacts are available, benchmarking falls back to `maze.txt`.")
+    return existing_readme.rstrip() + "\n" + "\n".join(lines) + "\n"
+
+
+def _append_maze_report_notes(existing_report: str) -> str:
+    return existing_report.rstrip() + (
+        "\n\n## Generated Benchmark Workflow\n\n"
+        "Run `python benchmark_maze.py [optional-maze-path]` after you add the official report maze. "
+        "Copy the metrics from `BENCHMARK_RESULTS.md` into the comparison sections above.\n"
+    )
+
+
 def build_assignment_specific_files(
     assignment_name: str,
     assignment_description: str,
@@ -1315,6 +1435,7 @@ def generate_starter_files(
     language: str = "python",
     short_description: str = "",
     course_context: Optional[List[Dict[str, Any]]] = None,
+    assignment_artifacts: Optional[List[Dict[str, Any]]] = None,
 ) -> dict:
     """
     Generate starter files for an assignment.
@@ -1373,8 +1494,32 @@ def generate_starter_files(
         )
     )
 
+    maze_artifacts = list(assignment_artifacts or [])
+
     if language.lower() in {"python", "py"} and "maze_solvers.py" in files:
+        maze_functions = [
+            function_name
+            for function_name in extract_required_function_names(assignment_description)
+            if function_name.startswith("maze_solver_") or function_name in {
+                "maze_solver_one",
+                "maze_solver_two",
+                "maze_solver_three",
+            }
+        ] or ["maze_solver_one", "maze_solver_two", "maze_solver_three"]
+        artifact_paths = [artifact.get("path") for artifact in maze_artifacts if artifact.get("path")]
+        files["benchmark_maze.py"] = _build_maze_benchmark_file(maze_functions, artifact_paths)
         files["README.md"] = _append_maze_readme_notes(files["README.md"])
+        files["README.md"] = _append_maze_benchmark_notes(files["README.md"], artifact_paths)
+        files["Report.md"] = _append_maze_report_notes(files["Report.md"])
+
+        for artifact in maze_artifacts:
+            artifact_path = artifact.get("path")
+            artifact_content = artifact.get("content")
+            if artifact_path and artifact_content:
+                files[artifact_path] = artifact_content
+
+        if maze_artifacts:
+            files["artifacts/README.md"] = _build_artifact_readme(maze_artifacts)
 
     if language.lower() in {"python", "py"}:
         inferred_imports = infer_python_assignment_imports(assignment_description)

tests/test_agent.py

@@ -312,19 +312,33 @@ def test_assignment_specific_scaffold_files(self):
             assignment_description=assignment_description,
             due_date="2026-03-11",
             language="python",
+            assignment_artifacts=[
+                {
+                    'path': 'artifacts/report-maze.txt',
+                    'content': '5 5\nS   X\nXX XX\nX   X\nX XXX\nX   E\n',
+                    'source_url': 'https://example.com/report-maze.txt',
+                }
+            ],
         )
 
         assert "maze_solvers.py" in files
         assert "main.py" in files
+        assert "benchmark_maze.py" in files
         assert "tests/test_maze_solvers.py" in files
+        assert "artifacts/report-maze.txt" in files
+        assert "artifacts/README.md" in files
         assert "def maze_solver_one(maze: str | Sequence[str]) -> str:" in files["maze_solvers.py"]
         assert "def maze_solver_two(maze: str | Sequence[str]) -> str:" in files["maze_solvers.py"]
         assert "def maze_solver_three(maze: str | Sequence[str]) -> str:" in files["maze_solvers.py"]
         assert "breadth-first search" in files["maze_solvers.py"]
         assert "A* search" in files["maze_solvers.py"]
         assert "pytest tests/test_maze_solvers.py" in files["README.md"]
+        assert "python benchmark_maze.py" in files["README.md"]
+        assert "report-maze.txt" in files["benchmark_maze.py"]
+        assert "BENCHMARK_RESULTS.md" in files["benchmark_maze.py"]
         assert "maze.txt" in files
         assert "Report.md" in files
+        assert "Generated Benchmark Workflow" in files["Report.md"]
 
         namespace: dict[str, object] = {}
         exec(files["maze_solvers.py"], namespace)
@@ -723,6 +737,42 @@ def test_search_course_module_context_ranks_relevant_items(self):
         assert len(result) == 1
         assert result[0]['section_title'] == 'Bayes theorem'
 
+    def test_download_assignment_maze_artifacts_from_description_links(self):
+        """Linked maze text files should be downloaded into repo-ready artifact records."""
+        from tools.canvas_tools import CanvasTools
+
+        assignment = {
+            'description': (
+                '<p>Use the following report maze: '
+                '<a href="/courses/123/files/report_maze.txt">Report Maze</a>.</p>'
+            )
+        }
+
+        with patch.dict('os.environ', {
+            'CANVAS_API_URL': 'https://test.canvas.com',
+            'CANVAS_API_TOKEN': 'test_token',
+        }):
+            tools = CanvasTools()
+
+            with patch('tools.canvas_tools.requests.get') as get_mock:
+                response_mock = Mock()
+                response_mock.raise_for_status.return_value = None
+                response_mock.text = '5 5\nS   X\nXX XX\nX   X\nX XXX\nX   E\n'
+                get_mock.return_value = response_mock
+
+                artifacts = tools.download_assignment_maze_artifacts(123, assignment)
+
+        assert artifacts == [
+            {
+                'course_id': 123,
+                'label': 'Report Maze',
+                'path': 'artifacts/report-maze.txt',
+                'content': '5 5\nS   X\nXX XX\nX   X\nX XXX\nX   E\n',
+                'source_url': 'https://test.canvas.com/courses/123/files/report_maze.txt',
+            }
+        ]
+        get_mock.assert_called_once()
+
 
 class TestGitHubTools:
     """Test GitHub tools (mock tests)."""
@@ -846,8 +896,16 @@ def test_run_passes_course_context_to_github_generation(self):
                     "text": "Section: Bayes Review\n\nPosterior is proportional to prior times likelihood.",
                 }
             ]
+            artifacts = [
+                {
+                    'path': 'artifacts/report-maze.txt',
+                    'content': '5 5\nS   X\nXX XX\nX   X\nX XXX\nX   E\n',
+                    'source_url': 'https://example.com/report-maze.txt',
+                }
+            ]
 
             agent.fetch_course_context = AsyncMock(return_value=context)
+            agent.fetch_assignment_artifacts = AsyncMock(return_value=artifacts)
             agent.create_repository_for_assignment = AsyncMock(return_value={
                 "repository": {"name": "posterior-homework", "owner": {"login": "testuser"}},
                 "files_created": ["README.md", "COURSE_CONTEXT.md"],
@@ -866,6 +924,7 @@ def test_run_passes_course_context_to_github_generation(self):
                 assignment,
                 "python",
                 course_context=context,
+                assignment_artifacts=artifacts,
             )
             assert result["course_context"] == context