From d1da1a209ecb2e276a4ccf32da035c70c04df9f3 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 00:52:12 +0530 Subject: [PATCH 1/3] one requirement txt file --- AGENT.md | 6 +++--- CONTRIBUTING.md | 2 +- Dockerfile | 6 +----- README.md | 6 +++--- docs/MCP.md | 2 +- requirements-browser.txt | 2 -- requirements-llm.txt | 4 ---- requirements-mcp.txt | 2 -- requirements-optional.txt | 3 --- requirements.txt | 15 +++++++++++++++ scripts/local-run.ps1 | 4 ---- scripts/local-run.sh | 2 -- scripts/local-test.ps1 | 2 -- scripts/local-test.sh | 1 - src/website_profiling/crawl/fetchers/browser.py | 2 +- .../crawl/fetchers/browser_deps.py | 5 +---- src/website_profiling/llm/agent.py | 4 ++-- src/website_profiling/llm/enrich.py | 2 +- src/website_profiling/llm/providers/anthropic.py | 4 ++-- src/website_profiling/llm/providers/gemini.py | 2 +- src/website_profiling/llm/providers/ollama.py | 2 +- src/website_profiling/llm/providers/openai.py | 4 ++-- src/website_profiling/mcp/server.py | 2 +- .../reporting/optional_audits.py | 2 +- tests/test_crawl_fetchers.py | 14 ++++++++++---- tests/test_fetchers_sitemap_config_unit.py | 3 +-- web/app/api/crawl/browser-status/route.ts | 2 +- web/src/lib/pipelineConfigSchema.ts | 2 +- web/src/strings.json | 4 ++-- 29 files changed, 52 insertions(+), 59 deletions(-) delete mode 100644 requirements-browser.txt delete mode 100644 requirements-llm.txt delete mode 100644 requirements-mcp.txt delete mode 100644 requirements-optional.txt diff --git a/AGENT.md b/AGENT.md index 0a383f85..e59a6fa9 100644 --- a/AGENT.md +++ b/AGENT.md @@ -14,7 +14,7 @@ **Local dev:** `./local-run` (Postgres in Docker `wp-pg`, Next.js on host). See `scripts/local-run.sh`. **Local tests (CI parity):** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%); `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. -**JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: `requirements-browser.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`. +**JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: Playwright from `requirements.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`. **Run / APIs** @@ -25,7 +25,7 @@ - **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline. - **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch. - **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/chat` POST (SSE agent); `/api/chat/sessions` GET/POST; `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run -- **MCP:** `python -m website_profiling.mcp` (stdio, **221 read-only audit tools** + MCP resources). See `docs/MCP.md`. Requires `pip install -r requirements-mcp.txt`. +- **MCP:** `python -m website_profiling.mcp` (stdio, **221 read-only audit tools** + MCP resources). See `docs/MCP.md`. Requires `pip install -r requirements.txt`. - **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages` tables, migration `012_chat_sessions`). - **Job store:** in-memory on `globalThis` in `web/src/server/pipelineJobs.ts` — job status/log is lost on server restart (single-process dev/Docker only). - **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`** @@ -38,7 +38,7 @@ | Report | `reporting/builder.py`, `reporting/categories.py` | | DB schema | `alembic/versions/` | | Local analysis | `analysis/local.py`, `requirements.txt` | -| AI insights (LLM) | `llm/enrich.py`, `llm/agent.py`, `llm_config.py`, `requirements-llm.txt` | +| AI insights (LLM) | `llm/enrich.py`, `llm/agent.py`, `llm_config.py`, `requirements.txt` | | Audit query tools (MCP + chat) | `tools/audit_tools/`, `mcp/server.py`, `commands/chat_cmd.py` | | Config / CLI | `config.py` (`load_config`, `load_config_from_db`), `cli.py`, `input.txt.example` | | UI pipeline schema | `web/src/lib/pipelineConfigSchema.ts` | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c858768..02962479 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,7 +19,7 @@ Thank you for helping improve this project. All contributions are welcome under Details: [README.md](README.md), [AGENT.md](AGENT.md). -JavaScript/auto crawl needs Playwright (`requirements-browser.txt`, installed by `./local-run setup`) and Chromium on `PATH` or `CHROME_PATH`. Unit tests mock the browser fetcher; integration tests use `@pytest.mark.browser` and run in the Docker CI job (`tests/test_crawl_fetchers.py`, `tests/test_crawler_browser_e2e.py`). Locally: `./local-test browser` (skips gracefully if Chromium is missing). +JavaScript/auto crawl needs Playwright (from `requirements.txt`, installed by `./local-run setup`) and Chromium on `PATH` or `CHROME_PATH`. Unit tests mock the browser fetcher; integration tests use `@pytest.mark.browser` and run in the Docker CI job (`tests/test_crawl_fetchers.py`, `tests/test_crawler_browser_e2e.py`). Locally: `./local-test browser` (skips gracefully if Chromium is missing). ## Running tests diff --git a/Dockerfile b/Dockerfile index c063004b..a8222a01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,18 +39,14 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ CHROME_PATH=/usr/bin/chromium \ LIGHTHOUSE_PATH=/usr/local/bin/lighthouse -# Python: base requirements + optional LLM API clients +# Python dependencies COPY requirements.txt /app/requirements.txt -COPY requirements-llm.txt /app/requirements-llm.txt -COPY requirements-browser.txt /app/requirements-browser.txt COPY alembic.ini /app/alembic.ini COPY alembic /app/alembic RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m venv /opt/venv \ && /opt/venv/bin/pip install --upgrade pip \ && /opt/venv/bin/pip install -r /app/requirements.txt \ - && /opt/venv/bin/pip install -r /app/requirements-llm.txt \ - && /opt/venv/bin/pip install -r /app/requirements-browser.txt \ && ln -sf /opt/venv/bin/python /usr/local/bin/python \ && ln -sf /opt/venv/bin/python /usr/local/bin/python3 diff --git a/README.md b/README.md index 4c723c7c..d531cd4f 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ WebsiteProfiling/ ├── Dockerfile # Production image ├── local-run # Dev setup & start script ├── local-test # Full test suite (CI parity) -├── requirements*.txt # Python deps (core, browser, LLM, MCP) +├── requirements.txt # Python dependencies └── pipeline-config.example.txt ``` @@ -173,9 +173,9 @@ Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup and Google Search Console / Analytics: connect via **Integrations** (gear icon) in the app. -**JavaScript crawl (optional):** In Audit settings, set **Crawl rendering** to `javascript` (always headless Chromium) or `auto` (static first, browser when SPA heuristics match). Install locally: `pip install -r requirements-browser.txt` and Chromium on `PATH` or `CHROME_PATH` (included in Docker). The UI preflights via `GET /api/crawl/browser-status` before runs when JS/auto is selected. +**JavaScript crawl (optional):** In Audit settings, set **Crawl rendering** to `javascript` (always headless Chromium) or `auto` (static first, browser when SPA heuristics match). Requires Playwright from `requirements.txt` and Chromium on `PATH` or `CHROME_PATH` (included in Docker). The UI preflights via `GET /api/crawl/browser-status` before runs when JS/auto is selected. -**AI Chat (optional):** Ask questions about your audit data at [http://localhost:3000/chat](http://localhost:3000/chat). Enable a provider under **Run audit → AI settings** (`llm_enabled`, provider, model). `./local-run setup` installs `requirements-llm.txt` (`httpx`, OpenAI, Anthropic SDKs). +**AI Chat (optional):** Ask questions about your audit data at [http://localhost:3000/chat](http://localhost:3000/chat). Enable a provider under **Run audit → AI settings** (`llm_enabled`, provider, model). `./local-run setup` installs all Python deps from `requirements.txt` (including `httpx`, OpenAI, and Anthropic SDKs). | Provider | Notes | |----------|--------| diff --git a/docs/MCP.md b/docs/MCP.md index 4b0424bf..2458bfa0 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -5,7 +5,7 @@ Read-only [Model Context Protocol](https://modelcontextprotocol.io) tools for qu ## Install ```bash -pip install -r requirements-mcp.txt +pip install -r requirements.txt export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling export PYTHONPATH=src ``` diff --git a/requirements-browser.txt b/requirements-browser.txt deleted file mode 100644 index 4415efe0..00000000 --- a/requirements-browser.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Optional: JavaScript rendering crawl (headless Chromium via Playwright) -playwright>=1.49.0 diff --git a/requirements-llm.txt b/requirements-llm.txt deleted file mode 100644 index 6800c042..00000000 --- a/requirements-llm.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Optional LLM providers for AI enrichment (configure via web UI AI tab only) -httpx>=0.27.0 -openai>=1.0.0 -anthropic>=0.25.0 diff --git a/requirements-mcp.txt b/requirements-mcp.txt deleted file mode 100644 index 8a09cd6f..00000000 --- a/requirements-mcp.txt +++ /dev/null @@ -1,2 +0,0 @@ -# MCP server for Cursor / Claude Desktop (optional) -mcp>=1.0.0 diff --git a/requirements-optional.txt b/requirements-optional.txt deleted file mode 100644 index 488a3046..00000000 --- a/requirements-optional.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Optional audit dependencies (install when enabling spell-check / HTML validation extras) -pyspellchecker>=0.8.1 -html5lib>=1.1 diff --git a/requirements.txt b/requirements.txt index d91a8c36..9138a6e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,21 @@ alembic>=1.13 # Audit export (PDF) reportlab>=4.0.0 +# JavaScript rendering crawl (headless Chromium via Playwright) +playwright>=1.49.0 + +# LLM providers for AI enrichment (configure via web UI AI tab) +httpx>=0.27.0 +openai>=1.0.0 +anthropic>=0.25.0 + +# Spell-check / HTML validation extras +pyspellchecker>=0.8.1 +html5lib>=1.1 + +# MCP server for Cursor / Claude Desktop +mcp>=1.0.0 + # Dev / test pytest>=7.0.0 pytest-cov>=5.0.0 diff --git a/scripts/local-run.ps1 b/scripts/local-run.ps1 index 95b26265..8a243cf0 100644 --- a/scripts/local-run.ps1 +++ b/scripts/local-run.ps1 @@ -181,10 +181,6 @@ function Invoke-Venv { Write-Log "Installing Python dependencies" & $VENV_PIP install -q -r (Join-Path $ROOT "requirements.txt") Assert-LastExitCode "Failed to install requirements.txt" - & $VENV_PIP install -q -r (Join-Path $ROOT "requirements-browser.txt") - Assert-LastExitCode "Failed to install requirements-browser.txt" - & $VENV_PIP install -q -r (Join-Path $ROOT "requirements-llm.txt") - Assert-LastExitCode "Failed to install requirements-llm.txt" } function Invoke-Migrate { diff --git a/scripts/local-run.sh b/scripts/local-run.sh index ca13e8bd..e9e79df5 100755 --- a/scripts/local-run.sh +++ b/scripts/local-run.sh @@ -83,8 +83,6 @@ cmd_venv() { fi log "Installing Python dependencies" "$VENV/bin/pip" install -q -r "$ROOT/requirements.txt" - "$VENV/bin/pip" install -q -r "$ROOT/requirements-browser.txt" - "$VENV/bin/pip" install -q -r "$ROOT/requirements-llm.txt" } cmd_migrate() { diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1 index b201d11b..ef9c7297 100644 --- a/scripts/local-test.ps1 +++ b/scripts/local-test.ps1 @@ -184,8 +184,6 @@ function Invoke-Venv { Write-Log "Installing Python dependencies" & $VENV_PIP install -q -r (Join-Path $ROOT "requirements.txt") Assert-LastExitCode "Failed to install requirements.txt" - & $VENV_PIP install -q -r (Join-Path $ROOT "requirements-browser.txt") - Assert-LastExitCode "Failed to install requirements-browser.txt" } } diff --git a/scripts/local-test.sh b/scripts/local-test.sh index 56a0e627..3d32f7e5 100755 --- a/scripts/local-test.sh +++ b/scripts/local-test.sh @@ -84,7 +84,6 @@ cmd_venv() { if [[ ! -x "$VENV/bin/pytest" ]]; then log "Installing Python dependencies" "$VENV/bin/pip" install -q -r "$ROOT/requirements.txt" - "$VENV/bin/pip" install -q -r "$ROOT/requirements-browser.txt" fi } diff --git a/src/website_profiling/crawl/fetchers/browser.py b/src/website_profiling/crawl/fetchers/browser.py index 61c7e3c0..b3c3678f 100644 --- a/src/website_profiling/crawl/fetchers/browser.py +++ b/src/website_profiling/crawl/fetchers/browser.py @@ -15,7 +15,7 @@ _BROWSER_INSTALL_MSG = ( "JavaScript crawl requires Playwright and Chromium. Install: " - "pip install -r requirements-browser.txt. " + "pip install -r requirements.txt. " "Chrome or Chromium must be available (set CHROME_PATH if needed)." ) diff --git a/src/website_profiling/crawl/fetchers/browser_deps.py b/src/website_profiling/crawl/fetchers/browser_deps.py index 055a5ea0..048e4163 100644 --- a/src/website_profiling/crawl/fetchers/browser_deps.py +++ b/src/website_profiling/crawl/fetchers/browser_deps.py @@ -67,11 +67,8 @@ def browser_status() -> dict[str, str | bool]: def _pip_install_browser_requirements() -> None: - req = _repo_root() / "requirements-browser.txt" - if not req.is_file(): - raise RuntimeError(f"Missing {req.name}; cannot auto-install Playwright.") subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", "-r", str(req)], + [sys.executable, "-m", "pip", "install", "-q", "playwright>=1.49.0"], check=True, cwd=_repo_root(), ) diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py index 21868726..9194ddae 100644 --- a/src/website_profiling/llm/agent.py +++ b/src/website_profiling/llm/agent.py @@ -185,9 +185,9 @@ def on_token(text: str) -> None: result = _react_step(client, llm_messages, _tools_description(compact=True), on_token) except Exception as e: msg = str(e).strip() or type(e).__name__ - if "httpx" in msg.lower() or "requirements-llm" in msg.lower(): + if "httpx" in msg.lower() or "requirements.txt" in msg.lower(): msg = ( - "LLM dependencies are missing. Run: pip install -r requirements-llm.txt " + "LLM dependencies are missing. Run: pip install -r requirements.txt " f"(or restart with ./local-run setup). Details: {msg}" ) _emit(on_event, {"type": "error", "message": msg}) diff --git a/src/website_profiling/llm/enrich.py b/src/website_profiling/llm/enrich.py index 8ff08af6..b2f068bc 100644 --- a/src/website_profiling/llm/enrich.py +++ b/src/website_profiling/llm/enrich.py @@ -22,7 +22,7 @@ SIMILAR_SYSTEM, ) -LLM_INSTALL_HINT = "Install LLM dependencies: pip install -r requirements-llm.txt" +LLM_INSTALL_HINT = "Install LLM dependencies: pip install -r requirements.txt" def _cfg_bool(cfg: dict[str, str] | None, key: str, default: bool = False) -> bool: diff --git a/src/website_profiling/llm/providers/anthropic.py b/src/website_profiling/llm/providers/anthropic.py index e41b8b24..c352f378 100644 --- a/src/website_profiling/llm/providers/anthropic.py +++ b/src/website_profiling/llm/providers/anthropic.py @@ -20,7 +20,7 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]: try: import anthropic except ImportError as e: - raise ImportError("pip install anthropic (or requirements-llm.txt)") from e + raise ImportError("pip install -r requirements.txt") from e client = anthropic.Anthropic(api_key=self._api_key, timeout=self._timeout) msg = client.messages.create( @@ -47,7 +47,7 @@ def chat_with_tools( try: import anthropic except ImportError as e: - raise ImportError("pip install anthropic (or requirements-llm.txt)") from e + raise ImportError("pip install -r requirements.txt") from e system_parts: list[str] = [] anthropic_messages: list[dict[str, Any]] = [] diff --git a/src/website_profiling/llm/providers/gemini.py b/src/website_profiling/llm/providers/gemini.py index 08b839b1..c692d8b1 100644 --- a/src/website_profiling/llm/providers/gemini.py +++ b/src/website_profiling/llm/providers/gemini.py @@ -18,7 +18,7 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]: try: import httpx except ImportError as e: - raise ImportError("pip install httpx (or requirements-llm.txt)") from e + raise ImportError("pip install -r requirements.txt") from e url = f"https://generativelanguage.googleapis.com/v1beta/models/{self._model}:generateContent" payload = { diff --git a/src/website_profiling/llm/providers/ollama.py b/src/website_profiling/llm/providers/ollama.py index acfafc65..94b599c5 100644 --- a/src/website_profiling/llm/providers/ollama.py +++ b/src/website_profiling/llm/providers/ollama.py @@ -112,7 +112,7 @@ def _client(self): try: import httpx except ImportError as e: - raise ImportError("pip install httpx (or requirements-llm.txt)") from e + raise ImportError("pip install -r requirements.txt") from e return httpx.Client(timeout=self._timeout) def _raise_for_status(self, response: Any) -> None: diff --git a/src/website_profiling/llm/providers/openai.py b/src/website_profiling/llm/providers/openai.py index 6619de2b..29170eb1 100644 --- a/src/website_profiling/llm/providers/openai.py +++ b/src/website_profiling/llm/providers/openai.py @@ -21,7 +21,7 @@ def complete_json(self, system: str, user: str) -> dict[str, Any]: try: import httpx except ImportError as e: - raise ImportError("pip install httpx (or requirements-llm.txt)") from e + raise ImportError("pip install -r requirements.txt") from e payload = { "model": self._model, @@ -53,7 +53,7 @@ def chat_with_tools( try: import httpx except ImportError as e: - raise ImportError("pip install httpx (or requirements-llm.txt)") from e + raise ImportError("pip install -r requirements.txt") from e payload: dict[str, Any] = { "model": self._model, diff --git a/src/website_profiling/mcp/server.py b/src/website_profiling/mcp/server.py index ded725f3..5b150e61 100644 --- a/src/website_profiling/mcp/server.py +++ b/src/website_profiling/mcp/server.py @@ -199,7 +199,7 @@ def main() -> None: from mcp.types import Resource, TextContent, Tool except ImportError as e: raise SystemExit( - "MCP SDK not installed. Run: pip install -r requirements-mcp.txt", + "MCP SDK not installed. Run: pip install -r requirements.txt", ) from e server = Server("site-audit") diff --git a/src/website_profiling/reporting/optional_audits.py b/src/website_profiling/reporting/optional_audits.py index 3867cb72..32c79f10 100644 --- a/src/website_profiling/reporting/optional_audits.py +++ b/src/website_profiling/reporting/optional_audits.py @@ -100,7 +100,7 @@ def spell_check_issues(df: pd.DataFrame, *, max_pages: int = 50) -> tuple[list[d try: from spellchecker import SpellChecker # type: ignore[import-untyped] except ImportError: - return issues, "pyspellchecker not installed (pip install -r requirements-optional.txt)" + return issues, "pyspellchecker not installed (pip install -r requirements.txt)" spell = SpellChecker() checked = 0 for _, row in df.iterrows(): diff --git a/tests/test_crawl_fetchers.py b/tests/test_crawl_fetchers.py index 6df58015..e5998097 100644 --- a/tests/test_crawl_fetchers.py +++ b/tests/test_crawl_fetchers.py @@ -254,7 +254,7 @@ def test_repo_root_defaults_to_project_root(monkeypatch): monkeypatch.delenv("WEBSITE_PROFILING_ROOT", raising=False) root = browser_deps._repo_root() - assert (root / "requirements-browser.txt").is_file() + assert (root / "requirements.txt").is_file() def test_playwright_chromium_unavailable_without_playwright(monkeypatch): @@ -345,12 +345,18 @@ def test_ensure_browser_deps_returns_ok_without_install(monkeypatch): assert status["ok"] is True -def test_pip_install_browser_requirements_missing_file(monkeypatch, tmp_path): +def test_pip_install_browser_requirements_runs_playwright_install(monkeypatch, tmp_path): from website_profiling.crawl.fetchers import browser_deps monkeypatch.setenv("WEBSITE_PROFILING_ROOT", str(tmp_path)) - with pytest.raises(RuntimeError, match="Missing requirements-browser.txt"): - browser_deps._pip_install_browser_requirements() + called: list = [] + + def fake_run(cmd, **kwargs): + called.append(cmd) + + monkeypatch.setattr(browser_deps.subprocess, "run", fake_run) + browser_deps._pip_install_browser_requirements() + assert called and "playwright>=1.49.0" in called[0] def test_ensure_browser_deps_reports_auto_install_failure(monkeypatch): diff --git a/tests/test_fetchers_sitemap_config_unit.py b/tests/test_fetchers_sitemap_config_unit.py index 9cb09d1f..d224b1d9 100644 --- a/tests/test_fetchers_sitemap_config_unit.py +++ b/tests/test_fetchers_sitemap_config_unit.py @@ -298,7 +298,6 @@ def close(self): def test_pip_install_browser_requirements_runs_subprocess(monkeypatch, tmp_path): from website_profiling.crawl.fetchers import browser_deps - (tmp_path / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8") monkeypatch.setenv("WEBSITE_PROFILING_ROOT", str(tmp_path)) called: list = [] @@ -307,7 +306,7 @@ def fake_run(cmd, **kwargs): monkeypatch.setattr(browser_deps.subprocess, "run", fake_run) browser_deps._pip_install_browser_requirements() - assert called and "requirements-browser.txt" in called[0][-1] + assert called and "playwright>=1.49.0" in called[0] def test_playwright_install_chromium_runs_subprocess(monkeypatch, tmp_path): diff --git a/web/app/api/crawl/browser-status/route.ts b/web/app/api/crawl/browser-status/route.ts index 53860c7a..87725239 100644 --- a/web/app/api/crawl/browser-status/route.ts +++ b/web/app/api/crawl/browser-status/route.ts @@ -61,7 +61,7 @@ export const GET: ApiRouteHandler = async (request): Promise => { ok: false, message: stderr.trim() || - 'JavaScript crawl requires Playwright and Chromium. Install: pip install -r requirements-browser.txt.', + 'JavaScript crawl requires Playwright and Chromium. Install: pip install -r requirements.txt.', error: stderr.trim() || `exit ${code}`, }); return; diff --git a/web/src/lib/pipelineConfigSchema.ts b/web/src/lib/pipelineConfigSchema.ts index 2362595c..ca1f0144 100644 --- a/web/src/lib/pipelineConfigSchema.ts +++ b/web/src/lib/pipelineConfigSchema.ts @@ -19,7 +19,7 @@ import { crawlRenderModeUsesBrowser } from '@/lib/browserCrawlStatus'; import type { PipelineConfigState } from '@/types/api'; export const BROWSER_CRAWL_UNAVAILABLE_MSG = - 'JavaScript crawl requires Playwright and Chromium. Install: pip install -r requirements-browser.txt. Chrome or Chromium must be on PATH or set CHROME_PATH.'; + 'JavaScript crawl requires Playwright and Chromium. Install: pip install -r requirements.txt. Chrome or Chromium must be on PATH or set CHROME_PATH.'; export interface PipelineConfigField { key: string; diff --git a/web/src/strings.json b/web/src/strings.json index c523f88e..1b5010e7 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -211,7 +211,7 @@ "settingsSubtitle": "Fine-tune crawl, audit report, Lighthouse, keywords, and AI options.", "settingsSaveWhileRunningHint": "A job is running; saved settings apply to the next run.", "browserCrawlBannerTitle": "Headless browser not available", - "browserCrawlBannerHint": "JavaScript and Auto crawl modes need Playwright Python packages and Chromium. Run: pip install -r requirements-browser.txt. Ensure Chrome or Chromium is on PATH or set CHROME_PATH.", + "browserCrawlBannerHint": "JavaScript and Auto crawl modes need Playwright Python packages and Chromium. Run: pip install -r requirements.txt. Ensure Chrome or Chromium is on PATH or set CHROME_PATH.", "browserCrawlChecking": "Checking browser availability…", "saveSettings": "Save settings", "saveAndClose": "Save & close", @@ -2637,7 +2637,7 @@ "writing": "Writing response…", "toolStatus": "Running {name}…", "elapsed": "{seconds}s", - "emptyResponse": "The assistant returned no response. Check AI settings, Ollama connection, and that LLM dependencies are installed (pip install -r requirements-llm.txt).", + "emptyResponse": "The assistant returned no response. Check AI settings, Ollama connection, and that LLM dependencies are installed (pip install -r requirements.txt).", "responseFailed": "Response failed", "emptyHeadline": "Ready when you are.", "emptySubline": "Ask about issues, crawl data, Core Web Vitals, or Search Console metrics.", From c907bd9046782c0d3e97bbe097986b68e71af109 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 00:57:45 +0530 Subject: [PATCH 2/3] Failing ci --- tests/test_mcp_server_helpers.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py index 4e5d3749..d092518b 100644 --- a/tests/test_mcp_server_helpers.py +++ b/tests/test_mcp_server_helpers.py @@ -190,9 +190,10 @@ def test_mcp_package_main(monkeypatch) -> None: def test_mcp_server_main_guard() -> None: # run_module executes __main__ in a fresh import; drop any prior import from this file. sys.modules.pop("website_profiling.mcp.server", None) - with pytest.raises(SystemExit, match="MCP SDK"): - runpy.run_module( - "website_profiling.mcp.server", - run_name="__main__", - alter_sys=False, - ) + with patch.dict(sys.modules, {"mcp.server": None, "mcp.server.stdio": None, "mcp.types": None}): + with pytest.raises(SystemExit, match="MCP SDK"): + runpy.run_module( + "website_profiling.mcp.server", + run_name="__main__", + alter_sys=False, + ) From f16360e1a74ecd4c01fb8432da9ecc2b3ab32707 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 01:07:54 +0530 Subject: [PATCH 3/3] updated --- tests/test_optional_audits.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/test_optional_audits.py b/tests/test_optional_audits.py index 99dfd643..f3830275 100644 --- a/tests/test_optional_audits.py +++ b/tests/test_optional_audits.py @@ -2,13 +2,17 @@ from __future__ import annotations import json +import sys +from unittest.mock import patch import pandas as pd from website_profiling.reporting.optional_audits import ( amp_audit_issues, apply_optional_audits, + html_validation_issues, pagination_issues, + spell_check_issues, ) @@ -38,12 +42,37 @@ def test_amp_audit_missing_canonical(): assert len(issues) == 1 -def test_apply_optional_audits_spell_skipped_without_package(): +def test_spell_check_issues_missing_pyspellchecker(): + df = pd.DataFrame([ + { + "url": "https://example.com/typo", + "status": "200", + "content_excerpt": "This sentense has many misspelled wrds that should trigger heuristics.", + }, + ]) + with patch.dict(sys.modules, {"spellchecker": None}): + issues, skip = spell_check_issues(df) + assert issues == [] + assert skip and "pyspellchecker" in skip + + +def test_html_validation_issues_missing_html5lib(): + html = "A" + ("x" * 120) + "" + df = pd.DataFrame([{"url": "https://example.com", "html": html}]) + with patch.dict(sys.modules, {"html5lib": None}): + issues, use_parser = html_validation_issues(df) + assert use_parser is False + assert isinstance(issues, list) + + +def test_apply_optional_audits_spell_skipped_without_package(capsys): categories = [ {"id": "technical_seo", "name": "Technical", "issues": [], "recommendations": []}, {"id": "intelligence", "name": "Content", "issues": [], "recommendations": []}, {"id": "html_accessibility", "name": "A11y", "issues": [], "recommendations": []}, ] df = pd.DataFrame([{"url": "https://example.com", "status": "200", "page_analysis": "{}"}]) - meta = apply_optional_audits(categories, df, {"enable_spell_check": "true"}) - assert isinstance(meta, dict) + with patch.dict(sys.modules, {"spellchecker": None}): + meta = apply_optional_audits(categories, df, {"enable_spell_check": "true"}) + assert meta["spell_check_skipped"] == "pyspellchecker not installed (pip install -r requirements.txt)" + assert "pyspellchecker" in capsys.readouterr().err