diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9a4389d5..134057fa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -43,6 +43,8 @@ jobs:
tests/test_indexation_coverage.py tests/test_crawl_segments.py \
tests/test_terminology.py tests/test_compare_payload.py \
tests/test_optional_audits.py tests/test_property_profile.py tests/test_reporting_gaps.py \
+ tests/test_text_content_analysis.py tests/test_builder_image_buckets.py \
+ tests/test_pipeline_report_pool_unit.py tests/test_reporting_builder_modules.py \
--cov=website_profiling.reporting --cov-config=.coveragerc.reporting \
--cov-report=term-missing --cov-fail-under=100 -q -o addopts=
- name: Pytest (tools coverage gate)
@@ -51,11 +53,13 @@ jobs:
tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_audit_tools_expanded.py \
tests/test_audit_tools_coverage.py tests/test_audit_tools_dispatch_coverage.py \
tests/test_audit_tools_links_extras.py tests/test_audit_tools_expansion.py \
- tests/test_audit_tools_expansion_coverage.py tests/test_export_custom_coverage.py \
+ tests/test_audit_tools_expansion_coverage.py tests/test_audit_tools_batch100_coverage.py tests/test_export_custom_coverage.py \
tests/test_export_artifacts_coverage.py tests/test_export_compare_coverage.py \
tests/test_export_tools_coverage.py tests/test_image_tools.py tests/test_export_custom.py \
tests/test_export_artifacts.py tests/test_export_compare.py tests/test_export_workbook.py \
- tests/test_export_sitemap.py tests/test_mcp_registry.py tests/test_mcp_resources.py \
+ tests/test_export_sitemap.py tests/test_mcp_registry.py tests/test_mcp_resources.py \
+ tests/test_router_tools.py tests/test_tool_selector.py \
+ tests/test_tools_gate100_coverage.py \
tests/test_tools_branch_coverage.py \
--cov=website_profiling.tools --cov-config=.coveragerc.tools \
--cov-report=term-missing --cov-fail-under=100 -q -o addopts=
diff --git a/AGENT.md b/AGENT.md
index e59a6fa9..ec06cbe5 100644
--- a/AGENT.md
+++ b/AGENT.md
@@ -25,7 +25,7 @@
- **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline.
- **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch.
- **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/chat` POST (SSE agent); `/api/chat/sessions` GET/POST; `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run
-- **MCP:** `python -m website_profiling.mcp` (stdio, **221 read-only audit tools** + MCP resources). See `docs/MCP.md`. Requires `pip install -r requirements.txt`.
+- **MCP:** `python -m website_profiling.mcp` (stdio, **340 read-only audit tools**, domain-scoped via `WP_MCP_DOMAIN`). See `docs/MCP.md`. Requires `pip install -r requirements.txt`.
- **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages` tables, migration `012_chat_sessions`).
- **Job store:** in-memory on `globalThis` in `web/src/server/pipelineJobs.ts` — job status/log is lost on server restart (single-process dev/Docker only).
- **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`**
diff --git a/README.md b/README.md
index d531cd4f..9059ecc6 100644
--- a/README.md
+++ b/README.md
@@ -73,7 +73,7 @@ Repository: [codefrydev/WebsiteProfiling](https://github.com/codefrydev/WebsiteP
-Also included: **AI chat** over audit data (optional), **221 MCP tools**, keyword explorer, backlinks, compare runs, and portfolio management for agencies.
+Also included: **AI chat** over audit data (optional), **340 MCP tools** (domain-scoped servers), keyword explorer, backlinks, compare runs, and portfolio management for agencies.
{html_module.escape(str(f.get('id', '')))} {html_module.escape(str(f.get('impact', '')))} {html_module.escape(str(f.get('helpText', ''))[:80])}... None {os.path.basename(p)} " for p in raw_reports[:5])
+ return f"""
+
+Lighthouse Report
+
+Lighthouse Report
+URL: {url}
+Median metrics
+
+Metric Value LCP (ms) {mm.get('lcp_ms') or '—'} CLS {mm.get('cls') or '—'} TBT (ms) {mm.get('tbt_ms') or '—'} FCP (ms) {mm.get('fcp_ms') or '—'}
+Category scores (0–100)
+
+Category Score performance {cs.get('performance') or '—'} accessibility {cs.get('accessibility') or '—'} best-practices {cs.get('best-practices') or '—'} seo {cs.get('seo') or '—'} pwa {cs.get('pwa') or '—'}
+Top failures
+Audit Impact Help
+Artifacts
+summary.json | human_summary.txt | diagnostics.json
+Raw runs: {raw_links or '—'}
+
+
+"""
+
+
+def _write_report_html(output_dir: str, summary: dict[str, Any]) -> None:
+ """Write report.html to output_dir (used when not using DB)."""
+ content = summary.get("report_html") or _build_report_html_content(summary)
+ report_path = os.path.join(output_dir, "report.html")
+ with open(report_path, "w", encoding="utf-8") as f:
+ f.write(content)
+
+
+def _url_safe(s: str) -> str:
+ """Return a filesystem-safe slug from URL for filenames."""
+ return re.sub(r"[^\w\-.]", "_", s.strip().rstrip("/"))[:80]
+
+
+def _lighthouse_cmd() -> list[str]:
+ """Return argv prefix: [resolved lighthouse] or [resolved npx, -y, lighthouse]. Paths from shutil.which (portable)."""
+ explicit = (os.environ.get("LIGHTHOUSE_PATH") or os.environ.get("LIGHTHOUSE_BIN") or "").strip()
+ if explicit and os.path.isfile(explicit) and os.access(explicit, os.X_OK):
+ return [explicit]
+ lh = shutil.which("lighthouse")
+ if lh is not None:
+ return [lh]
+ npx = shutil.which("npx")
+ if npx is not None:
+ return [npx, "-y", "lighthouse"]
+ raise RuntimeError(_LIGHTHOUSE_INSTALL_MSG)
+
+
+def _uses_npx(cmd: list[str]) -> bool:
+ base = os.path.basename(cmd[0]).lower()
+ return base in ("npx", "npx.cmd")
+
+
+def is_lighthouse_available() -> bool:
+ """Return True if lighthouse or npx is on PATH (so we can run Lighthouse)."""
+ try:
+ _lighthouse_cmd()
+ return True
+ except RuntimeError:
+ return False
+
+
+def _preset_for_strategy(strategy: str) -> str:
+ """Map user strategy 'mobile'|'desktop' to Lighthouse CLI preset. Newer Lighthouse only accepts perf, experimental, desktop."""
+ s = (strategy or "mobile").lower()
+ if s == "desktop":
+ return "desktop"
+ return "perf" # mobile -> perf (mobile-like throttling in current Lighthouse)
+
+
+# Valid Lighthouse category IDs for --only-categories
+LIGHTHOUSE_CATEGORY_IDS = {"performance", "accessibility", "best-practices", "seo", "pwa"}
+
+
+def _parse_categories(categories: str | list[str] | None) -> list[str] | None:
+ """Return list of valid category IDs, or None to run all categories."""
+ if categories is None:
+ return None
+ if isinstance(categories, str):
+ categories = [c.strip().lower() for c in categories.split(",") if c.strip()]
+ if not categories:
+ return None
+ out = [c for c in categories if c in LIGHTHOUSE_CATEGORY_IDS]
+ return out if out else None
+
diff --git a/src/website_profiling/lighthouse/result_parser.py b/src/website_profiling/lighthouse/result_parser.py
new file mode 100644
index 00000000..f3f195d5
--- /dev/null
+++ b/src/website_profiling/lighthouse/result_parser.py
@@ -0,0 +1,107 @@
+"""Parse Lighthouse JSON output into summary metrics."""
+from __future__ import annotations
+
+from typing import Any
+
+import statistics
+
+def _evidence_from_audit(audit: dict[str, Any]) -> list[str]:
+ """Extract resource URLs or selectors from audit details."""
+ evidence: list[str] = []
+ details = audit.get("details")
+ if not details or not isinstance(details, dict):
+ return evidence
+ items = details.get("items") or details.get("nodes") or []
+ if not isinstance(items, list):
+ return evidence
+ for item in items[:5]:
+ if isinstance(item, dict):
+ url = item.get("url")
+ if url and isinstance(url, str) and not str(url).startswith("data:"):
+ evidence.append(str(url)[:500])
+ node = item.get("node")
+ if isinstance(node, dict) and node.get("selector"):
+ evidence.append(str(node["selector"])[:200])
+ if item.get("selector"):
+ evidence.append(str(item["selector"])[:200])
+ return evidence[:15]
+
+
+def extract_from_lighthouse_json(data: dict) -> dict[str, Any]:
+ """Extract LCP, CLS, TBT, FCP, Speed Index, category scores (all 5), and top 10 failing audits with impact and evidence."""
+ out: dict[str, Any] = {
+ "lcp_ms": None,
+ "cls": None,
+ "tbt_ms": None,
+ "fcp_ms": None,
+ "speed_index_ms": None,
+ "performance_score": None,
+ "accessibility_score": None,
+ "seo_score": None,
+ "best_practices_score": None,
+ "pwa_score": None,
+ "category_scores": {},
+ "top_failures": [],
+ }
+ lr = data.get("lighthouseResult") or data
+ audits = lr.get("audits") or {}
+ cats = lr.get("categories") or {}
+
+ for audit_id, key in [
+ ("largest-contentful-paint", "lcp_ms"),
+ ("cumulative-layout-shift", "cls"),
+ ("total-blocking-time", "tbt_ms"),
+ ("first-contentful-paint", "fcp_ms"),
+ ("speed-index", "speed_index_ms"),
+ ]:
+ a = audits.get(audit_id)
+ if a is not None and "numericValue" in a:
+ out[key] = a["numericValue"]
+
+ for cat_id, key in [
+ ("performance", "performance_score"),
+ ("accessibility", "accessibility_score"),
+ ("seo", "seo_score"),
+ ("best-practices", "best_practices_score"),
+ ("pwa", "pwa_score"),
+ ]:
+ c = cats.get(cat_id)
+ if c is not None and "score" in c:
+ s = c["score"]
+ out[key] = s
+ out["category_scores"][cat_id] = round((s * 100)) if s is not None else None
+
+ # Resolve impact from warning_mapper for each failure
+ from ..tools.warnings import resolve_impact
+ failures = []
+ for aid, a in audits.items():
+ if a is None:
+ continue
+ score = a.get("score")
+ if score is None:
+ continue
+ if score < 1:
+ title = a.get("title") or aid
+ help_text = a.get("helpText") or ""
+ impact = resolve_impact(aid, title, help_text)
+ evidence = _evidence_from_audit(a)
+ failures.append({
+ "id": aid,
+ "score": score,
+ "helpText": help_text,
+ "impact": impact,
+ "evidence": evidence,
+ })
+ failures.sort(key=lambda x: (x["score"] or 0))
+ out["top_failures"] = failures[:10]
+
+ return out
+
+
+def median_or_none(values: list[float]) -> float | None:
+ """Return median of list; None if empty or all None."""
+ clean = [v for v in values if v is not None]
+ if not clean:
+ return None
+ return statistics.median(clean)
+
diff --git a/src/website_profiling/lighthouse/runner.py b/src/website_profiling/lighthouse/runner.py
index ee79e736..3b0d0ca7 100644
--- a/src/website_profiling/lighthouse/runner.py
+++ b/src/website_profiling/lighthouse/runner.py
@@ -1,68 +1,38 @@
"""
Run Lighthouse locally via CLI for a given URL; return machine-readable summary with median metrics.
-Writes raw_runs/, summary.json, diagnostics.json, human_summary.txt, and optionally report.html.
-Uses global lighthouse if on PATH (or LIGHTHOUSE_PATH), otherwise runs via npx (serialized to avoid cache races).
-Requires: Node + npm, Chrome/Chromium.
"""
+from __future__ import annotations
+
import json
import os
import re
import shutil
-import statistics
import subprocess
import sys
-import threading
from datetime import datetime, timezone
-from pathlib import Path
from typing import Any
from ..console_io import console_print
-
-# Lighthouse "good" thresholds for human summary
-LCP_GOOD_MS = 2500
-CLS_GOOD = 0.1
-TBT_GOOD_MS = 200
-FCP_GOOD_MS = 1800
-
-_LIGHTHOUSE_INSTALL_MSG = (
- "Lighthouse not found. Install Node/npm (https://nodejs.org), then run: npm install -g lighthouse. "
- "Chrome or Chromium is also required for headless mode."
+from .config import (
+ CLS_GOOD,
+ FCP_GOOD_MS,
+ LCP_GOOD_MS,
+ TBT_GOOD_MS,
+ _LIGHTHOUSE_INSTALL_MSG,
+ _LIGHTHOUSE_FLOW_MODES,
+ _NPX_LIGHTHOUSE_LOCK,
+ _lighthouse_cmd,
+ _lighthouse_flow_script,
+ _node_cmd,
+ _normalize_lighthouse_mode,
+ _parse_categories,
+ _preset_for_strategy,
+ _repo_root,
+ _url_safe,
+ _uses_npx,
+ is_lighthouse_available,
)
-
-# Serialise npx-on-demand installs — parallel npx runs corrupt /root/.npm/_npx cache in Docker.
-_NPX_LIGHTHOUSE_LOCK = threading.Lock()
-
-_LIGHTHOUSE_FLOW_MODES = frozenset({"snapshot", "timespan"})
-
-
-def _repo_root() -> str:
- explicit = (os.environ.get("WEBSITE_PROFILING_ROOT") or "").strip()
- if explicit:
- return explicit
- return str(Path(__file__).resolve().parents[3])
-
-
-def _lighthouse_flow_script() -> str:
- return os.path.join(_repo_root(), "scripts", "lighthouse_user_flow.mjs")
-
-
-def _normalize_lighthouse_mode(mode: str | None) -> str:
- m = (mode or "navigation").strip().lower() or "navigation"
- if m not in ("navigation", "snapshot", "timespan"):
- raise RuntimeError(
- f"Invalid lighthouse_mode {m!r}; use navigation, snapshot, or timespan."
- )
- return m
-
-
-def _node_cmd() -> str:
- node = shutil.which("node")
- if node is None:
- raise RuntimeError(
- "Node.js not found. Install Node.js (https://nodejs.org) for Lighthouse user flows."
- )
- return node
-
+from .result_parser import _evidence_from_audit, extract_from_lighthouse_json, median_or_none
def _build_report_html_content(summary: dict[str, Any]) -> str:
"""Build report.html content (for DB or file). Returns HTML string."""
@@ -272,108 +242,6 @@ def run_lighthouse_once(
except FileNotFoundError as e:
raise RuntimeError(_LIGHTHOUSE_INSTALL_MSG) from e
-
-def _evidence_from_audit(audit: dict[str, Any]) -> list[str]:
- """Extract resource URLs or selectors from audit details."""
- evidence: list[str] = []
- details = audit.get("details")
- if not details or not isinstance(details, dict):
- return evidence
- items = details.get("items") or details.get("nodes") or []
- if not isinstance(items, list):
- return evidence
- for item in items[:5]:
- if isinstance(item, dict):
- url = item.get("url")
- if url and isinstance(url, str) and not str(url).startswith("data:"):
- evidence.append(str(url)[:500])
- node = item.get("node")
- if isinstance(node, dict) and node.get("selector"):
- evidence.append(str(node["selector"])[:200])
- if item.get("selector"):
- evidence.append(str(item["selector"])[:200])
- return evidence[:15]
-
-
-def extract_from_lighthouse_json(data: dict) -> dict[str, Any]:
- """Extract LCP, CLS, TBT, FCP, Speed Index, category scores (all 5), and top 10 failing audits with impact and evidence."""
- out: dict[str, Any] = {
- "lcp_ms": None,
- "cls": None,
- "tbt_ms": None,
- "fcp_ms": None,
- "speed_index_ms": None,
- "performance_score": None,
- "accessibility_score": None,
- "seo_score": None,
- "best_practices_score": None,
- "pwa_score": None,
- "category_scores": {},
- "top_failures": [],
- }
- lr = data.get("lighthouseResult") or data
- audits = lr.get("audits") or {}
- cats = lr.get("categories") or {}
-
- for audit_id, key in [
- ("largest-contentful-paint", "lcp_ms"),
- ("cumulative-layout-shift", "cls"),
- ("total-blocking-time", "tbt_ms"),
- ("first-contentful-paint", "fcp_ms"),
- ("speed-index", "speed_index_ms"),
- ]:
- a = audits.get(audit_id)
- if a is not None and "numericValue" in a:
- out[key] = a["numericValue"]
-
- for cat_id, key in [
- ("performance", "performance_score"),
- ("accessibility", "accessibility_score"),
- ("seo", "seo_score"),
- ("best-practices", "best_practices_score"),
- ("pwa", "pwa_score"),
- ]:
- c = cats.get(cat_id)
- if c is not None and "score" in c:
- s = c["score"]
- out[key] = s
- out["category_scores"][cat_id] = round((s * 100)) if s is not None else None
-
- # Resolve impact from warning_mapper for each failure
- from ..tools.warnings import resolve_impact
- failures = []
- for aid, a in audits.items():
- if a is None:
- continue
- score = a.get("score")
- if score is None:
- continue
- if score < 1:
- title = a.get("title") or aid
- help_text = a.get("helpText") or ""
- impact = resolve_impact(aid, title, help_text)
- evidence = _evidence_from_audit(a)
- failures.append({
- "id": aid,
- "score": score,
- "helpText": help_text,
- "impact": impact,
- "evidence": evidence,
- })
- failures.sort(key=lambda x: (x["score"] or 0))
- out["top_failures"] = failures[:10]
-
- return out
-
-
-def median_or_none(values: list[float]) -> float | None:
- """Return median of list; None if empty or all None."""
- clean = [v for v in values if v is not None]
- if not clean:
- return None
- return statistics.median(clean)
-
-
def run_lighthouse_audit(
url: str,
strategy: str = "mobile",
diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py
index 9194ddae..ecdcaa9f 100644
--- a/src/website_profiling/llm/agent.py
+++ b/src/website_profiling/llm/agent.py
@@ -8,6 +8,12 @@
from ..text_sanitize import sanitize_unicode_deep, strip_surrogates
from ..tools.audit_tools import AuditToolContext
from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, openai_tools_schema
+from ..tools.audit_tools.tool_selector import (
+ apply_tool_cap,
+ chat_tool_mode,
+ chat_tool_search_cap,
+ select_tools_for_turn,
+)
from .base import ChatResult, ToolCall, get_llm_client
MAX_TOOL_ROUNDS = 10
@@ -15,24 +21,13 @@
SYSTEM_PROMPT = """You are Site Audit AI, a technical SEO assistant for a self-hosted site audit platform.
You help users understand crawl results, audit issues, Lighthouse scores, keywords, and Search Console data.
-Tool domains (prefer specific tools over generic list_issues):
-- Portfolio/report: get_report_summary, get_category_scores, list_audit_categories, get_executive_summary, get_audit_recommendations, list_report_history, get_portfolio_summary
-- Issues: list_issues, search_issues, list_top_impact_issues, prioritize_fix_roadmap, get_critical_issues, list_issues_by_category, get_category_issues, list_issues_with_ai_fixes, generate_issue_fix, list_issue_workflow
-- On-page: list_pages_missing_title, list_pages_noindex, list_seo_onpage_issues, list_content_url_issues, list_pages_missing_canonical, list_canonical_mismatch, list_pages_with_missing_alt, list_pages_missing_viewport
-- Crawl/pages: search_pages, search_pages_advanced, get_page_details, get_page_analysis, list_status_4xx_pages, list_pages_soft_404, list_dead_end_pages, list_duplicate_title_groups, list_heavy_pages_by_bytes, get_asset_weight_summary, get_readability_summary, get_status_code_breakdown, get_depth_distribution, list_long_redirect_chains, list_robots_blocked_urls, get_top_pages_by_pagerank
-- Schema/technical: get_schema_coverage, get_seo_health, get_security_findings, get_security_findings_summary, get_tech_stack_summary, list_pages_by_technology
-- Indexation: get_indexation_coverage, list_indexation_gaps, get_indexation_url_join
-- Keywords: get_keyword_summary, get_striking_distance_keywords, list_keywords_ctr_opportunity, list_keywords_by_position, get_keyword_serp_overlay, get_serp_feature_overlay, expand_keywords, generate_content_brief
-- Google: get_google_summary, get_gsc_top_queries, get_gsc_top_pages, get_gsc_ctr_opportunity_pages, get_google_integration_status, get_gsc_page_query_slice, get_gsc_url_inspection, get_gsc_index_coverage, get_ga4_page_metrics, analyze_serp_snippet_for_url
-- Links/backlinks: get_gsc_sample_links, get_backlinks_velocity, get_third_party_links_overlay, list_broken_link_sources, get_page_coach
-- Performance: get_lighthouse_summary, list_slow_pages, get_crux_summary, get_lighthouse_human_summary, list_lighthouse_poor_accessibility_pages, list_lighthouse_cwv_failures
-- Content/charts: get_issue_priority_breakdown, get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_outlink_distribution, get_content_analytics, get_top_crawled_pages, get_duplicate_cluster
-- Ops/logs: get_property_ops, list_crawl_runs, get_latest_log_analysis, get_log_top_paths, list_log_only_paths, list_crawl_only_paths, get_log_googlebot_stats
-- Drift: compare_reports, compare_category_deltas, compare_issue_deltas, compare_indexation_deltas, compare_orphan_deltas, compare_url_set_diff, compare_google_metrics, compare_security_deltas, compare_health_score_delta, get_health_history, get_category_health_history
-- GEO/AEO: get_geo_readiness_score, get_aeo_content_signals_for_url, get_llms_txt_status, draft_llms_txt, get_faq_schema_coverage, get_eeat_signals_summary, get_internal_link_suggestions, check_ai_citation_presence
-- Accessibility/assets: list_pages_with_axe_violations, get_axe_audit_summary, list_pages_with_mixed_content, list_pages_poor_cache_headers, get_rich_results_summary, list_rich_results_failures
-- Export/deliverables: export_audit_report, export_compare_csv, export_list_as_csv, compose_custom_report, export_custom_report, list_export_formats
-- Images: get_image_audit_summary, list_pages_with_missing_alt, list_pages_without_lazy_images, list_pages_with_images_missing_dimensions, list_site_image_urls, list_lighthouse_image_opportunities, list_largest_images, list_unoptimized_images, list_images_needing_attention
+Tool routing (only a subset of tools is loaded each turn):
+- Always available: search_audit_tools, list_tool_domains, get_data_coverage_report, run_insight_workflow, run_technical_workflow, run_keyword_workflow, run_domain_agent, plus top insight tools (get_report_summary, get_opportunity_matrix, get_traffic_health_check, etc.)
+- Use search_audit_tools(query) to discover specialized tools by topic (e.g. "broken links", "GSC CTR", "export PDF").
+- Use list_tool_domains to see domain groupings and example prompts.
+- Use run_*_workflow for common multi-step analyses (insight, technical, keyword).
+- Use run_domain_agent(task, domain) for deep exploration within one domain.
+- Use get_data_coverage_report when tools return empty or missing data.
Image playbook:
- Overview: get_image_audit_summary first — the UI renders summary cards, page preview lists (alt/lazy/OG/dimensions), and Lighthouse image findings. Write only ### Power Insights and ### Recommended actions (interpretation). Never repeat counts, URL lists, or markdown tables of pages.
@@ -120,9 +115,11 @@ def _react_step(
return ChatResult(content=text)
-def _tools_description(*, compact: bool = False) -> str:
+def _tools_description(*, names: set[str] | None = None, compact: bool = False) -> str:
lines = []
for t in TOOL_DEFINITIONS:
+ if names is not None and t["name"] not in names:
+ continue
if compact:
lines.append(f"- {t['name']}")
else:
@@ -130,6 +127,41 @@ def _tools_description(*, compact: bool = False) -> str:
return "\n".join(lines)
+def _last_user_message(messages: list[dict[str, str]]) -> str:
+ for msg in reversed(messages):
+ if msg.get("role") == "user":
+ return str(msg.get("content") or "")
+ return ""
+
+
+def _expand_active_tools_from_result(
+ tc_name: str,
+ tool_result: dict[str, Any],
+ active: set[str],
+) -> set[str]:
+ expanded = set(active)
+ pinned: set[str] = set()
+
+ if tc_name == "search_audit_tools":
+ names = tool_result.get("tool_names")
+ if isinstance(names, list):
+ for name in names[:12]:
+ if isinstance(name, str) and name:
+ expanded.add(name)
+ pinned.add(name)
+ elif tc_name == "run_domain_agent":
+ names = tool_result.get("tools_used")
+ if isinstance(names, list):
+ for name in names:
+ if isinstance(name, str) and name:
+ expanded.add(name)
+ pinned.add(name)
+
+ if chat_tool_mode() != "full" and pinned:
+ expanded = apply_tool_cap(expanded, chat_tool_search_cap(), pinned=pinned)
+ return expanded
+
+
def _build_openai_messages(history: list[dict[str, str]]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}]
for msg in history:
@@ -164,7 +196,9 @@ def run_agent_turn(
return {"ok": False, "error": msg}
openai_messages = _build_openai_messages(messages)
- tools = openai_tools_schema()
+ last_user = _last_user_message(messages)
+ active_names = select_tools_for_turn(last_user, messages)
+ tools = openai_tools_schema(active_names)
tool_events: list[dict[str, Any]] = []
final_message = ""
@@ -182,7 +216,12 @@ def on_token(text: str) -> None:
if _supports_native_tools(client):
result = client.chat_with_tools(llm_messages, tools, on_token=on_token)
else:
- result = _react_step(client, llm_messages, _tools_description(compact=True), on_token)
+ result = _react_step(
+ client,
+ llm_messages,
+ _tools_description(names=active_names, compact=True),
+ on_token,
+ )
except Exception as e:
msg = str(e).strip() or type(e).__name__
if "httpx" in msg.lower() or "requirements.txt" in msg.lower():
@@ -227,12 +266,22 @@ def on_token(text: str) -> None:
for tc in result.tool_calls:
_emit(on_event, {"type": "tool_start", "name": tc.name, "args": tc.arguments})
- tool_result = sanitize_unicode_deep(
- dispatch_tool(tc.name, tc.arguments, context=context),
- )
+ if chat_tool_mode() != "full" and tc.name not in active_names:
+ tool_result = {
+ "error": f"tool not loaded this turn: {tc.name}",
+ "hint": "Call search_audit_tools to load specialized tools, or rephrase your request.",
+ }
+ else:
+ tool_result = sanitize_unicode_deep(
+ dispatch_tool(tc.name, tc.arguments, context=context),
+ )
_emit(on_event, {"type": "tool_end", "name": tc.name, "result": tool_result})
tool_events.append({"name": tc.name, "args": tc.arguments, "result": tool_result})
+ active_names = _expand_active_tools_from_result(tc.name, tool_result, active_names)
+ if chat_tool_mode() != "full":
+ tools = openai_tools_schema(active_names)
+
tool_content = json.dumps(tool_result, default=str)
if ollama_format:
openai_messages.append({
diff --git a/src/website_profiling/mcp/core_server.py b/src/website_profiling/mcp/core_server.py
new file mode 100644
index 00000000..7026c4af
--- /dev/null
+++ b/src/website_profiling/mcp/core_server.py
@@ -0,0 +1,8 @@
+"""MCP core router server (Tier 0 + insight tools)."""
+from __future__ import annotations
+
+from .domain_server import run_domain_server
+
+
+def main() -> None:
+ run_domain_server("core")
diff --git a/src/website_profiling/mcp/domain_server.py b/src/website_profiling/mcp/domain_server.py
new file mode 100644
index 00000000..96f617dc
--- /dev/null
+++ b/src/website_profiling/mcp/domain_server.py
@@ -0,0 +1,11 @@
+"""MCP domain server entry — set WP_MCP_DOMAIN before starting."""
+from __future__ import annotations
+
+import os
+
+from .server import main
+
+
+def run_domain_server(domain: str) -> None:
+ os.environ["WP_MCP_DOMAIN"] = domain
+ main()
diff --git a/src/website_profiling/mcp/server.py b/src/website_profiling/mcp/server.py
index 5b150e61..4ca37356 100644
--- a/src/website_profiling/mcp/server.py
+++ b/src/website_profiling/mcp/server.py
@@ -9,7 +9,14 @@
from ..db.storage import db_session
from ..tools.audit_tools import AuditToolContext
-from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, tool_handler_names
+from ..tools.audit_tools.registry import (
+ TOOL_DEFINITIONS,
+ dispatch_tool,
+ list_domains_catalog,
+ mcp_tool_names,
+ tools_catalog_by_domain,
+)
+from ..tools.audit_tools.tool_domains import MCP_DOMAIN_BUNDLES
_URI_PROPERTY = re.compile(r"^audit://property/(\d+)$")
_URI_REPORT_LATEST = re.compile(r"^audit://property/(\d+)/report/latest$")
@@ -64,90 +71,41 @@ def _read_glossary_excerpt() -> str:
return text[:12000]
+def _mcp_domain() -> str:
+ return (os.environ.get("WP_MCP_DOMAIN") or "core").strip().lower()
+
+
+def _exposed_tool_names() -> set[str]:
+ return mcp_tool_names(_mcp_domain())
+
+
def _tools_catalog_json() -> str:
- domains: dict[str, list[str]] = {
- "portfolio": [],
- "issues": [],
- "crawl": [],
- "schema": [],
- "links": [],
- "indexation": [],
- "content": [],
- "keywords": [],
- "google": [],
- "backlinks": [],
- "performance": [],
- "drift": [],
- "security": [],
- "ops": [],
- "export": [],
- "images": [],
- "geo": [],
- "accessibility": [],
- "assets": [],
- "ctr": [],
- "integrations": [],
- }
- for tool in TOOL_DEFINITIONS:
- name = tool["name"]
- if name.startswith("export_") or name == "compose_custom_report" or name == "list_export_formats":
- domains["export"].append(name)
- elif name.startswith(("get_image_", "list_pages_without_lazy", "list_pages_with_images_missing", "list_site_image", "list_lighthouse_image", "list_largest_images", "list_unoptimized_images", "list_images_needing")):
- domains["images"].append(name)
- elif name.startswith(("list_propert", "get_propert", "get_report", "get_executive", "get_site", "list_report", "get_portfolio")) or name in (
- "get_ads_txt_status",
- "get_security_txt_status",
- "get_contact_intelligence",
- "get_rich_results_summary",
- "list_rich_results_failures",
- "get_competitor_keyword_gap",
- "get_pagination_audit_summary",
- ):
- domains["portfolio"].append(name)
- elif name in (
- "list_top_impact_issues",
- "prioritize_fix_roadmap",
- "generate_issue_fix",
- "summarize_category_for_client",
- ) or "issue" in name or "category" in name or "workflow" in name:
- domains["issues"].append(name)
- elif name.startswith(("get_geo_", "get_aeo_", "get_llms_", "get_eeat_", "get_faq_", "list_pages_missing_faq", "draft_llms", "check_ai_citation")):
- domains["geo"].append(name)
- elif "axe" in name or "mixed_content" in name or name == "get_heading_outline_for_url":
- domains["accessibility"].append(name)
- elif name in ("get_asset_weight_summary", "get_readability_summary", "list_heavy_pages_by_bytes", "list_pages_poor_cache_headers", "list_pages_low_content_ratio"):
- domains["assets"].append(name)
- elif "ctr" in name or name in ("list_keywords_ctr_opportunity", "analyze_serp_snippet_for_url"):
- domains["ctr"].append(name)
- elif name in ("get_gsc_url_inspection", "get_gsc_index_coverage", "get_bing_index_status", "get_serp_feature_overlay"):
- domains["integrations"].append(name)
- elif name.startswith(("list_pages_", "list_canonical", "list_long_", "list_robots_", "get_top_pages_by", "search_pages", "get_page_", "list_redirects", "list_broken", "list_status_", "get_status_code", "get_response_time", "get_depth", "get_crawl_", "get_browser", "list_pages_with", "list_pages_by")):
- domains["crawl"].append(name)
- elif "schema" in name or name == "get_seo_health":
- domains["schema"].append(name)
- elif "orphan" in name or "link" in name or "fingerprint" in name or "pagerank" in name:
- domains["links"].append(name)
- elif "indexation" in name or "hreflang" in name or "language" in name or name == "list_subdomains":
- domains["indexation"].append(name)
- elif "content" in name or "social" in name or "ner" in name or "thin" in name or "opportunit" in name or "duplicate" in name:
- domains["content"].append(name)
- elif "keyword" in name or "cannibal" in name or "misalignment" in name or "striking" in name or "semantic" in name or name == "expand_keywords" or name == "generate_content_brief":
- domains["keywords"].append(name)
- elif "google" in name or "gsc" in name or "ga4" in name:
- domains["google"].append(name)
- elif "backlink" in name or "competitor" in name or "bing" in name or "gsc_links" in name:
- domains["backlinks"].append(name)
- elif "lighthouse" in name or "crux" in name or "slow" in name or "cwv" in name:
- domains["performance"].append(name)
- elif "health" in name or "compare" in name or "alert" in name or "tech_stack" in name or name == "list_pages_by_technology":
- domains["drift"].append(name)
- elif "security" in name:
- domains["security"].append(name)
- elif "log" in name or name in ("get_property_ops", "list_crawl_runs", "list_log_uploads", "get_page_coach"):
- domains["ops"].append(name)
- else:
- domains["portfolio"].append(name)
- return json.dumps({"tool_count": len(TOOL_DEFINITIONS), "handlers": sorted(tool_handler_names()), "domains": domains}, indent=2)
+ domain = _mcp_domain()
+ exposed = _exposed_tool_names()
+ by_domain = tools_catalog_by_domain()
+ scoped: dict[str, list[str]] = {}
+ for d, names in by_domain.items():
+ filtered = [n for n in names if n in exposed]
+ if filtered:
+ scoped[d] = filtered
+ return json.dumps({
+ "mcp_domain": domain,
+ "tool_count": len(exposed),
+ "handlers": sorted(exposed),
+ "domains": scoped,
+ "available_mcp_domains": sorted(MCP_DOMAIN_BUNDLES.keys()),
+ }, indent=2)
+
+
+def _domains_resource_json() -> str:
+ return json.dumps({
+ "current_mcp_domain": _mcp_domain(),
+ "bundles": {
+ key: sorted(domains)
+ for key, domains in MCP_DOMAIN_BUNDLES.items()
+ },
+ "catalog": list_domains_catalog(),
+ }, indent=2)
def _resolve_resource(uri: str) -> str:
@@ -161,6 +119,9 @@ def _resolve_resource(uri: str) -> str:
if uri == "audit://tools":
return _tools_catalog_json()
+ if uri == "audit://domains":
+ return _domains_resource_json()
+
m = _URI_PROPERTY.match(uri)
if m:
pid = int(m.group(1))
@@ -202,13 +163,16 @@ def main() -> None:
"MCP SDK not installed. Run: pip install -r requirements.txt",
) from e
- server = Server("site-audit")
+ server = Server(f"site-audit-{_mcp_domain()}")
default_pid = _default_property_id()
+ exposed = _exposed_tool_names()
@server.list_tools()
async def list_tools() -> list[Tool]:
out: list[Tool] = []
for spec in TOOL_DEFINITIONS:
+ if spec["name"] not in exposed:
+ continue
out.append(
Tool(
name=spec["name"],
@@ -220,6 +184,12 @@ async def list_tools() -> list[Tool]:
@server.call_tool()
async def call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextContent]:
+ if name not in exposed:
+ result = {
+ "error": f"tool not exposed in MCP domain {_mcp_domain()}: {name}",
+ "hint": "Connect WP_MCP_DOMAIN=full or the domain server that includes this tool.",
+ }
+ return [TextContent(type="text", text=json.dumps(result, indent=2, default=str))]
args = dict(arguments or {})
ctx = _merge_context(args)
result = dispatch_tool(name, args, context=ctx)
@@ -231,7 +201,8 @@ async def list_resources() -> list[Resource]:
resources = [
Resource(uri="audit://properties", name="Properties", description="All configured site properties", mimeType="application/json"),
Resource(uri="audit://glossary", name="Glossary", description="Site Audit field glossary excerpt", mimeType="text/markdown"),
- Resource(uri="audit://tools", name="Tool catalog", description="MCP tool catalog grouped by domain", mimeType="application/json"),
+ Resource(uri="audit://tools", name="Tool catalog", description="MCP tool catalog for the connected domain server", mimeType="application/json"),
+ Resource(uri="audit://domains", name="MCP domain servers", description="Available WP_MCP_DOMAIN bundles and domain groupings", mimeType="application/json"),
]
if default_pid:
resources.extend([
diff --git a/src/website_profiling/parsing/__init__.py b/src/website_profiling/parsing/__init__.py
new file mode 100644
index 00000000..6af83821
--- /dev/null
+++ b/src/website_profiling/parsing/__init__.py
@@ -0,0 +1 @@
+"""HTML/CSV parsing utilities."""
diff --git a/src/website_profiling/parsing/content.py b/src/website_profiling/parsing/content.py
new file mode 100644
index 00000000..553f9f80
--- /dev/null
+++ b/src/website_profiling/parsing/content.py
@@ -0,0 +1,37 @@
+"""Content text and social meta parsing."""
+from __future__ import annotations
+
+from ..content_analysis.page import analyze_page_html
+
+
+def _count_syllables(word: str) -> int:
+ from ..content_analysis.reading_level import count_syllables
+
+ return count_syllables(word)
+
+
+def parse_content_text(soup, raw_html: str, excerpt_max_chars: int = 0) -> dict:
+ """Extract content analytics: word count, reading level, content-to-HTML ratio, top keywords.
+
+ excerpt_max_chars: when > 0, strip script/style from body and store a whitespace-normalized
+ plain-text excerpt (truncated) in ``content_excerpt`` for analysis / AI / UI.
+ """
+ del soup # analyze_page_html loads from raw_html for a single code path
+ return analyze_page_html(raw_html, excerpt_max_chars=excerpt_max_chars, strategy="full_body")
+
+
+def parse_social_meta(soup) -> dict:
+ """Extract Open Graph and Twitter Card meta tags."""
+ def _meta_content(attrs: dict) -> str:
+ tag = soup.find("meta", attrs=attrs)
+ return (tag.get("content") or "").strip() if tag else ""
+
+ return {
+ "og_title": _meta_content({"property": "og:title"}),
+ "og_description": _meta_content({"property": "og:description"}),
+ "og_image": _meta_content({"property": "og:image"}),
+ "og_type": _meta_content({"property": "og:type"}),
+ "twitter_card": _meta_content({"name": "twitter:card"}),
+ "twitter_title": _meta_content({"name": "twitter:title"}),
+ "twitter_image": _meta_content({"name": "twitter:image"}),
+ }
diff --git a/src/website_profiling/parsing/io.py b/src/website_profiling/parsing/io.py
new file mode 100644
index 00000000..758e0191
--- /dev/null
+++ b/src/website_profiling/parsing/io.py
@@ -0,0 +1,72 @@
+"""DataFrame and edge list I/O."""
+from __future__ import annotations
+
+import json
+import os
+
+import pandas as pd
+
+"""
+Shared helpers for crawler and report/plot scripts.
+"""
+import json
+import os
+import warnings
+from urllib.parse import urljoin, urldefrag, urlparse
+import urllib.robotparser as robotparser
+import ast
+import math
+
+import pandas as pd
+from bs4 import BeautifulSoup
+
+
+def load_dataframe(path: str) -> pd.DataFrame:
+ """Load a DataFrame from CSV or JSON (by extension)."""
+ if not os.path.isfile(path):
+ raise FileNotFoundError(path)
+ path_lower = path.lower()
+ if path_lower.endswith(".json"):
+ return pd.read_json(path, orient="records")
+ return pd.read_csv(path)
+
+
+def save_dataframe(df: pd.DataFrame, path: str) -> None:
+ """Save a DataFrame to CSV or JSON (by extension). Uses default_handler for JSON to avoid numpy types."""
+ path_lower = path.lower()
+ if path_lower.endswith(".json"):
+ df.to_json(path, orient="records", indent=2, date_format="iso", default_handler=str)
+ else:
+ df.to_csv(path, index=False)
+
+
+def load_edges(path: str) -> list[tuple[str, str]]:
+ """Load edge list from CSV or JSON (by extension). Returns list of (from_url, to_url)."""
+ if not os.path.isfile(path):
+ return []
+ path_lower = path.lower()
+ try:
+ if path_lower.endswith(".json"):
+ with open(path, "r", encoding="utf-8") as f:
+ data = json.load(f)
+ if isinstance(data, list) and data and isinstance(data[0], dict):
+ return [(str(o.get("from", "")), str(o.get("to", ""))) for o in data if o.get("from") and o.get("to")]
+ return []
+ edf = pd.read_csv(path)
+ if {"from", "to"}.issubset(edf.columns):
+ return [(str(a).rstrip("/"), str(b).rstrip("/")) for a, b in edf[["from", "to"]].values]
+ except Exception:
+ pass
+ return []
+
+
+def save_edges(edges: list[tuple[str, str]], path: str) -> None:
+ """Save edge list to CSV or JSON (by extension)."""
+ path_lower = path.lower()
+ if path_lower.endswith(".json"):
+ data = [{"from": a, "to": b} for a, b in edges]
+ with open(path, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=2)
+ else:
+ pd.DataFrame(edges, columns=["from", "to"]).to_csv(path, index=False)
+
diff --git a/src/website_profiling/parsing/links.py b/src/website_profiling/parsing/links.py
new file mode 100644
index 00000000..6f78fb31
--- /dev/null
+++ b/src/website_profiling/parsing/links.py
@@ -0,0 +1,155 @@
+"""Link normalization and extraction."""
+from __future__ import annotations
+
+import ast
+import math
+from urllib.parse import urldefrag, urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+_TRACKING_PARAM_PREFIXES = ("utm_",)
+_FACET_PARAM_NAMES = frozenset({"sort", "filter", "page", "offset", "limit"})
+
+
+def strip_crawl_query_params(url: str, ignore_params: list[str] | None = None) -> str:
+ """Remove tracking and facet query params for crawl deduplication."""
+ parsed = urlparse(url)
+ if not parsed.query:
+ return url.rstrip("/")
+ ignore = {p.lower() for p in (ignore_params or [])}
+ parts = []
+ for pair in parsed.query.split("&"):
+ if not pair:
+ continue
+ key = pair.split("=", 1)[0].lower()
+ if key in ignore:
+ continue
+ if any(key.startswith(p) for p in _TRACKING_PARAM_PREFIXES):
+ continue
+ if key in _FACET_PARAM_NAMES:
+ continue
+ parts.append(pair)
+ query = "&".join(parts)
+ rebuilt = parsed._replace(query=query).geturl()
+ return rebuilt.rstrip("/")
+
+
+def normalize_link(
+ base: str,
+ href: str,
+ strip_params: bool = True,
+ ignore_params: list[str] | None = None,
+) -> str | None:
+ if not href:
+ return None
+ href = href.strip()
+ if href.startswith(("mailto:", "javascript:", "tel:", "data:")):
+ return None
+ joined = urljoin(base, href)
+ joined, _ = urldefrag(joined)
+ parsed = urlparse(joined)
+ if parsed.scheme not in ("http", "https"):
+ return None
+ out = joined.rstrip("/")
+ if strip_params:
+ out = strip_crawl_query_params(out, ignore_params)
+ return out
+
+
+def _parse_rel_flags(rel_raw: str) -> tuple[bool, bool, bool]:
+ parts = {p.strip().lower() for p in (rel_raw or "").split() if p.strip()}
+ return ("nofollow" in parts, "sponsored" in parts, "ugc" in parts)
+
+
+def _anchor_text_from_tag(a) -> str:
+ parts: list[str] = []
+ for child in a.children:
+ if getattr(child, "name", None) == "img":
+ parts.append("[image]")
+ elif isinstance(child, str):
+ t = child.strip()
+ if t:
+ parts.append(t)
+ text = " ".join(parts).strip() or a.get_text(separator=" ", strip=True)
+ return (text or "")[:500]
+
+
+def parse_link_edges(base_url: str, html_text: str) -> tuple[str, list[dict]]:
+ """Extract title and rich outbound link records from HTML."""
+ soup = BeautifulSoup(html_text, "lxml")
+ title_tag = (
+ soup.title.string.strip()
+ if soup.title and soup.title.string
+ else ""
+ )
+ start_netloc = urlparse(base_url).netloc
+ edges: list[dict] = []
+ for a in soup.find_all("a", href=True):
+ ln = normalize_link(base_url, a["href"])
+ if not ln:
+ continue
+ rel_raw = a.get("rel") or ""
+ if isinstance(rel_raw, list):
+ rel_str = " ".join(str(x) for x in rel_raw)
+ else:
+ rel_str = str(rel_raw)
+ nofollow, sponsored, ugc = _parse_rel_flags(rel_str)
+ link_type = "internal" if urlparse(ln).netloc == start_netloc else "external"
+ edges.append({
+ "to_url": ln.rstrip("/"),
+ "anchor_text": _anchor_text_from_tag(a),
+ "rel": rel_str.strip(),
+ "is_nofollow": nofollow,
+ "is_sponsored": sponsored,
+ "is_ugc": ugc,
+ "link_type": link_type,
+ })
+ return title_tag, edges
+
+
+def parse_links(base_url: str, html_text: str) -> tuple[str, set[str]]:
+ """Extract page title and set of absolute links from HTML. Returns (title, links)."""
+ title, edges = parse_link_edges(base_url, html_text)
+ return title, {e["to_url"] for e in edges}
+
+def _is_empty(raw) -> bool:
+ if raw is None:
+ return True
+ if isinstance(raw, float) and math.isnan(raw):
+ return True
+ if raw == "":
+ return True
+ return False
+
+
+def parse_links_serialized(raw) -> list[str]:
+ """
+ Parse a serialized list of URLs from CSV/DataFrame (string list repr, comma-separated, or list).
+ """
+ if _is_empty(raw):
+ return []
+ if isinstance(raw, list):
+ return [str(x).strip().rstrip("/") for x in raw if x]
+ s = str(raw).strip()
+ if not s:
+ return []
+ if s.startswith("[") and s.endswith("]"):
+ try:
+ v = ast.literal_eval(s)
+ if isinstance(v, (list, tuple)):
+ return [str(x).strip().rstrip("/") for x in v if x]
+ except Exception:
+ pass
+ return [t.strip().rstrip("/") for t in s.split(",") if t.strip()]
+
+
+# Column names that may contain serialized outlink lists (for building edges from crawl CSV)
+LINK_COLUMN_NAMES = (
+ "links",
+ "edges",
+ "outlinks",
+ "outlink_targets",
+ "targets",
+ "link_targets",
+ "links_list",
+)
diff --git a/src/website_profiling/parsing/robots.py b/src/website_profiling/parsing/robots.py
new file mode 100644
index 00000000..de523770
--- /dev/null
+++ b/src/website_profiling/parsing/robots.py
@@ -0,0 +1,17 @@
+"""robots.txt loading."""
+from __future__ import annotations
+
+from urllib.parse import urlparse
+import urllib.robotparser as robotparser
+
+def load_robots(start_url: str):
+ """Load robots.txt for the given URL; returns RobotFileParser or None on error."""
+ parsed = urlparse(start_url)
+ robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+ rp = robotparser.RobotFileParser()
+ rp.set_url(robots_url)
+ try:
+ rp.read()
+ return rp
+ except Exception:
+ return None
diff --git a/src/website_profiling/parsing/seo.py b/src/website_profiling/parsing/seo.py
new file mode 100644
index 00000000..e64187d8
--- /dev/null
+++ b/src/website_profiling/parsing/seo.py
@@ -0,0 +1,138 @@
+"""SEO and resource parsing from HTML."""
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from bs4 import BeautifulSoup
+
+from .links import normalize_link
+
+def parse_seo(base_url: str, html_text: str) -> tuple[str, int, str, int, str]:
+ """
+ Extract SEO-related fields from HTML.
+ Returns (meta_description, meta_description_len, h1_text, h1_count, canonical_url).
+ """
+ soup = BeautifulSoup(html_text, "lxml")
+ meta_desc = ""
+ meta = soup.find("meta", attrs={"name": "description"})
+ if meta and meta.get("content"):
+ meta_desc = (meta["content"] or "").strip()
+ if not meta_desc:
+ og = soup.find("meta", attrs={"property": "og:description"})
+ if og and og.get("content"):
+ meta_desc = (og["content"] or "").strip()
+ meta_desc_len = len(meta_desc)
+
+ h1_tags = soup.find_all("h1")
+ h1_count = len(h1_tags)
+ h1_text = (h1_tags[0].get_text(separator=" ", strip=True) if h1_tags else "") or ""
+
+ canonical_url = ""
+ link_canonical = soup.find("link", attrs={"rel": "canonical"})
+ if link_canonical and link_canonical.get("href"):
+ canonical_url = normalize_link(base_url, link_canonical["href"]) or ""
+
+ return meta_desc, meta_desc_len, h1_text, h1_count, canonical_url
+
+
+def parse_seo_extended(html_text: str, base_url: str) -> dict:
+ """
+ Extract extended SEO/accessibility/performance-related fields from HTML.
+ Returns a dict with: viewport_present, viewport_content, noindex, has_schema,
+ heading_sequence, images_without_alt, images_total, img_without_lazy, img_without_dimensions,
+ aria_count, mixed_content_count.
+ """
+ soup = BeautifulSoup(html_text, "lxml")
+ out = {
+ "viewport_present": False,
+ "viewport_content": "",
+ "noindex": False,
+ "has_schema": False,
+ "heading_sequence": [],
+ "heading_text": [],
+ "images_without_alt": 0,
+ "images_total": 0,
+ "img_without_lazy": 0,
+ "img_without_dimensions": 0,
+ "aria_count": 0,
+ "mixed_content_count": 0,
+ }
+ # Viewport
+ viewport = soup.find("meta", attrs={"name": "viewport"})
+ if viewport and viewport.get("content"):
+ out["viewport_present"] = True
+ out["viewport_content"] = (viewport["content"] or "").strip()
+ # noindex
+ robots = soup.find("meta", attrs={"name": "robots"})
+ if robots and robots.get("content"):
+ content = (robots["content"] or "").lower()
+ out["noindex"] = "noindex" in content
+ # Structured data: JSON-LD or microdata
+ if soup.find("script", type="application/ld+json"):
+ out["has_schema"] = True
+ if soup.find(attrs={"itemscope": True}):
+ out["has_schema"] = True
+ # Heading order (h1..h6 tag names) and visible heading copy (for keywords / fingerprints)
+ for tag in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
+ if tag.name:
+ out["heading_sequence"].append(tag.name)
+ text = (tag.get_text(separator=" ", strip=True) or "").strip()
+ if text:
+ out["heading_text"].append(text)
+ # Images: alt, lazy, dimensions
+ base_scheme = urlparse(base_url).scheme.lower()
+ for img in soup.find_all("img"):
+ out["images_total"] += 1
+ if not img.get("alt") and not img.get("aria-label"):
+ out["images_without_alt"] += 1
+ loading = (img.get("loading") or "").strip().lower()
+ if loading != "lazy":
+ out["img_without_lazy"] += 1
+ if not img.get("width") and not img.get("height"):
+ out["img_without_dimensions"] += 1
+ src = img.get("src") or ""
+ if base_scheme == "https" and src.strip().lower().startswith("http://"):
+ out["mixed_content_count"] += 1
+ # ARIA: count elements with any aria- attribute
+ for el in soup.find_all(True):
+ if getattr(el, "attrs", None) and any(k.startswith("aria-") for k in el.attrs):
+ out["aria_count"] += 1
+ # Mixed content: links and other src/href
+ for tag in soup.find_all(True):
+ for attr in ("href", "src", "srcset"):
+ val = tag.get(attr)
+ if not val or base_scheme != "https":
+ continue
+ val = str(val).strip().lower()
+ if val.startswith("http://"):
+ out["mixed_content_count"] += 1
+ elif attr == "srcset":
+ for part in val.split(","):
+ part = part.strip().split()[0] if part.strip() else ""
+ if part.startswith("http://"):
+ out["mixed_content_count"] += 1
+ return out
+def parse_resources(html_text: str, base_url: str) -> dict:
+ """
+ Extract script/link resource counts and total sizes (same-origin only, no fetch).
+ Returns dict: script_count, link_stylesheet_count, script_urls, stylesheet_urls
+ (URLs for optional later HEAD/GET). Does not fetch; caller may fetch with limit.
+ """
+ soup = BeautifulSoup(html_text, "lxml")
+ parsed_base = urlparse(base_url)
+ script_urls = []
+ for s in soup.find_all("script", src=True):
+ url = normalize_link(base_url, s["src"])
+ if url and urlparse(url).netloc == parsed_base.netloc:
+ script_urls.append(url)
+ stylesheet_urls = []
+ for link in soup.find_all("link", rel=lambda r: r and "stylesheet" in (r.lower() if isinstance(r, str) else "")):
+ url = link.get("href") and normalize_link(base_url, link["href"])
+ if url and urlparse(url).netloc == parsed_base.netloc:
+ stylesheet_urls.append(url)
+ return {
+ "script_count": len(script_urls),
+ "link_stylesheet_count": len(stylesheet_urls),
+ "script_urls": script_urls,
+ "stylesheet_urls": stylesheet_urls,
+ }
diff --git a/src/website_profiling/parsing/tech.py b/src/website_profiling/parsing/tech.py
new file mode 100644
index 00000000..587e22f5
--- /dev/null
+++ b/src/website_profiling/parsing/tech.py
@@ -0,0 +1,116 @@
+"""Technology stack detection."""
+from __future__ import annotations
+
+import json
+import warnings
+
+_TECH_PATTERNS = [
+ ("WordPress", "html", "/wp-content/"),
+ ("WordPress", "html", "/wp-includes/"),
+ ("Drupal", "meta_generator", "Drupal"),
+ ("Joomla", "meta_generator", "Joomla"),
+ ("Shopify", "html", "cdn.shopify.com"),
+ ("Squarespace", "html", "squarespace.com"),
+ ("Wix", "html", "wix.com"),
+ ("Next.js", "html", "__NEXT_DATA__"),
+ ("Next.js", "html", "_next/static"),
+ ("Nuxt.js", "html", "__NUXT__"),
+ ("Gatsby", "html", "gatsby-"),
+ ("React", "html", "data-reactroot"),
+ ("React", "html", "__REACT_DEVTOOLS"),
+ ("React", "html", "react.production.min"),
+ ("Vue.js", "html", "__vue"),
+ ("Vue.js", "html", "vue.min.js"),
+ ("Angular", "html", "ng-version"),
+ ("Angular", "html", "ng-app"),
+ ("Svelte", "html", "svelte"),
+ ("jQuery", "html", "jquery"),
+ ("Bootstrap", "html", "bootstrap"),
+ ("Tailwind CSS", "html", "tailwindcss"),
+ ("Google Analytics", "html", "google-analytics.com/analytics.js"),
+ ("Google Analytics", "html", "googletagmanager.com/gtag"),
+ ("Google Tag Manager", "html", "googletagmanager.com/gtm.js"),
+ ("Facebook Pixel", "html", "connect.facebook.net"),
+ ("Hotjar", "html", "hotjar.com"),
+ ("Google Fonts", "html", "fonts.googleapis.com"),
+ ("Font Awesome", "html", "fontawesome"),
+ ("Cloudflare", "header", "cf-ray"),
+ ("Nginx", "header_server", "nginx"),
+ ("Apache", "header_server", "apache"),
+ ("LiteSpeed", "header_server", "litespeed"),
+ ("Vercel", "header_server", "vercel"),
+ ("Netlify", "header_server", "netlify"),
+ ("Amazon CloudFront", "header", "x-amz-cf-id"),
+ ("AWS", "header_server", "amazons3"),
+]
+
+# Module-level cache for Wappalyzer instance (avoids reloading technologies file per page).
+_wappalyzer_instance = None
+_wappalyzer_disabled = False
+
+
+def _is_wappalyzer_regex_warning(msg: str) -> bool:
+ lower = msg.lower()
+ return "compiling regex" in lower and "unbalanced parenthesis" in lower
+
+
+def detect_tech_wappalyzer(
+ url: str,
+ html: str,
+ headers: dict,
+ soup,
+ wappalyzer=None,
+) -> str:
+ """
+ Detect technologies using python-Wappalyzer from existing HTML and headers.
+ Returns JSON list of tech names. On any failure, falls back to parse_tech_stack(soup, headers, url).
+ """
+ global _wappalyzer_instance, _wappalyzer_disabled
+ if _wappalyzer_disabled:
+ return parse_tech_stack(soup, headers, url)
+ try:
+ from Wappalyzer import Wappalyzer, WebPage
+ except ImportError:
+ return parse_tech_stack(soup, headers, url)
+ try:
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ instance = wappalyzer if wappalyzer is not None else _wappalyzer_instance
+ if instance is None:
+ instance = Wappalyzer.latest()
+ if wappalyzer is None:
+ _wappalyzer_instance = instance
+ webpage = WebPage(url, html=html, headers=headers)
+ detected = instance.analyze(webpage)
+ if any(_is_wappalyzer_regex_warning(str(w.message)) for w in caught):
+ _wappalyzer_disabled = True
+ _wappalyzer_instance = None
+ return parse_tech_stack(soup, headers, url)
+ return json.dumps(sorted(detected))
+ except Exception:
+ return parse_tech_stack(soup, headers, url)
+
+
+def parse_tech_stack(soup, headers: dict, url: str) -> str:
+ """Detect technologies from HTML patterns and HTTP headers. Returns JSON list of tech names."""
+ detected = set()
+ html_str = str(soup).lower()
+ meta_gen = soup.find("meta", attrs={"name": "generator"})
+ generator = (meta_gen.get("content") or "").strip().lower() if meta_gen else ""
+ server_header = (headers.get("Server") or headers.get("server") or "").lower()
+
+ for name, source, pattern in _TECH_PATTERNS:
+ pat = pattern.lower()
+ if source == "html" and pat in html_str:
+ detected.add(name)
+ elif source == "meta_generator" and pat in generator:
+ detected.add(name)
+ elif source == "header":
+ for v in headers.values():
+ if isinstance(v, str) and pat in v.lower():
+ detected.add(name)
+ break
+ elif source == "header_server" and pat in server_header:
+ detected.add(name)
+
+ return json.dumps(sorted(detected))
diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py
index c178dd16..f0febd25 100644
--- a/src/website_profiling/reporting/builder.py
+++ b/src/website_profiling/reporting/builder.py
@@ -1,1181 +1,77 @@
"""
Generate report data from crawl and write to PostgreSQL. The Next.js UI in web/ reads via /api/report/*.
"""
-import hashlib
+from __future__ import annotations
+
import json
import os
-import socket
-import ssl
-import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from typing import Any, Optional
from urllib.parse import urlparse
import pandas as pd
import requests
-from bs4 import BeautifulSoup
-from tqdm.auto import tqdm
-
-from ..common import (
- LINK_COLUMN_NAMES,
- load_edges,
- normalize_link,
- parse_links_serialized,
-)
-from ..tools.keywords import cluster_keywords, extract_candidates_from_df, score_keywords
-from ..config import get_bool, get_int
+
from ..analysis import merge_bundles, run_local_enrichment
-from ..analysis.text_hygiene import filter_topic_clusters, is_junk_semantic_term
+from ..analysis.text_hygiene import is_junk_semantic_term
+from ..config import get_bool, get_int
from ..llm.enrich import cluster_keywords_llm, run_llm_enrichment
from ..llm_config import load_llm_config_from_db, llm_is_enabled
-from .categories import build_categories
from ..security_scanner import run_security_scan
-
-# SEO thresholds for recommendations
-TITLE_LEN_MIN = 30
-TITLE_LEN_MAX = 60
-META_DESC_LEN_MIN = 70
-META_DESC_LEN_MAX = 160
-THIN_CONTENT_CHARS = 300
-
-
-def fetch_site_ssl_expires_iso(hostname: str, timeout: float = 5.0) -> Optional[str]:
- """Return certificate notAfter as ISO 8601 UTC, or None on failure."""
- host = (hostname or "").strip().lower()
- if not host:
- return None
- try:
- ctx = ssl.create_default_context()
- with socket.create_connection((host, 443), timeout=timeout) as sock:
- with ctx.wrap_socket(sock, server_hostname=host) as ssock:
- cert = ssock.getpeercert()
- if not cert:
- return None
- na = cert.get("notAfter")
- if not na:
- return None
- ts = ssl.cert_time_to_seconds(na)
- return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
- except Exception:
- return None
-
-
-def _strip_www(host: str) -> str:
- h = (host or "").strip().lower()
- return h[4:] if h.startswith("www.") else h
-
-
-def _url_hostname(url: str) -> str:
- if not url:
- return ""
- try:
- return (urlparse(str(url).strip()).hostname or "").lower()
- except Exception:
- return ""
-
-
-def _hosts_match(a: str, b: str) -> bool:
- if not a or not b:
- return False
- a, b = a.lower(), b.lower()
- return a == b or _strip_www(a) == _strip_www(b)
-
-
-def filter_lighthouse_by_host(by_url: dict[str, Any], expected_host: str) -> dict[str, Any]:
- """Keep only Lighthouse entries whose URL hostname matches expected_host (www.-tolerant)."""
- if not by_url or not expected_host:
- return by_url or {}
- return {u: v for u, v in by_url.items() if _hosts_match(_url_hostname(u), expected_host)}
-
-
-def _derive_expected_host(start_url: str, df: pd.DataFrame) -> str:
- host = _url_hostname(start_url)
- if host:
- return host
- if df is not None and not df.empty and "url" in df.columns:
- for u in df["url"]:
- h = _url_hostname(str(u))
- if h:
- return h
- return ""
-
-
-def _pick_lighthouse_summary(
- lighthouse_by_url: dict[str, Any],
- start_url: str,
- global_summary: Optional[dict[str, Any]],
- expected_host: str,
-) -> Optional[dict[str, Any]]:
- """Prefer per-URL summary for this crawl; only use global summary if hostname matches."""
- if lighthouse_by_url and start_url:
- match = lighthouse_for_url(lighthouse_by_url, start_url)
- if match:
- return match
- if lighthouse_by_url:
- first_key = next(iter(lighthouse_by_url), None)
- if first_key is not None:
- return lighthouse_by_url[first_key]
- if global_summary:
- if not expected_host or _hosts_match(_url_hostname(str(global_summary.get("url") or "")), expected_host):
- return global_summary
- return None
-
-
-def build_lighthouse_by_url_for_report(conn: Any) -> dict[str, Any]:
- """
- Merge per-URL Lighthouse page summaries with latest lighthouse_runs row: full audits/items
- from normalized tables, uncapped top_failures and diagnostics from stored LHR JSON.
- """
- from ..db import (
- read_lh_audits_with_items,
- read_lh_runs_by_url,
- read_lighthouse_page_summaries,
- read_lighthouse_run_json,
- )
- from ..lighthouse.runner import _evidence_from_audit, extract_from_lighthouse_json
- from ..tools.warnings import parse_lighthouse_to_diagnostics, resolve_impact
-
- summaries = read_lighthouse_page_summaries(conn)
- runs_map = read_lh_runs_by_url(conn)
-
- summaries_norm: dict[str, Any] = {}
- for k, v in summaries.items():
- nk = str(k).strip().rstrip("/")
- summaries_norm[nk] = v
-
- all_urls = set(summaries_norm.keys()) | set(runs_map.keys())
- out: dict[str, Any] = {}
-
- for u in sorted(all_urls):
- base: dict[str, Any] = dict(summaries_norm[u]) if u in summaries_norm else {}
- run_ids = runs_map.get(u, [])
- run_id = run_ids[-1] if run_ids else None
-
- if run_id is not None:
- raw = read_lighthouse_run_json(conn, run_id)
- if not base and raw:
- ex = extract_from_lighthouse_json(raw)
- lr = raw.get("lighthouseResult") or raw
- final_u = lr.get("finalUrl") or lr.get("requestedUrl") or u
- base = {
- "url": str(final_u).strip().rstrip("/"),
- "median_metrics": {
- "lcp_ms": ex.get("lcp_ms"),
- "cls": ex.get("cls"),
- "tbt_ms": ex.get("tbt_ms"),
- "fcp_ms": ex.get("fcp_ms"),
- "speed_index_ms": ex.get("speed_index_ms"),
- "performance_score": ex.get("performance_score"),
- "accessibility_score": ex.get("accessibility_score"),
- "seo_score": ex.get("seo_score"),
- "best_practices_score": ex.get("best_practices_score"),
- "pwa_score": ex.get("pwa_score"),
- },
- "category_scores": dict(ex.get("category_scores") or {}),
- "strategy": "mobile",
- "device": "mobile",
- "mode": "navigation",
- }
- base["audits"] = read_lh_audits_with_items(conn, run_id)
- if raw:
- lr = raw.get("lighthouseResult") or raw
- audits_map = lr.get("audits") or {}
- failures: list[dict[str, Any]] = []
- for aid, a in audits_map.items():
- if not isinstance(a, dict):
- continue
- score = a.get("score")
- if score is None or score >= 1:
- continue
- title = a.get("title") or aid
- help_text = a.get("helpText") or ""
- failures.append(
- {
- "id": aid,
- "score": score,
- "helpText": help_text,
- "impact": resolve_impact(aid, title, help_text),
- "evidence": _evidence_from_audit(a),
- }
- )
- failures.sort(key=lambda x: (x["score"] or 0))
- base["top_failures"] = failures
- base["diagnostics"] = parse_lighthouse_to_diagnostics(raw, max_nodes_in_refs=None)
- elif not base:
- continue
-
- if not base.get("url"):
- base["url"] = u
- out[u] = base
-
- return out
-
-
-def lighthouse_for_url(lighthouse_by_url: dict[str, Any], url: str) -> Optional[dict[str, Any]]:
- """Resolve Lighthouse summary for a crawled URL (trailing-slash tolerant)."""
- if not lighthouse_by_url or not url:
- return None
- u = str(url).strip().rstrip("/")
- if u in lighthouse_by_url:
- return lighthouse_by_url[u]
- for k, v in lighthouse_by_url.items():
- if str(k).strip().rstrip("/") == u:
- return v
- return None
-
-
-def build_edges_from_df(
- df: pd.DataFrame,
- edges_csv: str,
- same_domain_only: bool,
- max_fetch_for_edges: int,
- concurrency: int,
- timeout: int,
- polite_delay: float,
- render_mode: str = "static",
- js_timeout: int = 30,
- js_concurrency: int = 3,
- js_wait_until: str = "domcontentloaded",
- js_extra_wait_ms: int = 1500,
- js_block_resources: bool = True,
-) -> list[tuple[str, str]]:
- """Build or load edges; return list of (from, to) tuples."""
- edges = load_edges(edges_csv) if (edges_csv or "").strip() else []
- if edges:
- return edges
-
- # Prefer columns that hold URL lists (e.g. outlink_targets); skip "outlinks" (numeric count)
- candidate_cols = [
- c for c in df.columns
- if c.lower() in LINK_COLUMN_NAMES and c.lower() != "outlinks"
- ]
- if candidate_cols:
- for col in candidate_cols:
- if df[col].notna().sum() == 0:
- continue
- for src, raw in zip(df["url"], df[col].fillna("")):
- for t in parse_links_serialized(raw):
- if not t:
- continue
- if same_domain_only and urlparse(src).netloc != urlparse(t).netloc:
- continue
- edges.append((src, t))
- if edges:
- return edges
-
- session = requests.Session()
- session.headers.update({"User-Agent": "WebsiteProfiling/1.0"})
- urls = df["url"].tolist()[:max_fetch_for_edges]
- mode = (render_mode or "static").strip().lower()
- use_js = mode in ("javascript", "auto")
- fetcher = None
- if use_js:
- from ..crawl.fetchers import build_fetcher
-
- fetcher = build_fetcher(
- render_mode="javascript" if mode == "javascript" else "auto",
- timeout=timeout,
- user_agent="WebsiteProfiling/1.0",
- session=session,
- js_timeout=js_timeout,
- js_concurrency=js_concurrency,
- js_wait_until=js_wait_until,
- js_extra_wait_ms=js_extra_wait_ms,
- js_block_resources=js_block_resources,
- )
-
- def fetch(src):
- try:
- if fetcher is not None:
- r = fetcher.fetch(src)
- if r.status != 200 or not r.text:
- return []
- html = r.text
- else:
- resp = session.get(src, timeout=timeout, allow_redirects=True)
- if resp.status_code != 200 or not resp.headers.get("Content-Type", "").lower().startswith("text/html"):
- return []
- html = resp.text
- soup = BeautifulSoup(html, "lxml")
- out = set()
- for a in soup.find_all("a", href=True):
- ln = normalize_link(src, a["href"])
- if not ln or (same_domain_only and urlparse(src).netloc != urlparse(ln).netloc):
- continue
- out.add(ln)
- if polite_delay:
- time.sleep(polite_delay)
- return list(out)
- except Exception:
- return []
-
- try:
- with ThreadPoolExecutor(max_workers=concurrency) as ex:
- futures = {ex.submit(fetch, u): u for u in urls}
- for f in tqdm(as_completed(futures), total=len(futures), desc="Extracting links"):
- src = futures[f]
- try:
- outs = f.result()
- except Exception:
- outs = []
- for t in outs:
- edges.append((src, t))
- finally:
- if fetcher is not None:
- fetcher.close()
- return edges
-
-
-def _fetch_site_level(start_url: str, timeout: int = 8) -> dict:
- """Fetch robots.txt, sitemap.xml, ads.txt, and security.txt from start_url origin."""
- from .site_files import fetch_ads_txt, fetch_security_txt, merge_site_file_fields
-
- parsed = urlparse(start_url)
- if not parsed.scheme or not parsed.netloc:
- return {
- "robots_present": False,
- "sitemap_present": False,
- "sitemap_valid": False,
- "ads_txt_present": False,
- "security_txt_present": False,
- }
- base = f"{parsed.scheme}://{parsed.netloc}"
- session = requests.Session()
- session.headers.update({"User-Agent": "WebsiteProfiling/1.0"})
- out: dict[str, Any] = {
- "robots_present": False,
- "sitemap_present": False,
- "sitemap_valid": False,
- }
- try:
- r = session.get(f"{base}/robots.txt", timeout=timeout)
- if r.status_code == 200 and r.text:
- out["robots_present"] = True
- for line in r.text.splitlines():
- line = line.strip()
- if line.lower().startswith("sitemap:"):
- break
- except Exception:
- pass
- try:
- r = session.get(f"{base}/sitemap.xml", timeout=timeout)
- if r.status_code == 200 and r.text:
- out["sitemap_present"] = True
- out["sitemap_valid"] = "<" in r.text and ">" in r.text and ("urlset" in r.text or "sitemapindex" in r.text)
- except Exception:
- pass
- merge_site_file_fields(out, fetch_ads_txt(session, base, timeout=timeout))
- merge_site_file_fields(out, fetch_security_txt(session, base, timeout=timeout))
- return out
-
-
-def _compute_summary_seo_issues(df: pd.DataFrame) -> dict:
- """Compute crawl summary, SEO health metrics, issues list, and recommendations from crawl DataFrame."""
- total = len(df)
- status_str = df["status"].astype(str) if "status" in df.columns else pd.Series(["unknown"] * len(df))
- count_2xx = int((status_str.str.match(r"2\d{2}").fillna(False)).sum())
- count_3xx = int((status_str.str.match(r"3\d{2}").fillna(False)).sum())
- count_4xx = int((status_str.str.match(r"4\d{2}").fillna(False)).sum())
- count_5xx = int((status_str.str.match(r"5\d{2}").fillna(False)).sum())
- count_error = int((status_str.isin(["error", "blocked_by_robots"])).sum())
- success_rate = round(100 * count_2xx / total, 1) if total else 0
-
- outlinks = (
- pd.to_numeric(df["outlinks"], errors="coerce").fillna(0).astype(int)
- if "outlinks" in df.columns
- else pd.Series([0] * len(df))
- )
- title_len = (
- df["title"].fillna("").astype(str).apply(len)
- if "title" in df.columns
- else pd.Series([0] * len(df))
- )
- crawl_time_s = float(df["crawl_time_s"].iloc[0]) if "crawl_time_s" in df.columns and len(df) else None
-
- summary = {
- "total_urls": total,
- "count_2xx": count_2xx,
- "count_3xx": count_3xx,
- "count_4xx": count_4xx,
- "count_5xx": count_5xx,
- "count_error": count_error,
- "success_rate": success_rate,
- "avg_outlinks": round(float(outlinks.mean()), 1) if total else 0,
- "avg_title_len": round(float(title_len.mean()), 1) if total else 0,
- "crawl_time_s": round(crawl_time_s, 1) if crawl_time_s is not None else None,
- }
-
- # SEO health (when columns exist)
- seo_health = {}
- if "title" in df.columns:
- titles = df["title"].fillna("").astype(str)
- seo_health["missing_title"] = int((titles.str.len() == 0).sum())
- seo_health["title_short"] = int(((title_len > 0) & (title_len < TITLE_LEN_MIN)).sum())
- seo_health["title_long"] = int((title_len > TITLE_LEN_MAX).sum())
- seo_health["title_ok"] = int(((title_len >= TITLE_LEN_MIN) & (title_len <= TITLE_LEN_MAX)).sum())
- if "meta_description_len" in df.columns:
- md_len = pd.to_numeric(df["meta_description_len"], errors="coerce").fillna(0).astype(int)
- seo_health["missing_meta_desc"] = int((md_len == 0).sum())
- seo_health["meta_desc_short"] = int(((md_len > 0) & (md_len < META_DESC_LEN_MIN)).sum())
- seo_health["meta_desc_long"] = int((md_len > META_DESC_LEN_MAX).sum())
- seo_health["meta_desc_ok"] = int(((md_len >= META_DESC_LEN_MIN) & (md_len <= META_DESC_LEN_MAX)).sum())
- if "h1_count" in df.columns:
- h1c = pd.to_numeric(df["h1_count"], errors="coerce").fillna(-1).astype(int)
- seo_health["h1_zero"] = int((h1c == 0).sum())
- seo_health["h1_one"] = int((h1c == 1).sum())
- seo_health["h1_multi"] = int((h1c > 1).sum())
- if "content_length" in df.columns:
- cl = pd.to_numeric(df["content_length"], errors="coerce").fillna(0).astype(int)
- seo_health["thin_content"] = int(((cl > 0) & (cl < THIN_CONTENT_CHARS)).sum())
-
- # Issues: broken, redirects, SEO
- issues = {"broken": [], "redirects": [], "seo": []}
- for _, row in df.iterrows():
- u = row.get("url")
- if pd.isna(u) or not u:
- continue
- u = str(u).strip()
- st = str(row.get("status", "")).strip()
- if st.startswith("4") or st.startswith("5") or st in ("error", "blocked_by_robots"):
- issues["broken"].append({"url": u, "status": st})
- elif st.startswith("3"):
- final = row.get("final_url") or ""
- issues["redirects"].append({"url": u, "status": st, "final_url": str(final) if pd.notna(final) else ""})
-
- if "title" in df.columns:
- for _, row in df.iterrows():
- u = row.get("url")
- if pd.isna(u):
- continue
- u = str(u).strip()
- t = row.get("title") or ""
- tl = len(str(t).strip())
- if tl == 0:
- issues["seo"].append({"type": "missing_title", "url": u, "message": "Missing title"})
- elif tl < TITLE_LEN_MIN:
- issues["seo"].append({"type": "title_short", "url": u, "message": f"Title too short ({tl} chars)"})
- elif tl > TITLE_LEN_MAX:
- issues["seo"].append({"type": "title_long", "url": u, "message": f"Title too long ({tl} chars)"})
- if "meta_description_len" in df.columns:
- for _, row in df.iterrows():
- md_len = pd.to_numeric(row.get("meta_description_len"), errors="coerce")
- if pd.isna(md_len) or md_len == 0:
- continue
- u = row.get("url")
- if pd.isna(u):
- continue
- u = str(u).strip()
- ml = int(md_len)
- if ml < META_DESC_LEN_MIN:
- issues["seo"].append({"type": "meta_desc_short", "url": u, "message": f"Meta description too short ({ml} chars)"})
- elif ml > META_DESC_LEN_MAX:
- issues["seo"].append({"type": "meta_desc_long", "url": u, "message": f"Meta description too long ({ml} chars)"})
- if "h1_count" in df.columns:
- for _, row in df.iterrows():
- h1c = pd.to_numeric(row.get("h1_count"), errors="coerce")
- if pd.isna(h1c) or h1c == 1:
- continue
- u = row.get("url")
- if pd.isna(u):
- continue
- u = str(u).strip()
- if int(h1c) == 0:
- issues["seo"].append({"type": "h1_missing", "url": u, "message": "Missing H1"})
- else:
- issues["seo"].append({"type": "h1_multi", "url": u, "message": f"Multiple H1s ({int(h1c)})"})
- if "content_length" in df.columns:
- for _, row in df.iterrows():
- cl = pd.to_numeric(row.get("content_length"), errors="coerce")
- cl = 0 if pd.isna(cl) else int(cl)
- if cl >= THIN_CONTENT_CHARS or cl == 0:
- continue
- u = row.get("url")
- if pd.isna(u):
- continue
- issues["seo"].append({"type": "thin_content", "url": str(u).strip(), "message": f"Thin content ({int(cl)} chars)"})
-
- # Recommendations (actionable bullets)
- recommendations = []
- if issues["broken"]:
- recommendations.append(f"Fix {len(issues['broken'])} broken or error URL(s).")
- if issues["redirects"]:
- recommendations.append(f"Review {len(issues['redirects'])} redirect(s); consolidate if possible.")
- if seo_health.get("missing_title", 0) > 0:
- recommendations.append(f"Add titles to {seo_health['missing_title']} page(s).")
- if seo_health.get("title_short", 0) + seo_health.get("title_long", 0) > 0:
- n = seo_health.get("title_short", 0) + seo_health.get("title_long", 0)
- recommendations.append(f"Optimize title length on {n} page(s) (aim 30–60 chars).")
- if seo_health.get("missing_meta_desc", 0) > 0:
- recommendations.append(f"Add meta descriptions to {seo_health['missing_meta_desc']} page(s).")
- if seo_health.get("meta_desc_short", 0) + seo_health.get("meta_desc_long", 0) > 0:
- n = seo_health.get("meta_desc_short", 0) + seo_health.get("meta_desc_long", 0)
- recommendations.append(f"Optimize meta description length on {n} page(s) (aim 70–160 chars).")
- if seo_health.get("h1_zero", 0) > 0:
- recommendations.append(f"Add one H1 per page on {seo_health['h1_zero']} page(s).")
- if seo_health.get("h1_multi", 0) > 0:
- recommendations.append(f"Use a single H1 per page on {seo_health['h1_multi']} page(s).")
- if seo_health.get("thin_content", 0) > 0:
- recommendations.append(f"Expand thin content on {seo_health['thin_content']} page(s) (under {THIN_CONTENT_CHARS} chars).")
-
- return {
- "summary": summary,
- "seo_health": seo_health,
- "issues": issues,
- "recommendations": recommendations,
- }
-
-
-def _build_content_analytics(df: pd.DataFrame) -> dict:
- """Build content analytics: word count stats, reading level distribution, content ratio, top keywords."""
- from collections import Counter
-
- result = {
- "word_count_stats": {"mean": 0, "median": 0, "p25": 0, "p75": 0, "min": 0, "max": 0},
- "word_count_distribution": {},
- "reading_level_distribution": {},
- "content_ratio_distribution": {},
- "top_keywords_site": [],
- "thin_pages": [],
- }
- if "word_count" not in df.columns or df.empty:
- return result
-
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df
- if success_df.empty:
- return result
-
- wc = pd.to_numeric(success_df["word_count"], errors="coerce").fillna(0).astype(int)
- result["word_count_stats"] = {
- "mean": round(float(wc.mean()), 1),
- "median": round(float(wc.median()), 1),
- "p25": round(float(wc.quantile(0.25)), 1),
- "p75": round(float(wc.quantile(0.75)), 1),
- "min": int(wc.min()),
- "max": int(wc.max()),
- }
-
- wc_bins = [(0, 100), (101, 300), (301, 600), (601, 1000), (1001, 2000), (2001, 999999)]
- wc_labels = ["0-100", "101-300", "301-600", "601-1000", "1001-2000", "2001+"]
- result["word_count_distribution"] = {
- lbl: int(((wc >= lo) & (wc <= hi)).sum()) for (lo, hi), lbl in zip(wc_bins, wc_labels)
- }
-
- if "reading_level" in success_df.columns:
- rl = pd.to_numeric(success_df["reading_level"], errors="coerce").fillna(0)
- rl_bins = [(0, 5), (6, 8), (9, 12), (13, 99)]
- rl_labels = ["Elementary (0-5)", "Middle School (6-8)", "High School (9-12)", "College (13+)"]
- result["reading_level_distribution"] = {
- lbl: int(((rl >= lo) & (rl <= hi)).sum()) for (lo, hi), lbl in zip(rl_bins, rl_labels)
- }
-
- if "content_html_ratio" in success_df.columns:
- cr = pd.to_numeric(success_df["content_html_ratio"], errors="coerce").fillna(0)
- cr_bins = [(0, 10), (10.01, 20), (20.01, 40), (40.01, 100)]
- cr_labels = ["<10%", "10-20%", "20-40%", ">40%"]
- result["content_ratio_distribution"] = {
- lbl: int(((cr >= lo) & (cr <= hi)).sum()) for (lo, hi), lbl in zip(cr_bins, cr_labels)
- }
-
- if "top_keywords" in success_df.columns:
- kw_counter = Counter()
- for raw in success_df["top_keywords"].fillna("[]"):
- try:
- items = json.loads(str(raw)) if isinstance(raw, str) else raw
- if isinstance(items, list):
- for item in items:
- if isinstance(item, dict):
- kw_counter[item.get("word", "")] += item.get("count", 0)
- except (json.JSONDecodeError, TypeError):
- pass
- result["top_keywords_site"] = [
- {"word": w, "count": c}
- for w, c in kw_counter.most_common(50)
- if w and not is_junk_semantic_term(str(w))
- ][:30]
-
- for _, row in success_df.iterrows():
- u = row.get("url")
- if pd.isna(u) or not u:
- continue
- w = int(pd.to_numeric(row.get("word_count"), errors="coerce") or 0)
- if 0 < w < 300:
- result["thin_pages"].append({"url": str(u).strip(), "word_count": w})
-
- return result
-
-
-def _parse_top_keywords_items(raw: Any) -> list[dict[str, Any]]:
- """Parse per-page top_keywords JSON into dict items with word/count."""
- if raw is None or (isinstance(raw, float) and pd.isna(raw)):
- return []
- try:
- items = json.loads(str(raw)) if isinstance(raw, str) else raw
- except (json.JSONDecodeError, TypeError, ValueError):
- return []
- if not isinstance(items, list):
- return []
- out: list[dict[str, Any]] = []
- for item in items:
- if isinstance(item, dict):
- word = str(item.get("word") or "").strip()
- if word:
- out.append({"word": word, "count": int(item.get("count") or 1)})
- return out
-
-
-def _build_text_content_analysis(df: pd.DataFrame) -> dict:
- """Cross-page keyword aggregates for the text content analysis view."""
- empty = {
- "vocabulary_stats": {
- "unique_terms": 0,
- "pages_with_keywords": 0,
- "avg_terms_per_page": 0.0,
- "total_term_occurrences": 0,
- },
- "keyword_index": [],
- "keyword_frequency_histogram": {"1": 0, "2-5": 0, "6-20": 0, "21+": 0},
- }
- if df.empty or "top_keywords" not in df.columns:
- return empty
-
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df
- if success_df.empty:
- return empty
-
- # word -> { total_count, pages: { url -> count } }
- index: dict[str, dict[str, Any]] = {}
- pages_with_keywords = 0
- total_occurrences = 0
-
- for _, row in success_df.iterrows():
- url = row.get("url")
- if pd.isna(url) or not url:
- continue
- url_str = str(url).strip()
- items = _parse_top_keywords_items(row.get("top_keywords"))
- page_had_kw = False
- for item in items:
- word = item["word"].lower()
- if is_junk_semantic_term(word):
- continue
- count = max(1, int(item.get("count") or 1))
- if word not in index:
- index[word] = {"total_count": 0, "pages": {}}
- index[word]["total_count"] += count
- index[word]["pages"][url_str] = index[word]["pages"].get(url_str, 0) + count
- total_occurrences += count
- page_had_kw = True
- if page_had_kw:
- pages_with_keywords += 1
-
- unique_terms = len(index)
- avg_terms = round(total_occurrences / pages_with_keywords, 1) if pages_with_keywords else 0.0
-
- histogram = {"1": 0, "2-5": 0, "6-20": 0, "21+": 0}
- for data in index.values():
- pc = len(data["pages"])
- if pc == 1:
- histogram["1"] += 1
- elif pc <= 5:
- histogram["2-5"] += 1
- elif pc <= 20:
- histogram["6-20"] += 1
- else:
- histogram["21+"] += 1
-
- sorted_words = sorted(index.items(), key=lambda x: x[1]["total_count"], reverse=True)
- keyword_index: list[dict[str, Any]] = []
- for word, data in sorted_words:
- top_pages = sorted(data["pages"].items(), key=lambda x: x[1], reverse=True)[:5]
- keyword_index.append(
- {
- "word": word,
- "total_count": data["total_count"],
- "page_count": len(data["pages"]),
- "top_pages": [{"url": u, "count": c} for u, c in top_pages],
- }
- )
-
- return {
- "vocabulary_stats": {
- "unique_terms": unique_terms,
- "pages_with_keywords": pages_with_keywords,
- "avg_terms_per_page": avg_terms,
- "total_term_occurrences": total_occurrences,
- },
- "keyword_index": keyword_index,
- "keyword_frequency_histogram": histogram,
- }
-
-
-def _build_social_coverage(df: pd.DataFrame) -> dict:
- """Build social meta coverage stats: OG and Twitter Card presence percentages."""
- result = {
- "og_coverage_pct": 0,
- "twitter_coverage_pct": 0,
- "og_image_coverage_pct": 0,
- "missing_og": [],
- "missing_twitter": [],
- "og_image_missing": [],
- }
- if df.empty:
- return result
-
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df
- html_df = success_df
- if "content_type" in success_df.columns:
- html_df = success_df[success_df["content_type"].fillna("").str.contains("text/html", case=False, na=False)]
- if html_df.empty:
- return result
-
- total = len(html_df)
-
- if "og_title" in html_df.columns:
- has_og = (html_df["og_title"].fillna("").astype(str).str.strip() != "").sum()
- result["og_coverage_pct"] = round(100 * int(has_og) / total, 1)
- for _, row in html_df.iterrows():
- u = row.get("url")
- if pd.isna(u):
- continue
- u = str(u).strip()
- og = str(row.get("og_title") or "").strip()
- if not og:
- result["missing_og"].append(u)
-
- if "twitter_card" in html_df.columns:
- has_tw = (html_df["twitter_card"].fillna("").astype(str).str.strip() != "").sum()
- result["twitter_coverage_pct"] = round(100 * int(has_tw) / total, 1)
- for _, row in html_df.iterrows():
- u = row.get("url")
- if pd.isna(u):
- continue
- u = str(u).strip()
- tw = str(row.get("twitter_card") or "").strip()
- if not tw:
- result["missing_twitter"].append(u)
-
- if "og_image" in html_df.columns:
- has_og_img = (html_df["og_image"].fillna("").astype(str).str.strip() != "").sum()
- result["og_image_coverage_pct"] = round(100 * int(has_og_img) / total, 1)
- for _, row in html_df.iterrows():
- u = row.get("url")
- if pd.isna(u):
- continue
- u = str(u).strip()
- img = str(row.get("og_image") or "").strip()
- if not img:
- result["og_image_missing"].append(u)
-
- result["missing_og"] = result["missing_og"][:100]
- result["missing_twitter"] = result["missing_twitter"][:100]
- result["og_image_missing"] = result["og_image_missing"][:100]
- return result
-
-
-def _build_tech_stack_summary(df: pd.DataFrame) -> dict:
- """Build tech stack summary: detected technologies with counts and sample URLs."""
- from collections import defaultdict
-
- result = {"technologies": [], "total_pages_analyzed": 0}
- if "tech_stack" not in df.columns or df.empty:
- return result
-
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df
- html_df = success_df
- if "content_type" in success_df.columns:
- html_df = success_df[success_df["content_type"].fillna("").str.contains("text/html", case=False, na=False)]
- if html_df.empty:
- return result
-
- result["total_pages_analyzed"] = len(html_df)
- tech_urls = defaultdict(list)
-
- for _, row in html_df.iterrows():
- u = str(row.get("url", "")).strip()
- raw = row.get("tech_stack") or "[]"
- try:
- techs = json.loads(str(raw)) if isinstance(raw, str) else raw
- if isinstance(techs, list):
- for t in techs:
- if isinstance(t, str) and t:
- tech_urls[t].append(u)
- except (json.JSONDecodeError, TypeError):
- pass
-
- result["technologies"] = sorted(
- [{"name": name, "count": len(urls), "sample_urls": urls[:3]} for name, urls in tech_urls.items()],
- key=lambda x: x["count"],
- reverse=True,
- )
- return result
-
-
-def _build_response_time_stats(df: pd.DataFrame) -> dict:
- """Build response time statistics and distribution."""
- result = {
- "p25": 0, "p50": 0, "p75": 0, "p95": 0, "p99": 0,
- "slow_pages": [],
- "distribution": {},
- }
- if "response_time_ms" not in df.columns or df.empty:
- return result
-
- rt = pd.to_numeric(df["response_time_ms"], errors="coerce").dropna()
- if rt.empty:
- return result
-
- result["p25"] = round(float(rt.quantile(0.25)), 0)
- result["p50"] = round(float(rt.quantile(0.50)), 0)
- result["p75"] = round(float(rt.quantile(0.75)), 0)
- result["p95"] = round(float(rt.quantile(0.95)), 0)
- result["p99"] = round(float(rt.quantile(0.99)), 0)
-
- rt_bins = [(0, 200), (200, 500), (500, 1000), (1000, 2000), (2000, 999999)]
- rt_labels = ["<200ms", "200-500ms", "500ms-1s", "1-2s", ">2s"]
- rt_full = pd.to_numeric(df["response_time_ms"], errors="coerce").fillna(0)
- result["distribution"] = {
- lbl: int(((rt_full >= lo) & (rt_full < hi)).sum()) for (lo, hi), lbl in zip(rt_bins, rt_labels)
- }
-
- for _, row in df.iterrows():
- u = row.get("url")
- ms = pd.to_numeric(row.get("response_time_ms"), errors="coerce")
- if pd.isna(u) or pd.isna(ms) or ms <= 2000:
- continue
- result["slow_pages"].append({"url": str(u).strip(), "response_time_ms": int(ms)})
- result["slow_pages"] = sorted(result["slow_pages"], key=lambda x: x["response_time_ms"], reverse=True)[:50]
- return result
-
-
-def _build_depth_distribution(df: pd.DataFrame) -> dict:
- """Build crawl depth distribution."""
- result = {"by_depth": {}, "max_depth": 0, "avg_depth": 0}
- if "depth" not in df.columns or df.empty:
- return result
-
- depths = pd.to_numeric(df["depth"], errors="coerce").dropna().astype(int)
- if depths.empty:
- return result
-
- result["max_depth"] = int(depths.max())
- result["avg_depth"] = round(float(depths.mean()), 1)
- counts = depths.value_counts().sort_index()
- result["by_depth"] = {str(int(k)): int(v) for k, v in counts.items()}
- return result
-
-
-def _parse_page_analysis_cell(raw: object) -> dict[str, Any]:
- if raw is None or (isinstance(raw, float) and pd.isna(raw)):
- return {}
- s = str(raw).strip()
- if not s or s == "{}":
- return {}
- try:
- o = json.loads(s)
- return o if isinstance(o, dict) else {}
- except json.JSONDecodeError:
- return {}
-
-
-def _build_outbound_link_domains(
- df: pd.DataFrame,
- start_url: str,
- max_rows: int,
-) -> list[dict[str, Any]]:
- """Aggregate external hosts linked from crawled pages (outbound), not referring domains."""
- site_host = urlparse((start_url or "").strip()).netloc.lower()
- host_pages: dict[str, set[str]] = {}
- host_link_count: dict[str, int] = {}
- for _, row in df.iterrows():
- st = str(row.get("status", "")).strip()
- if st.startswith(("4", "5")):
- continue
- u = str(row.get("url") or "").strip().rstrip("/")
- if not u:
- continue
- seen_on_page: set[str] = set()
- pa = _parse_page_analysis_cell(row.get("page_analysis")) if "page_analysis" in df.columns else {}
- for link in pa.get("external_links") or []:
- if not isinstance(link, str):
- continue
- h = urlparse(link).netloc.lower()
- if not h or h == site_host:
- continue
- host_pages.setdefault(h, set()).add(u)
- host_link_count[h] = host_link_count.get(h, 0) + 1
- seen_on_page.add(link)
- if "outlink_targets" in df.columns:
- for link in parse_links_serialized(row.get("outlink_targets")):
- h = urlparse(link).netloc.lower()
- if not h or h == site_host:
- continue
- host_pages.setdefault(h, set()).add(u)
- if link not in seen_on_page:
- host_link_count[h] = host_link_count.get(h, 0) + 1
- seen_on_page.add(link)
- rows: list[dict[str, Any]] = []
- for h in host_pages:
- rows.append({
- "host": h,
- "page_count": len(host_pages[h]),
- "link_count": host_link_count.get(h, 0),
- })
- rows.sort(key=lambda x: (-x["link_count"], -x["page_count"], x["host"]))
- return rows[:max_rows]
-
-
-def _build_url_fingerprints(df: pd.DataFrame) -> list[dict[str, Any]]:
- """Stable fingerprints for comparing page content/structure between report runs (no raw HTML stored)."""
- out: list[dict[str, Any]] = []
- for _, row in df.iterrows():
- u = str(row.get("url") or "").strip().rstrip("/")
- if not u:
- continue
- title = str(row.get("title") or "")
- meta = str(row.get("meta_description") or "")
- h1 = str(row.get("h1") or "")
- headings = str(row.get("heading_sequence") or "")
- wc = int(pd.to_numeric(row.get("word_count"), errors="coerce") or 0)
- cl = int(pd.to_numeric(row.get("content_length"), errors="coerce") or 0)
- h1c = int(pd.to_numeric(row.get("h1_count"), errors="coerce") or 0)
- sc = int(pd.to_numeric(row.get("script_count"), errors="coerce") or 0)
- lc = int(pd.to_numeric(row.get("link_stylesheet_count"), errors="coerce") or 0)
- # heading_sequence is structural (h1,h2,...) — keep it in structure fingerprint only.
- raw_c = "|".join([title, meta, h1, str(wc), str(cl)]).encode("utf-8")
- content_fp = hashlib.sha256(raw_c).hexdigest()
- raw_s = "|".join([str(cl), str(sc), str(lc), str(h1c), headings]).encode("utf-8")
- structure_fp = hashlib.sha256(raw_s).hexdigest()
- out.append({
- "url": u,
- "content_fingerprint": content_fp,
- "structure_fingerprint": structure_fp,
- })
- return out
-
-
-def _build_hreflang_summary(df: pd.DataFrame) -> dict[str, Any]:
- total = 0
- missing_lang = 0
- with_hreflang = 0
- for _, row in df.iterrows():
- st = str(row.get("status", "")).strip()
- if not st.startswith("2"):
- continue
- total += 1
- pa = _parse_page_analysis_cell(row.get("page_analysis")) if "page_analysis" in df.columns else {}
- if not (pa.get("html_lang") or "").strip():
- missing_lang += 1
- if pa.get("hreflang_alternates"):
- with_hreflang += 1
- return {
- "pages_200": total,
- "pages_missing_html_lang": missing_lang,
- "pages_with_hreflang_links": with_hreflang,
- }
-
-
-def _validate_report_url_counts(report_data: dict[str, Any], df_row_count: int) -> None:
- """Ensure crawled URL counts are consistent across report payload fields."""
- links = report_data.get("links") or []
- summary = report_data.get("summary") or {}
- scope = (report_data.get("report_meta") or {}).get("crawl_scope") or {}
- link_count = len(links) if isinstance(links, list) else 0
- total_urls = int(summary.get("total_urls") or 0)
- pages_crawled = int(scope.get("pages_crawled") or 0)
- counts = {link_count, total_urls, pages_crawled, df_row_count}
- if len(counts) > 1:
- msg = (
- f"report count mismatch: links={link_count}, "
- f"summary.total_urls={total_urls}, "
- f"pages_crawled={pages_crawled}, df_rows={df_row_count}"
- )
- print(f" WARNING: {msg}", flush=True)
- report_data.setdefault("ml_errors", []).append(msg)
-
-
-def _build_report_metadata(
- df: pd.DataFrame,
- config: Optional[dict[str, str]],
- lighthouse_summary: Optional[dict[str, Any]],
- google_data: Optional[dict[str, Any]],
- keywords_data: Optional[dict[str, Any]],
- ml_bundle: dict[str, Any],
- run_id: Optional[int],
- crawl_run_created_at: Optional[str],
- gsc_links_data: Optional[dict[str, Any]] = None,
-) -> dict[str, Any]:
- """Provenance and crawl scope for agency-facing audits."""
- sources: list[str] = ["crawl"]
- if lighthouse_summary:
- sources.append("lighthouse")
- if google_data:
- if google_data.get("gsc") or google_data.get("gsc_summary"):
- sources.append("search_console")
- if google_data.get("ga4") or google_data.get("ga4_summary"):
- sources.append("analytics")
- if gsc_links_data and "search_console" not in sources:
- sources.append("search_console")
- llm_meta = ml_bundle.get("llm_meta")
- if isinstance(llm_meta, dict) and llm_meta.get("model"):
- sources.append("ai")
- kw_rows = (keywords_data or {}).get("rows") or []
- has_gsc_kw = any(
- (r.get("gsc_impressions") or r.get("gsc_clicks")) and r.get("source") in ("gsc", "site+gsc", None)
- for r in kw_rows[:500]
- if isinstance(r, dict)
- )
- if kw_rows and not has_gsc_kw and "estimated" not in sources:
- sources.append("estimated")
-
- max_pages_cfg = get_int(config or {}, "max_pages", 0) or 0
- pages_crawled = len(df)
- blocked = 0
- if not df.empty and "status" in df.columns:
- blocked = int((df["status"].astype(str) == "blocked_by_robots").sum())
-
- render_mode = (str((config or {}).get("crawl_render_mode") or "static")).strip().lower()
- js_concurrency = get_int(config or {}, "crawl_js_concurrency", 3) or 3
- static_html_only = render_mode == "static"
-
- crawl_scope: dict[str, Any] = {
- "pages_crawled": pages_crawled,
- "max_pages_configured": max_pages_cfg or pages_crawled,
- "robots_blocked_count": blocked,
- "static_html_only": static_html_only,
- "render_mode": render_mode,
- "js_concurrency": js_concurrency if not static_html_only else None,
- "crawl_limited": bool(max_pages_cfg and pages_crawled >= max_pages_cfg),
- }
- if not df.empty and "fetch_method" in df.columns:
- fm = df["fetch_method"].astype(str).str.strip().str.lower()
- pages_static = int((fm == "static").sum())
- pages_rendered = int((fm == "rendered").sum())
- if render_mode == "auto" or pages_rendered > 0:
- crawl_scope["pages_static"] = pages_static
- crawl_scope["pages_rendered"] = pages_rendered
-
- from ..crawl.fetchers.browser_diagnostics import aggregate_browser_diagnostics_df
-
- browser_agg = aggregate_browser_diagnostics_df(df)
- if browser_agg and (render_mode != "static" or browser_agg.get("total_console_errors", 0) > 0):
- crawl_scope["browser_diagnostics"] = browser_agg
-
- meta: dict[str, Any] = {
- "data_sources": sources,
- "generated_at": datetime.now(timezone.utc).isoformat(),
- "crawl_scope": crawl_scope,
- }
- if run_id is not None:
- meta["crawl_run_id"] = run_id
- if crawl_run_created_at:
- meta["crawl_run_created_at"] = crawl_run_created_at
- if google_data:
- meta["google_fetched_at"] = google_data.get("fetched_at")
- meta["google_date_range_days"] = google_data.get("date_range_days")
- gsc = google_data.get("gsc") or {}
- if isinstance(gsc, dict) and gsc.get("row_count") is not None:
- meta["gsc_row_count"] = gsc.get("row_count")
- if keywords_data:
- meta["keywords_enriched_at"] = keywords_data.get("enriched_at") or keywords_data.get("fetched_at")
- if gsc_links_data:
- meta["gsc_links_imported_at"] = gsc_links_data.get("imported_at")
- meta["gsc_links_referring_domains"] = len(gsc_links_data.get("top_linking_sites") or [])
- sample_n = len(gsc_links_data.get("sample_links") or [])
- latest_n = len(gsc_links_data.get("latest_links") or [])
- meta["gsc_links_sample_count"] = sample_n + latest_n
- if isinstance(llm_meta, dict):
- meta["llm"] = llm_meta
- logo_url = (str((config or {}).get("export_logo_url") or "")).strip()
- if logo_url:
- meta["export_logo_url"] = logo_url
- return meta
-
-
-def _build_keyword_opportunities(df: pd.DataFrame, config: dict[str, str] | None) -> dict[str, Any]:
- if not get_bool(config or {}, "include_keyword_opportunities", True):
- return {}
- if "status" not in df.columns or df.empty:
- return {"quick_wins": [], "high_value": [], "token_topic_clusters": []}
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)]
- if success_df.empty:
- return {"quick_wins": [], "high_value": [], "token_topic_clusters": []}
- candidates = extract_candidates_from_df(success_df)
- if not candidates:
- return {"quick_wins": [], "high_value": [], "token_topic_clusters": []}
- corpus_size = len(success_df)
- scored = score_keywords(candidates, corpus_size=corpus_size)
- clusters = cluster_keywords(scored)
- quick_wins = [s for s in scored if s.get("difficulty", 100) < 60][:10]
- high_value = [s for s in scored if (s.get("volume") or 0) >= 0.5][:10]
- if not high_value:
- high_value = scored[:10]
- return {
- "quick_wins": quick_wins[:10],
- "high_value": high_value[:10],
- "token_topic_clusters": filter_topic_clusters(clusters)[:50],
- }
-
-
-def _build_image_inventory(
- links: list[dict[str, Any]],
- config: Optional[dict[str, str]],
-) -> tuple[list[dict[str, Any]], dict[str, Any]]:
- from ..analysis.image_probe import collect_image_refs_from_links, probe_image_urls
-
- refs = collect_image_refs_from_links(links)
- unoptimized_min_kb = get_int(config or {}, "image_unoptimized_min_kb", 200) or 200
- summary: dict[str, Any] = {
- "probed": 0,
- "failed": 0,
- "total_bytes": 0,
- "over_threshold_count": 0,
- "unoptimized_min_kb": unoptimized_min_kb,
- "inventory_available": False,
- }
- if not get_bool(config or {}, "probe_image_inventory", False):
- return [], summary
-
- max_urls = get_int(config or {}, "max_image_probe_urls", 500) or 500
- concurrency = get_int(config or {}, "image_probe_concurrency", 6) or 6
- probe_timeout = get_int(config or {}, "image_probe_timeout", 8) or 8
- url_list = list(refs.keys())[:max_urls]
- if not url_list:
- return [], summary
-
- print(f" Probing up to {len(url_list)} image URL(s)...", flush=True)
- probed = probe_image_urls(
- url_list,
- concurrency=concurrency,
- timeout=probe_timeout,
- )
- threshold_bytes = unoptimized_min_kb * 1024
- inventory: list[dict[str, Any]] = []
- for row in probed:
- url = row.get("url")
- meta = refs.get(str(url or ""), {"source_pages": set(), "kinds": set()})
- size = row.get("size_bytes")
- entry = {
- "url": url,
- "status": row.get("status"),
- "content_type": row.get("content_type"),
- "size_bytes": size,
- "error": row.get("error"),
- "source_pages": sorted(meta.get("source_pages") or []),
- "kinds": sorted(meta.get("kinds") or []),
- }
- inventory.append(entry)
- summary["probed"] += 1
- if row.get("error") or row.get("status") is None:
- summary["failed"] += 1
- if size is not None:
- summary["total_bytes"] += int(size)
- if int(size) >= threshold_bytes:
- summary["over_threshold_count"] += 1
- summary["inventory_available"] = True
- print(f" Image probe complete ({summary['probed']} URLs, {summary['failed']} failed).", flush=True)
- return inventory, summary
-
+from .categories import build_categories
+from .content_analytics import (
+ _build_content_analytics,
+ _build_depth_distribution,
+ _build_image_inventory,
+ _build_keyword_opportunities,
+ _build_response_time_stats,
+ _build_social_coverage,
+ _build_tech_stack_summary,
+ _build_text_content_analysis,
+ _parse_top_keywords_items,
+)
+from .edges_report import build_edges_from_df
+from .lighthouse_report import (
+ _derive_expected_host,
+ _pick_lighthouse_summary,
+ build_lighthouse_by_url_for_report,
+ fetch_site_ssl_expires_iso,
+ filter_lighthouse_by_host,
+ lighthouse_for_url,
+)
+from .report_metadata import (
+ _build_hreflang_summary,
+ _build_outbound_link_domains,
+ _build_report_metadata,
+ _build_url_fingerprints,
+ _parse_page_analysis_cell,
+ _validate_report_url_counts,
+)
+from .seo_summary import (
+ META_DESC_LEN_MAX,
+ META_DESC_LEN_MIN,
+ THIN_CONTENT_CHARS,
+ TITLE_LEN_MAX,
+ TITLE_LEN_MIN,
+ _compute_summary_seo_issues,
+)
+from .site_level import _fetch_site_level
+
+# Backward-compatible re-exports for tests and external imports.
+__all__ = [
+ "run_simple_report",
+ "build_edges_from_df",
+ "build_lighthouse_by_url_for_report",
+ "fetch_site_ssl_expires_iso",
+ "filter_lighthouse_by_host",
+ "lighthouse_for_url",
+ "_fetch_site_level",
+ "_compute_summary_seo_issues",
+ "_build_content_analytics",
+ "_build_text_content_analysis",
+ "_build_image_inventory",
+ "_build_report_metadata",
+]
def run_simple_report(
max_fetch_for_edges: int = 300,
@@ -1737,6 +633,79 @@ def _bool_col(col):
"images_total": int(pd.to_numeric(row.get("images_total"), errors="coerce") or 0),
})
+ title_short: list[dict[str, Any]] = []
+ title_long: list[dict[str, Any]] = []
+ if "title" in df.columns:
+ titles = df["title"].fillna("").astype(str)
+ tl = titles.str.len()
+ for i, row in df.iterrows():
+ u = row.get("url")
+ if pd.isna(u) or not u:
+ continue
+ u = str(u).strip()
+ title_str = titles.iloc[i].strip()
+ n = int(tl.iloc[i])
+ if n == 0:
+ continue
+ if n < TITLE_LEN_MIN:
+ title_short.append({"url": u, "title": title_str, "title_length": n})
+ elif n > TITLE_LEN_MAX:
+ title_long.append({"url": u, "title": title_str, "title_length": n})
+
+ slow_response: list[dict[str, Any]] = []
+ if "response_time_ms" in df.columns:
+ rt = pd.to_numeric(df["response_time_ms"], errors="coerce")
+ for i, row in df.iterrows():
+ ms = rt.iloc[i]
+ if pd.isna(ms) or float(ms) <= 2000:
+ continue
+ u = row.get("url")
+ if pd.isna(u) or not u:
+ continue
+ slow_response.append({"url": str(u).strip(), "response_time_ms": int(ms)})
+
+ missing_html_lang: list[dict[str, Any]] = []
+ invalid_viewport: list[dict[str, Any]] = []
+ if "html_lang" in success_df_urls.columns:
+ for _, row in success_df_urls.iterrows():
+ u = row.get("url")
+ if pd.isna(u) or not u:
+ continue
+ lang = str(row.get("html_lang") or "").strip()
+ if not lang:
+ missing_html_lang.append({"url": str(u).strip()})
+ if "viewport_present" in success_df_urls.columns:
+ vp = success_df_urls["viewport_present"]
+ for _, row in success_df_urls.iterrows():
+ u = row.get("url")
+ if pd.isna(u) or not u:
+ continue
+ if not bool(row.get("viewport_present")):
+ invalid_viewport.append({"url": str(u).strip()})
+
+ high_reading_level: list[dict[str, Any]] = []
+ very_thin_content: list[dict[str, Any]] = []
+ if "reading_level" in success_df_urls.columns:
+ rl = pd.to_numeric(success_df_urls["reading_level"], errors="coerce")
+ for i, row in success_df_urls.iterrows():
+ val = rl.loc[i]
+ if pd.isna(val) or float(val) <= 12:
+ continue
+ u = row.get("url")
+ if pd.isna(u) or not u:
+ continue
+ high_reading_level.append({"url": str(u).strip(), "reading_level": float(val)})
+ if "word_count" in success_df_urls.columns:
+ wc = pd.to_numeric(success_df_urls["word_count"], errors="coerce").fillna(0).astype(int)
+ for i, row in success_df_urls.iterrows():
+ w = int(wc.loc[i])
+ if w <= 0 or w >= 100:
+ continue
+ u = row.get("url")
+ if pd.isna(u) or not u:
+ continue
+ very_thin_content.append({"url": str(u).strip(), "word_count": w})
+
content_urls = {
"missing_h1": missing_h1,
"missing_title": missing_title,
@@ -1750,6 +719,13 @@ def _bool_col(col):
"missing_alt": missing_alt,
"missing_lazy": missing_lazy,
"missing_dimensions": missing_dimensions,
+ "title_short": title_short,
+ "title_long": title_long,
+ "slow_response": slow_response,
+ "missing_html_lang": missing_html_lang,
+ "invalid_viewport": invalid_viewport,
+ "high_reading_level": high_reading_level,
+ "very_thin_content": very_thin_content,
}
emit_progress("report", "content_analytics", message="Building content analytics")
@@ -1781,6 +757,65 @@ def _bool_col(col):
depth_distribution = _build_depth_distribution(df)
image_inventory, image_inventory_summary = _build_image_inventory(links, config)
+ hreflang_issue_urls: list[dict[str, Any]] = []
+ try:
+ from .categories._helpers import _hreflang_issues
+
+ for issue in _hreflang_issues(success_df_urls if len(success_df_urls) else df):
+ hreflang_issue_urls.append({
+ "url": issue.get("url") or "",
+ "message": issue.get("message") or "",
+ "priority": issue.get("priority") or "Medium",
+ })
+ except Exception:
+ hreflang_issue_urls = []
+
+ lighthouse_failure_urls: dict[str, list[dict[str, Any]]] = {
+ "lcp": [], "inp": [], "cls": [], "seo": [],
+ }
+ if lighthouse_by_url:
+ audit_map = {
+ "lcp": "largest-contentful-paint",
+ "inp": "interaction-to-next-paint",
+ "cls": "cumulative-layout-shift",
+ "seo": "seo",
+ }
+ for url, lh in lighthouse_by_url.items():
+ if not isinstance(lh, dict):
+ continue
+ audits = lh.get("audits") if isinstance(lh.get("audits"), dict) else {}
+ for bucket, audit_id in audit_map.items():
+ audit = audits.get(audit_id) if isinstance(audits, dict) else None
+ if not isinstance(audit, dict):
+ continue
+ score = audit.get("score")
+ if score is not None and float(score) < 0.9:
+ lighthouse_failure_urls[bucket].append({
+ "url": str(url),
+ "score": score,
+ "displayValue": audit.get("displayValue"),
+ })
+
+ optional_audit_urls: dict[str, list[dict[str, Any]]] = {
+ "spell": [], "html": [], "amp": [], "pagination": [],
+ }
+ for cat in categories:
+ if not isinstance(cat, dict):
+ continue
+ for issue in cat.get("issues") or []:
+ if not isinstance(issue, dict):
+ continue
+ msg = str(issue.get("message") or "").lower()
+ rec = {"url": issue.get("url") or "", "message": issue.get("message") or ""}
+ if "spell" in msg:
+ optional_audit_urls["spell"].append(rec)
+ elif "html" in msg and "validation" in msg:
+ optional_audit_urls["html"].append(rec)
+ elif "amp" in msg:
+ optional_audit_urls["amp"].append(rec)
+ elif "pagination" in msg or "rel=prev" in msg or "rel=next" in msg:
+ optional_audit_urls["pagination"].append(rec)
+
report_data = {
"site_name": site_display,
"report_title": report_display_title,
@@ -1809,6 +844,9 @@ def _bool_col(col):
"top_pages": top_pages,
"links": links,
"content_urls": content_urls,
+ "hreflang_issue_urls": hreflang_issue_urls,
+ "lighthouse_failure_urls": lighthouse_failure_urls,
+ "optional_audit_urls": optional_audit_urls,
"security_findings": security_findings,
"content_analytics": content_analytics,
"text_content_analysis": text_content_analysis,
diff --git a/src/website_profiling/reporting/categories.py b/src/website_profiling/reporting/categories.py
deleted file mode 100644
index 15014e7d..00000000
--- a/src/website_profiling/reporting/categories.py
+++ /dev/null
@@ -1,1053 +0,0 @@
-"""
-Report categories for site audits: Technical SEO, Core Web Vitals, Performance,
-Accessibility & markup, Links, Mobile SEO, Security, Content quality.
-"""
-import json
-from typing import Any, Optional
-from urllib.parse import urlparse
-
-import pandas as pd
-
-from .terminology import (
- CATEGORY_ACCESSIBILITY,
- CATEGORY_CONTENT_QUALITY,
- CATEGORY_CORE_WEB_VITALS,
- CATEGORY_LINKS,
- CATEGORY_MOBILE,
- CATEGORY_PERFORMANCE,
- CATEGORY_SECURITY,
- CATEGORY_TECHNICAL_SEO,
-)
-
-# Priority order for sorting
-PRIORITY_ORDER = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3}
-
-# Thresholds
-RESPONSE_TIME_SLOW_MS = 2000
-THIN_CONTENT_CHARS = 300
-TITLE_LEN_MIN = 30
-TITLE_LEN_MAX = 60
-META_DESC_LEN_MIN = 70
-META_DESC_LEN_MAX = 160
-REDIRECT_CHAIN_LONG = 2
-
-
-def _issue(message: str, url: Optional[str] = None, priority: str = "Medium", recommendation: str = "") -> dict:
- return {"message": message, "url": url or "", "priority": priority, "recommendation": recommendation}
-
-
-def _sort_issues(issues: list[dict]) -> list[dict]:
- return sorted(issues, key=lambda x: PRIORITY_ORDER.get(x.get("priority", "Low"), 99))
-
-
-def _page_analysis_dict(row: pd.Series) -> dict:
- """Parse page_analysis JSON cell from a crawl row."""
- import json
-
- raw = row.get("page_analysis")
- if raw is None or (isinstance(raw, float) and pd.isna(raw)):
- return {}
- s = str(raw).strip()
- if not s or s == "{}":
- return {}
- try:
- o = json.loads(s)
- return o if isinstance(o, dict) else {}
- except json.JSONDecodeError:
- return {}
-
-
-def _score_deductions(max_score: int, deductions: list[tuple[int, bool]]) -> int:
- """Return max(0, max_score - sum of deduction for each True)."""
- total = sum(d for d, apply in deductions if apply)
- return max(0, max_score - total)
-
-
-def _hreflang_issues(success_df: pd.DataFrame) -> list[dict]:
- """Hreflang cluster consistency (return tags, self-reference)."""
- issues: list[dict] = []
- if "page_analysis" not in success_df.columns:
- return issues
- for _, row in success_df.iterrows():
- pa = _page_analysis_dict(row)
- alts = pa.get("hreflang_alternates") or []
- if not alts:
- continue
- url = str(row.get("url") or "").strip()
- langs = [str(a.get("hreflang") or a.get("lang") or "").strip().lower() for a in alts if isinstance(a, dict)]
- hrefs = [str(a.get("href") or "").strip() for a in alts if isinstance(a, dict)]
- if langs and len(set(langs)) < len(langs):
- issues.append(_issue(
- "Duplicate hreflang language codes on page.",
- url=url,
- priority="High",
- recommendation="Each hreflang alternate should use a unique language/region code.",
- ))
- break
- if url and hrefs and url.rstrip("/") not in [h.rstrip("/") for h in hrefs]:
- issues.append(_issue(
- "Hreflang cluster missing self-referencing alternate.",
- url=url,
- priority="Medium",
- recommendation="Include a hreflang link pointing to this page URL.",
- ))
- break
- return issues
-
-
-def _schema_issues(success_df: pd.DataFrame) -> list[dict]:
- issues: list[dict] = []
- invalid = 0
- for _, row in success_df.iterrows():
- pa = _page_analysis_dict(row)
- schemas = pa.get("json_ld_types") or pa.get("schema_types") or []
- if isinstance(schemas, str):
- schemas = [schemas]
- url = str(row.get("url") or "").strip()
- has_schema = str(row.get("has_schema", "")).lower() in ("true", "1", "yes")
- if has_schema and not schemas:
- invalid += 1
- if invalid == 1:
- issues.append(_issue(
- "Structured data present but could not parse JSON-LD @type.",
- url=url,
- priority="Low",
- recommendation="Validate JSON-LD with Google Rich Results Test.",
- ))
- return issues
-
-
-def _soft_404_issues(success_df: pd.DataFrame) -> list[dict]:
- issues: list[dict] = []
- markers = ("not found", "404", "page not found", "doesn't exist", "does not exist")
- for _, row in success_df.iterrows():
- title = str(row.get("title") or "").lower()
- if any(m in title for m in markers):
- url = str(row.get("url") or "").strip()
- issues.append(_issue(
- "Possible soft 404: page returns 200 but title suggests not found.",
- url=url,
- priority="High",
- recommendation="Return 404 status or redirect to a relevant page.",
- ))
- if len(issues) >= 10:
- break
- return issues
-
-
-def _broken_link_sources(edges: list[tuple[str, str]], broken_urls: set[str]) -> list[dict]:
- """Issues listing which pages link to broken URLs."""
- issues: list[dict] = []
- if not broken_urls:
- return issues
- sources: dict[str, list[str]] = {}
- for src, tgt in edges:
- if tgt in broken_urls:
- sources.setdefault(tgt, []).append(src)
- for tgt, srcs in list(sources.items())[:15]:
- sample = ", ".join(srcs[:3])
- more = f" (+{len(srcs) - 3} more)" if len(srcs) > 3 else ""
- issues.append(_issue(
- f"Broken URL linked from {len(srcs)} page(s): {sample}{more}",
- url=tgt,
- priority="High",
- recommendation="Fix or remove links pointing to this URL.",
- ))
- return issues
-
-
-def _indexation_coverage_issues(
- df: pd.DataFrame,
- indexation: dict | None,
-) -> list[dict]:
- """Sitemap vs crawl mismatches and noindex URLs listed in sitemap."""
- issues: list[dict] = []
- if not indexation:
- return issues
- lists = indexation.get("lists") if isinstance(indexation.get("lists"), dict) else {}
- sitemap_only = lists.get("sitemap_only") or []
- for url in sitemap_only[:15]:
- issues.append(_issue(
- f"URL in sitemap but not crawled: {url}",
- url=str(url),
- priority="High",
- recommendation="Verify the URL is linked internally, not blocked by robots, and within crawl scope.",
- ))
- sitemap_urls = indexation.get("sitemap_urls") or []
- if sitemap_urls and "noindex" in df.columns:
- from ..integrations.google.normalize import normalize_url
-
- sitemap_norm = {normalize_url(u) for u in sitemap_urls}
- success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df
- for _, row in success.iterrows():
- url = str(row.get("url") or "").strip()
- if not url:
- continue
- noindex = str(row.get("noindex") or "").lower() in ("true", "1", "yes")
- if noindex and normalize_url(url) in sitemap_norm:
- issues.append(_issue(
- "Page has noindex but is listed in XML sitemap.",
- url=url,
- priority="Critical",
- recommendation="Remove the URL from the sitemap or remove noindex if the page should be indexed.",
- ))
- break
- return issues
-
-
-def merge_indexation_issues(categories: list[dict], df: pd.DataFrame, indexation: dict | None) -> None:
- """Append indexation coverage issues to the technical SEO category."""
- extra = _indexation_coverage_issues(df, indexation)
- if not extra:
- return
- for cat in categories:
- if cat.get("id") == "technical_seo":
- cat["issues"] = _sort_issues((cat.get("issues") or []) + extra)
- recs = {i["recommendation"] for i in cat["issues"] if i.get("recommendation")}
- cat["recommendations"] = list(recs)
- break
-
-
-def merge_subdomain_issues(categories: list[dict], subdomains: dict | None) -> None:
- """Append GSC subdomain gap summary to technical SEO."""
- if not subdomains or subdomains.get("disabled"):
- return
- hosts = subdomains.get("gsc_hosts_not_crawled") or []
- if not hosts:
- return
- preview = ", ".join(hosts[:5])
- suffix = f" (+{len(hosts) - 5} more)" if len(hosts) > 5 else ""
- msg = f"GSC shows URLs on subdomain(s) not reached by crawl: {preview}{suffix}."
- issue = _issue(
- msg,
- priority="Medium",
- recommendation="Include these hosts in crawl scope or verify they are intentional separate properties.",
- )
- for cat in categories:
- if cat.get("id") == "technical_seo":
- cat["issues"] = _sort_issues((cat.get("issues") or []) + [issue])
- recs = {i["recommendation"] for i in cat["issues"] if i.get("recommendation")}
- cat["recommendations"] = list(recs)
- break
-
-
-def _orphan_hub_suggestions(edges: list[tuple[str, str]], orphan_urls: list[str]) -> list[dict]:
- issues: list[dict] = []
- if not edges or not orphan_urls:
- return issues
- in_deg: dict[str, int] = {}
- out_from: dict[str, list[str]] = {}
- for src, tgt in edges:
- in_deg[tgt] = in_deg.get(tgt, 0) + 1
- out_from.setdefault(src, []).append(tgt)
- hubs = sorted(in_deg.keys(), key=lambda u: -in_deg.get(u, 0))[:5]
- hub_label = hubs[0] if hubs else ""
- for orphan in orphan_urls[:10]:
- issues.append(_issue(
- f"Orphan page (no inlinks). Consider linking from hub page: {hub_label}" if hub_label else "Orphan page (no inlinks).",
- url=orphan,
- priority="Medium",
- recommendation="Add internal links from category or hub pages to this URL.",
- ))
- return issues
-
-
-def category_technical_seo(
- df: pd.DataFrame,
- site_level: dict,
-) -> dict:
- """Technical SEO: robots, sitemap, canonical, duplicate content, noindex, schema."""
- issues = []
- deductions = []
- total = len(df)
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame()
-
- if not site_level.get("robots_present", True):
- issues.append(_issue(
- "robots.txt is missing or unreachable.",
- priority="High",
- recommendation="Add a robots.txt at the site root to control crawler access.",
- ))
- deductions.append((15, True))
- if not site_level.get("sitemap_present", True):
- issues.append(_issue(
- "sitemap.xml (or sitemap index) is missing or unreachable.",
- priority="High",
- recommendation="Add a sitemap at /sitemap.xml or link it in robots.txt.",
- ))
- deductions.append((10, True))
- if site_level.get("sitemap_present") and not site_level.get("sitemap_valid", True):
- issues.append(_issue(
- "sitemap.xml could not be parsed as valid XML.",
- priority="Medium",
- recommendation="Ensure sitemap is valid XML and follows sitemaps.org format.",
- ))
- deductions.append((5, True))
- if site_level.get("ads_txt_present") is False:
- issues.append(_issue(
- "ads.txt is missing or unreachable.",
- priority="Low",
- recommendation="Add an ads.txt file at the site root if you run programmatic advertising.",
- ))
- if site_level.get("security_txt_present") is False:
- issues.append(_issue(
- "security.txt is missing or unreachable.",
- priority="Low",
- recommendation="Publish security.txt at /.well-known/security.txt with a Contact field for security reporting.",
- ))
-
- # Canonical: missing or self-mismatch
- if "canonical_url" in df.columns and len(success_df) > 0:
- for _, row in success_df.iterrows():
- url = row.get("url")
- canon = row.get("canonical_url")
- if pd.isna(url):
- continue
- url = str(url).strip()
- canon = "" if pd.isna(canon) else str(canon).strip()
- if not canon:
- issues.append(_issue("Missing canonical URL.", url=url, priority="Medium", recommendation="Add a canonical link tag pointing to the preferred URL."))
- break
- missing_canon = success_df["canonical_url"].fillna("").astype(str).str.strip().eq("").sum()
- if missing_canon > 0:
- deductions.append((min(15, missing_canon * 2), True))
- # Self-canonical mismatch: canonical points to different URL
- for _, row in success_df.iterrows():
- url = row.get("url")
- canon = row.get("canonical_url")
- if pd.isna(url) or pd.isna(canon) or not str(canon).strip():
- continue
- url = str(url).rstrip("/")
- canon = str(canon).strip().rstrip("/")
- if url != canon:
- issues.append(_issue(f"Canonical points to different URL: {canon}", url=url, priority="High", recommendation="Set canonical to this page URL or the preferred duplicate."))
- deductions.append((10, True))
- break
-
- # Noindex on important pages (CSV may store True/False as strings)
- if "noindex" in df.columns and len(success_df) > 0:
- noindex_ser = success_df["noindex"].astype(str).str.lower().isin(("true", "1", "yes"))
- noindex_count = int(noindex_ser.sum())
- if noindex_count > 0:
- issues.append(_issue(
- f"{int(noindex_count)} page(s) have noindex.",
- priority="High" if noindex_count > 5 else "Medium",
- recommendation="Remove noindex from pages that should be indexed, or keep for intentional no-index pages.",
- ))
- deductions.append((min(15, noindex_count * 3), True))
-
- # Duplicate content heuristic: same title + meta description
- if "title" in df.columns and "meta_description" in df.columns and len(success_df) > 1:
- key = success_df["title"].fillna("").astype(str) + "|" + success_df["meta_description"].fillna("").astype(str)
- dupes = key.value_counts()
- dupes = dupes[dupes > 1]
- if len(dupes) > 0:
- issues.append(_issue(
- f"Possible duplicate content: {len(dupes)} group(s) of pages share same title and meta description.",
- priority="Medium",
- recommendation="Differentiate titles and meta descriptions, or use canonicals to designate the preferred URL.",
- ))
- deductions.append((10, True))
-
- # Social meta tags
- if "og_title" in df.columns and len(success_df) > 0:
- og_present = (success_df["og_title"].fillna("").astype(str).str.strip() != "").sum()
- og_pct = og_present / len(success_df) if len(success_df) > 0 else 1
- if og_pct < 0.5:
- issues.append(_issue(
- f"Open Graph tags missing on {int((1 - og_pct) * 100)}% of pages.",
- priority="Medium",
- recommendation="Add og:title, og:description, and og:image meta tags for social sharing.",
- ))
- deductions.append((5, True))
-
- if "twitter_card" in df.columns and len(success_df) > 0:
- tw_present = (success_df["twitter_card"].fillna("").astype(str).str.strip() != "").sum()
- tw_pct = tw_present / len(success_df) if len(success_df) > 0 else 1
- if tw_pct < 0.2:
- issues.append(_issue(
- f"Twitter Card tags missing on {int((1 - tw_pct) * 100)}% of pages.",
- priority="Low",
- recommendation="Add twitter:card meta tags for better Twitter/X sharing previews.",
- ))
- deductions.append((3, True))
-
- # Structured data
- if "has_schema" in df.columns and len(success_df) > 0:
- with_schema = int(success_df["has_schema"].astype(str).str.lower().isin(("true", "1", "yes")).sum())
- if with_schema == 0:
- issues.append(_issue(
- "No structured data (JSON-LD or microdata) detected.",
- priority="Low",
- recommendation="Add schema.org markup (e.g. Organization, Article) for rich results.",
- ))
- deductions.append((5, True))
-
- # Internationalization: from page_analysis (re-crawl to populate)
- if "page_analysis" in df.columns and len(success_df) > 0:
- missing_lang = 0
- for _, row in success_df.iterrows():
- pa = _page_analysis_dict(row)
- if not (pa.get("html_lang") or "").strip():
- missing_lang += 1
- if missing_lang > 0 and len(success_df) >= 3:
- ratio = missing_lang / len(success_df)
- if ratio > 0.1:
- issues.append(_issue(
- f"{missing_lang} page(s) missing (of {len(success_df)} OK responses).",
- priority="Medium" if ratio > 0.5 else "Low",
- recommendation="Add matching the primary language of each page.",
- ))
- deductions.append((min(10, max(2, missing_lang // 5)), True))
-
- issues.extend(_hreflang_issues(success_df))
- issues.extend(_schema_issues(success_df))
- issues.extend(_soft_404_issues(success_df))
-
- if "page_analysis" in df.columns and len(success_df) > 0:
- from ..crawl.fetchers.browser_diagnostics import browser_summary_from_page_analysis
-
- pages_with_console = 0
- for _, row in success_df.iterrows():
- pa = _page_analysis_dict(row)
- counts = browser_summary_from_page_analysis(pa)
- url = str(row.get("url") or "").strip()
- if counts["console_error_count"] > 0:
- pages_with_console += 1
- if counts["page_error_count"] > 0 and url:
- issues.append(_issue(
- "Uncaught JavaScript error during browser render.",
- url=url,
- priority="High",
- recommendation="Fix runtime JS errors that may break page functionality or SEO signals.",
- ))
- deductions.append((5, True))
- if pages_with_console > 0:
- issues.append(_issue(
- f"{pages_with_console} page(s) logged console errors during JavaScript rendering.",
- priority="High" if pages_with_console > 3 else "Medium",
- recommendation="Inspect browser console errors on affected URLs; fix broken scripts or API calls.",
- ))
- deductions.append((min(15, pages_with_console * 2), True))
-
- score = _score_deductions(100, deductions)
- return {
- "id": "technical_seo",
- "name": CATEGORY_TECHNICAL_SEO,
- "score": score,
- "issues": _sort_issues(issues),
- "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}),
- }
-
-
-def category_core_web_vitals() -> dict:
- """Core Web Vitals: not measured; recommend Lighthouse."""
- return {
- "id": "core_web_vitals",
- "name": CATEGORY_CORE_WEB_VITALS,
- "score": None,
- "issues": [_issue(
- "LCP, INP, and CLS are not measured by this crawl.",
- priority="Medium",
- recommendation="Run Lighthouse (PageSpeed Insights) from Run audit to measure Core Web Vitals.",
- )],
- "recommendations": ["Run Lighthouse from Run audit to measure LCP, INP, and CLS."],
- }
-
-
-def category_core_web_vitals_from_lighthouse(
- lighthouse_summary: dict,
- crux_summary: Optional[dict] = None,
-) -> dict:
- """Core Web Vitals from Lighthouse summary: score 0–100 from performance score, issues from top_failures."""
- issues = []
- recommendations = []
- perf_score = None
- mm = lighthouse_summary.get("median_metrics") or {}
- if isinstance(mm.get("performance_score"), (int, float)):
- perf_score = max(0, min(100, int(round(mm["performance_score"] * 100))))
- for f in lighthouse_summary.get("top_failures") or []:
- aid = f.get("id") or ""
- help_text = (f.get("helpText") or "")[:200]
- msg = f"{aid}: {help_text}" if aid else help_text or "Audit failed"
- issues.append(_issue(
- msg,
- priority="High" if (f.get("score") or 0) < 0.5 else "Medium",
- recommendation="See Performance (Core Web Vitals) in this audit, or re-run Lighthouse from Run audit.",
- ))
- if not issues and perf_score is not None and perf_score < 80:
- recommendations.append("Improve Core Web Vitals (LCP, CLS, TBT) per Lighthouse recommendations.")
- if crux_summary and crux_summary.get("ok"):
- pw = crux_summary.get("pass") or {}
- for metric, label, rec in (
- ("lcp", "LCP", "Improve largest contentful paint (field data)."),
- ("inp", "INP", "Reduce interaction to next paint (field data)."),
- ("cls", "CLS", "Reduce cumulative layout shift (field data)."),
- ):
- if pw.get(metric) is False:
- issues.append(_issue(
- f"CrUX field data: {label} does not pass Core Web Vitals threshold.",
- priority="High",
- recommendation=rec,
- ))
- return {
- "id": "core_web_vitals",
- "name": CATEGORY_CORE_WEB_VITALS,
- "score": perf_score,
- "issues": _sort_issues(issues),
- "recommendations": recommendations or ["Core Web Vitals measured by Lighthouse; see median_metrics in lighthouse_summary.json."],
- }
-
-
-def category_performance(df: pd.DataFrame) -> dict:
- """Performance: response time, JS/CSS size, images, lazy loading, caching."""
- issues = []
- deductions = []
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame()
- if len(success_df) == 0:
- return {"id": "performance", "name": CATEGORY_PERFORMANCE, "score": 0, "issues": [], "recommendations": []}
-
- if "response_time_ms" in success_df.columns:
- rt = pd.to_numeric(success_df["response_time_ms"], errors="coerce").fillna(0)
- slow = (rt > RESPONSE_TIME_SLOW_MS).sum()
- if slow > 0:
- issues.append(_issue(
- f"{int(slow)} page(s) have server response time > {RESPONSE_TIME_SLOW_MS // 1000}s.",
- priority="High" if slow > 5 else "Medium",
- recommendation="Optimize server response time (TTFB): caching, CDN, or backend tuning.",
- ))
- deductions.append((min(20, int(slow) * 2), True))
- valid_rt = rt[rt > 0]
- if len(valid_rt) > 5:
- p95 = float(valid_rt.quantile(0.95))
- if p95 > 3000:
- issues.append(_issue(
- f"95th percentile response time is {int(p95)}ms (over 3s).",
- priority="High",
- recommendation="Investigate slowest pages; consider CDN, server-side caching, or database optimization.",
- ))
- deductions.append((10, True))
-
- if "images_total" in success_df.columns:
- total_imgs = success_df["images_total"].fillna(0).astype(int).sum()
- if total_imgs > 0 and "img_without_lazy" in success_df.columns:
- no_lazy = success_df["img_without_lazy"].fillna(0).astype(int).sum()
- if no_lazy > total_imgs * 0.5:
- issues.append(_issue(
- "Many images without lazy loading.",
- priority="Medium",
- recommendation="Add loading='lazy' to off-screen images.",
- ))
- deductions.append((10, True))
- if total_imgs > 0 and "img_without_dimensions" in success_df.columns:
- no_dims = success_df["img_without_dimensions"].fillna(0).astype(int).sum()
- if no_dims > 0:
- issues.append(_issue(
- f"{int(no_dims)} image(s) without width/height (can cause CLS).",
- priority="High",
- recommendation="Set width and height attributes on img tags to avoid layout shift.",
- ))
- deductions.append((10, True))
-
- if "cache_control" in success_df.columns:
- cache = success_df["cache_control"].fillna("").astype(str)
- no_cache = (cache.str.strip() == "").sum()
- if no_cache > len(success_df) * 0.5:
- issues.append(_issue(
- "Many pages without Cache-Control header.",
- priority="Medium",
- recommendation="Set Cache-Control (and optionally ETag) for static and cacheable pages.",
- ))
- deductions.append((10, True))
-
- if "script_count" in success_df.columns:
- scripts = success_df["script_count"].fillna(0).astype(int)
- if scripts.sum() > len(success_df) * 10:
- issues.append(_issue(
- "High number of script tags across pages.",
- priority="Low",
- recommendation="Consider bundling and code-splitting to reduce JS payload.",
- ))
- deductions.append((5, True))
-
- score = _score_deductions(100, deductions)
- return {
- "id": "performance",
- "name": CATEGORY_PERFORMANCE,
- "score": score,
- "issues": _sort_issues(issues),
- "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}),
- }
-
-
-def _parse_page_analysis_cell(raw: object) -> dict[str, Any]:
- if isinstance(raw, dict):
- return raw
- if not raw or not isinstance(raw, str):
- return {}
- try:
- parsed = json.loads(raw)
- return parsed if isinstance(parsed, dict) else {}
- except Exception:
- return {}
-
-
-def contrast_issues_from_sources(
- df: pd.DataFrame,
- lighthouse_by_url: Optional[dict[str, Any]] = None,
-) -> list[dict]:
- """Contrast issues from axe crawl data and per-URL Lighthouse failures."""
- issues: list[dict] = []
- seen_urls: set[str] = set()
-
- if df is not None and not df.empty and "page_analysis" in df.columns:
- for _, row in df.iterrows():
- url = str(row.get("url") or "").strip()
- if not url:
- continue
- pa = _parse_page_analysis_cell(row.get("page_analysis"))
- axe = pa.get("axe_violations")
- if not isinstance(axe, list):
- continue
- contrast_hits = [
- v for v in axe
- if isinstance(v, dict) and "color-contrast" in str(v.get("id") or "")
- ]
- if not contrast_hits:
- continue
- seen_urls.add(url.rstrip("/"))
- first = contrast_hits[0]
- msg = str(first.get("description") or first.get("help") or "Color contrast violation")
- issues.append(_issue(
- f"axe: {msg}",
- url=url,
- priority="Medium",
- recommendation=str(
- first.get("help")
- or "Fix text/background contrast to meet WCAG AA (axe-core)."
- ),
- ))
-
- lh_map = lighthouse_by_url or {}
- for url, summary in lh_map.items():
- if not isinstance(summary, dict):
- continue
- u = str(url or summary.get("url") or "").strip().rstrip("/")
- if not u or u in seen_urls:
- continue
- for fail in summary.get("top_failures") or []:
- if not isinstance(fail, dict):
- continue
- if str(fail.get("id") or "") != "color-contrast":
- continue
- seen_urls.add(u)
- help_text = str(fail.get("helpText") or "Low color contrast")
- issues.append(_issue(
- f"Lighthouse: {help_text}",
- url=u,
- priority="Medium",
- recommendation="Increase contrast ratio between text and background to meet WCAG AA.",
- ))
- break
-
- return issues[:40]
-
-
-def category_html_accessibility(
- df: pd.DataFrame,
- lighthouse_by_url: Optional[dict[str, Any]] = None,
-) -> dict:
- """HTML and Accessibility: semantic HTML, heading structure, alt, ARIA, contrast."""
- issues = []
- deductions = []
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame()
- if len(success_df) == 0:
- return {"id": "html_accessibility", "name": CATEGORY_ACCESSIBILITY, "score": 0, "issues": [], "recommendations": []}
-
- if "h1_count" in df.columns:
- h1c = pd.to_numeric(success_df["h1_count"], errors="coerce").fillna(-1).astype(int)
- zero_h1 = (h1c == 0).sum()
- multi_h1 = (h1c > 1).sum()
- if zero_h1 > 0:
- issues.append(_issue(
- f"{int(zero_h1)} page(s) missing H1.",
- priority="High",
- recommendation="Add exactly one H1 per page describing the main content.",
- ))
- deductions.append((min(20, int(zero_h1) * 3), True))
- if multi_h1 > 0:
- issues.append(_issue(
- f"{int(multi_h1)} page(s) have multiple H1s.",
- priority="Medium",
- recommendation="Use a single H1 per page; use H2–H6 for subsections.",
- ))
- deductions.append((min(10, int(multi_h1) * 2), True))
-
- if "heading_sequence" in df.columns:
- pages_with_skipped_heading = 0
- for _, row in success_df.iterrows():
- seq = row.get("heading_sequence")
- if pd.isna(seq) or not str(seq).strip():
- continue
- parts = [p.strip() for p in str(seq).split(",") if p.strip()]
- if not parts:
- continue
- levels = [int(h[1]) for h in parts if len(h) == 2 and h[0] == "h" and h[1] in "123456"]
- for i in range(1, len(levels)):
- if levels[i] > levels[i - 1] + 1:
- if pages_with_skipped_heading == 0:
- issues.append(_issue(
- "Skipped heading level (e.g. H1 then H3).",
- url=str(row.get("url", "")),
- priority="Medium",
- recommendation="Use heading levels in order (H1, H2, H3) without skipping.",
- ))
- pages_with_skipped_heading += 1
- break
- if pages_with_skipped_heading > 0:
- deductions.append((min(15, pages_with_skipped_heading * 5), True))
-
- if "images_total" in df.columns and "images_without_alt" in df.columns:
- total = success_df["images_total"].fillna(0).astype(int).sum()
- missing_alt = success_df["images_without_alt"].fillna(0).astype(int).sum()
- if total > 0 and missing_alt > 0:
- issues.append(_issue(
- f"{int(missing_alt)} image(s) without alt (or aria-label).",
- priority="High",
- recommendation="Add meaningful alt text to all images; use alt='' for decorative images.",
- ))
- deductions.append((min(15, int(missing_alt) * 2), True))
-
- if "word_count" in success_df.columns:
- wc = pd.to_numeric(success_df["word_count"], errors="coerce").fillna(0).astype(int)
- very_thin = int(((wc > 0) & (wc < 100)).sum())
- if very_thin > 0:
- issues.append(_issue(
- f"{very_thin} page(s) with very thin content (under 100 words).",
- priority="High",
- recommendation="Expand thin pages with meaningful content (aim for 300+ words).",
- ))
- deductions.append((min(15, very_thin * 3), True))
-
- if "reading_level" in success_df.columns:
- rl = pd.to_numeric(success_df["reading_level"], errors="coerce").fillna(0)
- complex_pages = int((rl > 14).sum())
- if complex_pages > 0:
- issues.append(_issue(
- f"{complex_pages} page(s) have very complex content (reading level > 14).",
- priority="Medium",
- recommendation="Simplify language for broader audience accessibility (aim for grade 8-10).",
- ))
- deductions.append((min(10, complex_pages * 2), True))
-
- contrast_issues = contrast_issues_from_sources(df, lighthouse_by_url)
- if contrast_issues:
- issues.extend(contrast_issues)
- deductions.append((min(25, len(contrast_issues) * 4), True))
- else:
- issues.append(_issue(
- "Color contrast is not measured by this tool.",
- priority="Low",
- recommendation="Enable axe (browser crawl) or Lighthouse to check contrast.",
- ))
-
- score = _score_deductions(100, deductions)
- if len(success_df) > 0 and score == 0:
- score = 5
- score = min(100, max(0, score))
- return {
- "id": "html_accessibility",
- "name": CATEGORY_ACCESSIBILITY,
- "score": score,
- "issues": _sort_issues(issues),
- "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}),
- }
-
-
-def category_link_health(
- df: pd.DataFrame,
- edges: list[tuple[str, str]],
- issues_broken: list[dict],
- issues_redirects: list[dict],
-) -> dict:
- """Link Health: broken links, redirect chains, internal linking."""
- issues = []
- deductions = []
-
- for b in issues_broken[:30]:
- status = str(b.get("status", ""))
- priority = "Critical" if status.startswith("5") else "High"
- issues.append(_issue(
- f"Broken URL: {status}",
- url=b.get("url", ""),
- priority=priority,
- recommendation="Fix or remove the link; return 200 or redirect to a valid URL.",
- ))
- broken_url_set = {str(b.get("url") or "").strip() for b in issues_broken if b.get("url")}
- issues.extend(_broken_link_sources(edges, broken_url_set))
- if issues_broken:
- deductions.append((min(30, len(issues_broken) * 2), True))
-
- for r in issues_redirects[:20]:
- issues.append(_issue(
- f"Redirect: {r.get('status', '')} to {r.get('final_url', '')}",
- url=r.get("url", ""),
- priority="Medium",
- recommendation="Prefer direct URLs or shorten redirect chains.",
- ))
- if issues_redirects:
- deductions.append((min(15, len(issues_redirects)), True))
-
- if "redirect_chain_length" in df.columns and len(df) > 0:
- rcl = pd.to_numeric(df["redirect_chain_length"], errors="coerce").fillna(0).astype(int)
- long_chains = (rcl >= REDIRECT_CHAIN_LONG).sum()
- if long_chains > 0:
- issues.append(_issue(
- f"{int(long_chains)} URL(s) have redirect chains (2+ hops).",
- priority="Medium",
- recommendation="Consolidate redirects to a single hop where possible.",
- ))
- deductions.append((min(10, int(long_chains)), True))
-
- if edges:
- import networkx as nx
- G = nx.DiGraph()
- G.add_edges_from(edges)
- in_deg = dict(G.in_degree())
- orphans = [n for n in G.nodes() if in_deg.get(n, 0) == 0]
- if len(orphans) > len(G.nodes()) * 0.3:
- issues.append(_issue(
- f"Many pages have no internal links pointing to them ({len(orphans)}).",
- priority="Low",
- recommendation="Add internal links to important pages to improve crawlability and internal link equity.",
- ))
- deductions.append((5, True))
- issues.extend(_orphan_hub_suggestions(edges, orphans[:15]))
-
- score = _score_deductions(100, deductions)
- return {
- "id": "link_health",
- "name": CATEGORY_LINKS,
- "score": score,
- "issues": _sort_issues(issues),
- "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}),
- }
-
-
-def category_mobile(df: pd.DataFrame) -> dict:
- """Mobile: viewport, responsive heuristic."""
- issues = []
- deductions = []
- success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame()
- if len(success_df) == 0:
- return {"id": "mobile", "name": CATEGORY_MOBILE, "score": 0, "issues": [], "recommendations": []}
-
- if "viewport_present" in df.columns:
- viewport_ok = success_df["viewport_present"].astype(str).str.lower().isin(("true", "1", "yes"))
- no_viewport = int((~viewport_ok).sum())
- if no_viewport > 0:
- issues.append(_issue(
- f"{int(no_viewport)} page(s) missing viewport meta tag.",
- priority="Critical",
- recommendation="Add " in str(rec.get("html") or "").lower()
+ has_definition = bool(re.search(r"\b(is|are|means|refers to)\b", lead[:400], re.I))
+ try:
+ wc = int(rec.get("word_count") or 0)
+ except (TypeError, ValueError):
+ wc = 0
+ quotability = 0
+ if wc >= 200:
+ quotability += 25
+ if has_list:
+ quotability += 20
+ if has_definition:
+ quotability += 25
+ if _has_faq_schema(rec):
+ quotability += 30
+ schema_types = _row_schema_types_list(rec)
+ if schema_types:
+ quotability += 10
+ return {
+ "word_count": wc,
+ "has_lists": has_list,
+ "has_definition_pattern": has_definition,
+ "quotability_score": min(100, quotability),
+ "schema_types": schema_types[:5],
+ }
+
+
+def _parse_robots_txt(domain: str) -> str:
+ if not domain:
+ return ""
+ base = f"https://{domain.lstrip('https://').lstrip('http://').split('/')[0]}"
+ url = urljoin(base + "/", "robots.txt")
+ try:
+ resp = requests.get(url, timeout=8, headers={"User-Agent": "SiteAudit/1.0"})
+ if resp.status_code == 200:
+ return resp.text
+ except requests.RequestException:
+ return ""
+ return ""
+
+
+def _agent_blocked(robots_text: str, agent: str) -> bool:
+ blocks: dict[str, bool] = {}
+ current_agent = "*"
+ for line in robots_text.splitlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ lower = line.lower()
+ if lower.startswith("user-agent:"):
+ current_agent = line.split(":", 1)[1].strip()
+ continue
+ if lower.startswith("disallow:"):
+ path = line.split(":", 1)[1].strip()
+ if path == "/":
+ blocks[current_agent.lower()] = True
+ agent_lower = agent.lower()
+ return bool(blocks.get(agent_lower) or blocks.get("*"))
+
+
+def _llms_urls(llms_preview: str, llms_url: str) -> set[str]:
+ urls: set[str] = set()
+ for line in (llms_preview or "").splitlines():
+ for match in re.findall(r"https?://[^\s)>]+", line):
+ urls.add(match.rstrip("/").lower())
+ if llms_url:
+ urls.add(llms_url.rstrip("/").lower())
+ return urls
+
+
+def list_pages_missing_howto_schema(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False, "missing": True}
+ pages: list[dict[str, Any]] = []
+ for _, row in df.iterrows():
+ rec = row.to_dict()
+ if not str(rec.get("status") or "").startswith("2"):
+ continue
+ if not _looks_like_howto_page(rec) or _has_howto_schema(rec):
+ continue
+ pages.append({
+ "url": str(rec.get("url") or ""),
+ "title": str(rec.get("title") or ""),
+ "reason": "howto_heuristic_no_schema",
+ })
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Estimated"}
+
+
+def list_pages_ai_citation_signals(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False, "missing": True}
+ try:
+ min_score = int(args.get("min_score") or 0)
+ except (TypeError, ValueError):
+ min_score = 0
+ scored: list[dict[str, Any]] = []
+ for _, row in df.iterrows():
+ rec = row.to_dict()
+ if not str(rec.get("status") or "").startswith("2"):
+ continue
+ signals = _aeo_score(rec)
+ if signals["quotability_score"] < min_score:
+ continue
+ scored.append({
+ "url": str(rec.get("url") or ""),
+ "title": str(rec.get("title") or ""),
+ **signals,
+ })
+ scored.sort(key=lambda p: -int(p.get("quotability_score") or 0))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(scored, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Estimated"}
+
+
+def list_pages_missing_llms_txt_reference(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ domain = scoped.resolve_property_domain(conn)
+ llms = _fetch_llms_txt(domain)
+ if not llms.get("found"):
+ return {
+ "pages": [],
+ "total": 0,
+ "truncated": False,
+ "missing": True,
+ "note": "llms.txt not found on domain",
+ "domain": domain,
+ }
+ listed = _llms_urls(str(llms.get("preview") or ""), str(llms.get("url") or ""))
+ payload = scoped.load_payload(conn)
+ candidates: list[str] = []
+ if payload:
+ for page in payload.get("top_pages") or payload.get("links") or []:
+ if isinstance(page, dict) and page.get("url"):
+ candidates.append(str(page["url"]))
+ df = scoped.load_crawl_df(conn)
+ if df is not None and not df.empty:
+ for _, row in df.iterrows():
+ rec = row.to_dict()
+ if str(rec.get("status") or "").startswith("2") and rec.get("url"):
+ candidates.append(str(rec["url"]))
+ seen: set[str] = set()
+ missing: list[dict[str, Any]] = []
+ for url in candidates:
+ norm = url.rstrip("/").lower()
+ if norm in seen:
+ continue
+ seen.add(norm)
+ if norm in listed or url in listed:
+ continue
+ missing.append({"url": url, "llms_txt_url": llms.get("url")})
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(missing, limit, max_cap=50)
+ return {
+ "pages": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "llms_txt_url": llms.get("url"),
+ "provenance": "Estimated",
+ }
+
+
+def list_robots_blocked_ai_crawlers(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ domain = scoped.resolve_property_domain(conn)
+ if not domain:
+ return {"error": "domain unknown", "agents": [], "total": 0, "truncated": False}
+ robots_text = _parse_robots_txt(domain)
+ if not robots_text.strip():
+ return {
+ "domain": domain,
+ "agents": [],
+ "total": 0,
+ "truncated": False,
+ "missing": True,
+ "note": "robots.txt not reachable",
+ }
+ blocked: list[dict[str, Any]] = []
+ for agent in _AI_CRAWLER_AGENTS:
+ if _agent_blocked(robots_text, agent):
+ blocked.append({"agent": agent, "blocked": True, "scope": "disallow: /"})
+ limit = parse_limit(args.get("limit"), 10, 20)
+ sliced = cap_list(blocked, limit, max_cap=20)
+ return {
+ "domain": domain,
+ "agents": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "robots_txt_checked": True,
+ "provenance": "Crawl",
+ }
diff --git a/src/website_profiling/tools/audit_tools/google.py b/src/website_profiling/tools/audit_tools/google.py
index 14184fec..5043131b 100644
--- a/src/website_profiling/tools/audit_tools/google.py
+++ b/src/website_profiling/tools/audit_tools/google.py
@@ -95,7 +95,7 @@ def get_gsc_page_query_slice(conn: Connection, ctx: AuditToolContext, args: dict
url = str(args.get("url") or "").strip()
if not url:
return {"error": "url is required"}
- data = scoped.load_google(conn)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
if not data:
return {"error": "no google data found"}
slice_data = slice_from_google_row(data, url)
@@ -187,3 +187,58 @@ def get_gsc_ctr_opportunity_pages(conn: Connection, ctx: AuditToolContext, args:
"truncated": sliced["truncated"],
"provenance": "Search Console",
}
+
+
+def _google_series(data: dict[str, Any] | None, section: str, key: str) -> dict[str, Any]:
+ if not data:
+ return {"error": "no google data found", "missing": True}
+ block = data.get(section) if isinstance(data.get(section), dict) else {}
+ series = block.get(key) or []
+ return {
+ key: series if isinstance(series, list) else [],
+ "fetched_at": data.get("fetched_at"),
+ "date_range": data.get("date_range"),
+ "provenance": "Search Console" if section == "gsc" else "Google Analytics 4",
+ }
+
+
+def get_gsc_daily_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ return _google_series(scoped.load_google(conn), "gsc", "daily")
+
+
+def get_ga4_daily_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ return _google_series(scoped.load_google(conn), "ga4", "daily")
+
+
+def get_ga4_by_device(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ return _google_series(scoped.load_google(conn), "ga4", "by_device")
+
+
+def get_ga4_by_channel(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ return _google_series(scoped.load_google(conn), "ga4", "by_channel")
+
+
+def get_gsc_page_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or "").strip()
+ if not url:
+ return {"error": "url is required"}
+ raw = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not raw:
+ return {"error": "no google data found", "missing": True}
+ slice_data = slice_from_google_row(raw, url)
+ gsc = slice_data.get("gsc") if isinstance(slice_data.get("gsc"), dict) else {}
+ queries = gsc.get("queries") or []
+ limit = parse_limit(args.get("limit"), 25, 50)
+ sliced = cap_list(queries if isinstance(queries, list) else [], limit, max_cap=50)
+ return {
+ "url": url,
+ "queries": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "provenance": "Search Console",
+ }
diff --git a/src/website_profiling/tools/audit_tools/google_lists.py b/src/website_profiling/tools/audit_tools/google_lists.py
new file mode 100644
index 00000000..fac019aa
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/google_lists.py
@@ -0,0 +1,474 @@
+"""Google Search Console and GA4 list/delta tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...integrations.google.keyword_enrich import ctr_as_fraction, industry_ctr
+from ...integrations.google.normalize import normalize_url, url_to_path
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+from .insight_helpers import blend_landing_pages, provenance_block, traffic_health_ratio, _num
+
+
+def _gsc_ga4_blobs(raw: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
+ gsc = raw.get("gsc_full") if isinstance(raw.get("gsc_full"), dict) else raw.get("gsc") or {}
+ ga4 = raw.get("ga4_full") if isinstance(raw.get("ga4_full"), dict) else raw.get("ga4") or {}
+ return gsc if isinstance(gsc, dict) else {}, ga4 if isinstance(ga4, dict) else {}
+
+
+def _load_google_pair(ctx: AuditToolContext, conn: Connection) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+ """Current + prior Google snapshots (read_prior_google_snapshot, else 2nd google_data row)."""
+ current, prior = ctx.load_google_pair(conn)
+ if prior is not None or ctx.property_id is None:
+ return current, prior
+ try:
+ from ...integrations.google.store import read_prior_google_snapshot
+
+ prior = read_prior_google_snapshot(conn, ctx.property_id, skip=1)
+ except Exception:
+ prior = None
+ if prior is None:
+ try:
+ cur = conn.execute(
+ """
+ SELECT data FROM google_data
+ WHERE property_id = %s
+ ORDER BY id DESC LIMIT 2
+ """,
+ (int(ctx.property_id),),
+ )
+ rows = cur.fetchall() or []
+ if len(rows) >= 2:
+ from ...db.storage import _parse_row_json
+
+ prior_data = _parse_row_json(rows[1])
+ prior = prior_data if isinstance(prior_data, dict) else None
+ except Exception:
+ prior = None
+ return current, prior
+
+
+def _gsc_rows(data: dict[str, Any] | None, key: str) -> list[dict[str, Any]]:
+ if not data:
+ return []
+ gsc, _ = _gsc_ga4_blobs(data)
+ rows = gsc.get(key) or gsc.get(f"top_{key}") or []
+ if isinstance(rows, list):
+ return [r for r in rows if isinstance(r, dict)]
+ return []
+
+
+def _sort_gsc_rows(rows: list[dict[str, Any]], field: str, limit: int) -> dict[str, Any]:
+ sorted_rows = sorted(rows, key=lambda r: -_num(r.get(field)))
+ sliced = cap_list(sorted_rows, limit, max_cap=50)
+ return {"items": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_gsc_pages_by_impressions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ result = _sort_gsc_rows(_gsc_rows(data, "pages"), "impressions", limit)
+ return {"pages": result["items"], "total": result["total"], "truncated": result["truncated"]}
+
+
+def list_gsc_pages_by_clicks(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ result = _sort_gsc_rows(_gsc_rows(data, "pages"), "clicks", limit)
+ return {"pages": result["items"], "total": result["total"], "truncated": result["truncated"]}
+
+
+def list_gsc_queries_by_impressions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ result = _sort_gsc_rows(_gsc_rows(data, "queries"), "impressions", limit)
+ return {"queries": result["items"], "total": result["total"], "truncated": result["truncated"]}
+
+
+def list_gsc_queries_by_clicks(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ result = _sort_gsc_rows(_gsc_rows(data, "queries"), "clicks", limit)
+ return {"queries": result["items"], "total": result["total"], "truncated": result["truncated"]}
+
+
+def list_gsc_ctr_underperformers(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ pages = _gsc_rows(data, "pages")
+ ctrs = [ctr_as_fraction(r.get("ctr")) for r in pages if 1 <= _num(r.get("position"), 99) <= 10]
+ ctrs = [c for c in ctrs if c > 0]
+ site_median = sorted(ctrs)[len(ctrs) // 2] if ctrs else 0.05
+ under: list[dict[str, Any]] = []
+ for row in pages:
+ pos = _num(row.get("position"), 99)
+ if pos < 1 or pos > 10:
+ continue
+ ctr = ctr_as_fraction(row.get("ctr"))
+ expected = industry_ctr(pos)
+ if ctr > 0 and ctr < min(site_median * 0.7, expected * 0.7):
+ under.append({
+ "page": row.get("page") or row.get("url"),
+ "clicks": row.get("clicks"),
+ "impressions": row.get("impressions"),
+ "ctr": row.get("ctr"),
+ "position": pos,
+ "site_median_ctr": round(site_median, 4),
+ })
+ under.sort(key=lambda r: -_num(r.get("impressions")))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(under, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _index_gsc_rows(rows: list[dict[str, Any]], key_fields: tuple[str, ...]) -> dict[str, dict[str, Any]]:
+ out: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ key = ""
+ for field in key_fields:
+ key = str(row.get(field) or "").strip()
+ if key:
+ break
+ if key:
+ out[key] = row
+ return out
+
+
+def _gsc_deltas(
+ current_rows: list[dict[str, Any]],
+ prior_rows: list[dict[str, Any]],
+ key_fields: tuple[str, ...],
+ *,
+ decay: bool = True,
+) -> list[dict[str, Any]]:
+ curr = _index_gsc_rows(current_rows, key_fields)
+ prev = _index_gsc_rows(prior_rows, key_fields)
+ deltas: list[dict[str, Any]] = []
+ for key, row in curr.items():
+ old = prev.get(key)
+ if not old:
+ continue
+ click_delta = _num(row.get("clicks")) - _num(old.get("clicks"))
+ imp_delta = _num(row.get("impressions")) - _num(old.get("impressions"))
+ pos_delta = _num(row.get("position")) - _num(old.get("position"))
+ if decay:
+ if click_delta >= 0 and imp_delta >= 0 and pos_delta <= 0:
+ continue
+ else:
+ if click_delta <= 0 and imp_delta <= 0:
+ continue
+ entry = dict(row)
+ entry["key"] = key
+ entry["click_delta"] = int(click_delta)
+ entry["impression_delta"] = int(imp_delta)
+ entry["position_delta"] = round(pos_delta, 2)
+ deltas.append(entry)
+ deltas.sort(key=lambda r: (r.get("click_delta", 0), r.get("impression_delta", 0)))
+ return deltas
+
+
+def list_gsc_decaying_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ current, prior = _load_google_pair(scoped, conn)
+ if not current:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ if not prior:
+ return {"error": "no prior google snapshot for decay comparison", "missing": True, "pages": [], "total": 0, "truncated": False}
+ deltas = _gsc_deltas(_gsc_rows(current, "pages"), _gsc_rows(prior, "pages"), ("page", "url"), decay=True)
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(deltas, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_gsc_decaying_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ current, prior = _load_google_pair(scoped, conn)
+ if not current:
+ return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False}
+ if not prior:
+ return {"error": "no prior google snapshot for decay comparison", "missing": True, "queries": [], "total": 0, "truncated": False}
+ deltas = _gsc_deltas(_gsc_rows(current, "queries"), _gsc_rows(prior, "queries"), ("query",), decay=True)
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(deltas, limit, max_cap=50)
+ return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_gsc_new_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ current, prior = _load_google_pair(scoped, conn)
+ if not current:
+ return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False}
+ if not prior:
+ return {"error": "no prior google snapshot", "missing": True, "queries": [], "total": 0, "truncated": False}
+ curr = _index_gsc_rows(_gsc_rows(current, "queries"), ("query",))
+ prev = _index_gsc_rows(_gsc_rows(prior, "queries"), ("query",))
+ new_rows = [row for key, row in curr.items() if key not in prev]
+ new_rows.sort(key=lambda r: -_num(r.get("impressions")))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(new_rows, limit, max_cap=50)
+ return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_ga4_landing_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ _, ga4 = _gsc_ga4_blobs(data)
+ pages = ga4.get("top_pages") or []
+ if not pages and isinstance(ga4.get("by_path"), dict):
+ pages = [{"path": k, **v} for k, v in ga4["by_path"].items() if isinstance(v, dict)]
+ if not isinstance(pages, list):
+ pages = []
+ pages = sorted([p for p in pages if isinstance(p, dict)], key=lambda r: -_num(r.get("sessions")))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_ga4_pages_by_bounce_rate(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ _, ga4 = _gsc_ga4_blobs(data)
+ pages = list(ga4.get("top_pages") or [])
+ if not pages and isinstance(ga4.get("by_path"), dict):
+ pages = [{"path": k, **v} for k, v in ga4["by_path"].items() if isinstance(v, dict)]
+ pages = [p for p in pages if isinstance(p, dict) and p.get("bounceRate") is not None]
+ pages.sort(key=lambda r: -_num(r.get("bounceRate")))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_ga4_pages_by_engagement_rate(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ _, ga4 = _gsc_ga4_blobs(data)
+ pages = list(ga4.get("top_pages") or [])
+ if not pages and isinstance(ga4.get("by_path"), dict):
+ pages = [{"path": k, **v} for k, v in ga4["by_path"].items() if isinstance(v, dict)]
+ pages = [p for p in pages if isinstance(p, dict) and p.get("engagementRate") is not None]
+ pages.sort(key=lambda r: -_num(r.get("engagementRate")))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _daily_series(data: dict[str, Any] | None, section: str, match_key: str, match_val: str) -> list[dict[str, Any]]:
+ if not data:
+ return []
+ block = data.get(section) if isinstance(data.get(section), dict) else {}
+ daily = block.get("daily") or []
+ if not isinstance(daily, list):
+ return []
+ needle = match_val.strip().lower()
+ out: list[dict[str, Any]] = []
+ for row in daily:
+ if not isinstance(row, dict):
+ continue
+ for field in ("query", "page", "path", "url"):
+ if field in row and str(row.get(field) or "").strip().lower() == needle:
+ out.append(row)
+ break
+ else:
+ dims = row.get("dimensions") if isinstance(row.get("dimensions"), dict) else {}
+ if any(str(dims.get(k) or "").strip().lower() == needle for k in dims):
+ out.append(row)
+ return out
+
+
+def get_gsc_query_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ query = str(args.get("query") or "").strip()
+ if not query:
+ return {"error": "query is required"}
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True}
+ series = _daily_series(data, "gsc", "query", query)
+ if not series:
+ gsc, _ = _gsc_ga4_blobs(data)
+ for row in gsc.get("queries") or []:
+ if isinstance(row, dict) and str(row.get("query") or "").lower() == query.lower():
+ return {"query": query, "snapshot": row, "daily": [], "missing": True, "note": "daily series not stored"}
+ return {"query": query, "daily": series, "fetched_at": data.get("fetched_at")}
+
+
+def get_gsc_page_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or args.get("page") or "").strip()
+ if not url:
+ return {"error": "url is required"}
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True}
+ series = _daily_series(data, "gsc", "page", url)
+ norm = normalize_url(url)
+ if not series:
+ series = _daily_series(data, "gsc", "page", norm)
+ return {"url": url, "daily": series, "fetched_at": data.get("fetched_at"), "missing": not bool(series)}
+
+
+def get_ga4_path_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ path = str(args.get("path") or args.get("url") or "").strip()
+ if not path:
+ return {"error": "path is required"}
+ if path.startswith(("http://", "https://")):
+ path = url_to_path(path)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True}
+ series = _daily_series(data, "ga4", "path", path)
+ return {"path": path, "daily": series, "fetched_at": data.get("fetched_at"), "missing": not bool(series)}
+
+
+def list_gsc_ga4_mismatch_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ gsc, ga4 = _gsc_ga4_blobs(data)
+ by_page = gsc.get("by_page") if isinstance(gsc.get("by_page"), dict) else {}
+ by_path = ga4.get("by_path") if isinstance(ga4.get("by_path"), dict) else {}
+ if not by_page and gsc.get("pages"):
+ by_page = {str(r.get("page")): r for r in (gsc.get("pages") or []) if isinstance(r, dict) and r.get("page")}
+ rows = blend_landing_pages(by_page, by_path, limit=200, min_impressions=0)
+ mismatches: list[dict[str, Any]] = []
+ for row in rows:
+ clicks = _num(row.get("gsc_clicks"))
+ sessions = _num(row.get("ga4_sessions"))
+ if clicks >= 10 and sessions == 0:
+ mismatches.append({**row, "mismatch": "gsc_clicks_no_ga4_sessions"})
+ elif sessions >= 10 and clicks == 0:
+ mismatches.append({**row, "mismatch": "ga4_sessions_no_gsc_clicks"})
+ elif clicks > 0 and sessions / clicks > 3:
+ mismatches.append({**row, "mismatch": "ga4_sessions_high_vs_clicks"})
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(mismatches, limit, max_cap=50)
+ return {
+ "pages": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "provenance": provenance_block(["gsc", "ga4"], data.get("fetched_at")),
+ }
+
+
+def list_gsc_pages_by_position_band(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False}
+ try:
+ min_pos = float(args.get("min_position") or 1)
+ max_pos = float(args.get("max_position") or 20)
+ except (TypeError, ValueError):
+ min_pos, max_pos = 1.0, 20.0
+ pages = [
+ r for r in _gsc_rows(data, "pages")
+ if min_pos <= _num(r.get("position"), 99) <= max_pos
+ ]
+ pages.sort(key=lambda r: _num(r.get("position")))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_gsc_site_benchmarks(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not data:
+ return {"error": "no google data found", "missing": True}
+ gsc, ga4 = _gsc_ga4_blobs(data)
+ pages = _gsc_rows(data, "pages")
+ ctrs = [ctr_as_fraction(r.get("ctr")) for r in pages if _num(r.get("impressions")) > 0]
+ positions = [_num(r.get("position")) for r in pages if _num(r.get("position")) > 0]
+ ctrs.sort()
+ positions.sort()
+ return {
+ "median_ctr": round(ctrs[len(ctrs) // 2], 4) if ctrs else None,
+ "median_position": round(positions[len(positions) // 2], 2) if positions else None,
+ "page_count": len(pages),
+ "gsc_summary": gsc.get("summary") if isinstance(gsc.get("summary"), dict) else {},
+ "ga4_summary": ga4.get("summary") if isinstance(ga4.get("summary"), dict) else {},
+ "fetched_at": data.get("fetched_at"),
+ }
+
+
+def list_gsc_branded_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", "missing": True, "queries": [], "total": 0, "truncated": False}
+ branded = [r for r in (data.get("rows") or []) if isinstance(r, dict) and r.get("is_branded")]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(branded, limit, max_cap=50)
+ return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_gsc_non_branded_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", "missing": True, "queries": [], "total": 0, "truncated": False}
+ non_branded = [r for r in (data.get("rows") or []) if isinstance(r, dict) and not r.get("is_branded")]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(non_branded, limit, max_cap=50)
+ return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def compare_gsc_periods(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ current, prior = _load_google_pair(scoped, conn)
+ if not current:
+ return {"error": "no google data found", "missing": True}
+ if not prior:
+ return {"error": "no prior google snapshot for period comparison", "missing": True}
+ gsc_curr, ga4_curr = _gsc_ga4_blobs(current)
+ gsc_prev, ga4_prev = _gsc_ga4_blobs(prior)
+ curr_summary = gsc_curr.get("summary") if isinstance(gsc_curr.get("summary"), dict) else {}
+ prev_summary = gsc_prev.get("summary") if isinstance(gsc_prev.get("summary"), dict) else {}
+ ga4_curr_summary = ga4_curr.get("summary") if isinstance(ga4_curr.get("summary"), dict) else {}
+ ga4_prev_summary = ga4_prev.get("summary") if isinstance(ga4_prev.get("summary"), dict) else {}
+
+ def _delta(key: str, cur: dict[str, Any], prev_d: dict[str, Any]) -> dict[str, Any]:
+ c = _num(cur.get(key))
+ p = _num(prev_d.get(key))
+ return {"current": c, "prior": p, "delta": round(c - p, 2)}
+
+ return {
+ "gsc": {
+ "clicks": _delta("clicks", curr_summary, prev_summary),
+ "impressions": _delta("impressions", curr_summary, prev_summary),
+ "ctr": _delta("ctr", curr_summary, prev_summary),
+ "position": _delta("position", curr_summary, prev_summary),
+ },
+ "ga4": {
+ "sessions": _delta("sessions", ga4_curr_summary, ga4_prev_summary),
+ "users": _delta("users", ga4_curr_summary, ga4_prev_summary),
+ },
+ "traffic_health": traffic_health_ratio(curr_summary, ga4_curr_summary),
+ "current_fetched_at": current.get("fetched_at"),
+ "prior_fetched_at": prior.get("fetched_at"),
+ "provenance": provenance_block(["gsc", "ga4"], current.get("fetched_at")),
+ }
diff --git a/src/website_profiling/tools/audit_tools/indexation_lists.py b/src/website_profiling/tools/audit_tools/indexation_lists.py
new file mode 100644
index 00000000..bdca54be
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/indexation_lists.py
@@ -0,0 +1,345 @@
+"""Indexation, log analysis, redirect chain, and hreflang list tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...integrations.google.normalize import normalize_url, url_to_path
+from ...reporting.categories import REDIRECT_CHAIN_LONG
+from ._slice import _parse_page_analysis, cap_list, parse_limit
+from .context import AuditToolContext
+from .ops import _load_log_analysis
+
+_REDIRECT_CHAIN_MIN = REDIRECT_CHAIN_LONG
+
+
+def _norm_path(url: str) -> str:
+ try:
+ return url_to_path(str(url or "")) or "/"
+ except Exception:
+ return str(url or "")
+
+
+def _indexation_cov(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return None, {"error": "no report found", "urls": [], "total": 0, "truncated": False}
+ cov = payload.get("indexation_coverage")
+ if not isinstance(cov, dict):
+ return payload, {
+ "error": "indexation_coverage not in report",
+ "missing": True,
+ "urls": [],
+ "total": 0,
+ "truncated": False,
+ }
+ return payload, cov
+
+
+def _cap_indexation_urls(
+ urls: list[Any],
+ args: dict[str, Any],
+ *,
+ gap_type: str = "",
+ totals: dict[str, Any] | None = None,
+ total_key: str = "",
+) -> dict[str, Any]:
+ if not isinstance(urls, list):
+ urls = []
+ items = [{"url": str(u)} if not isinstance(u, dict) else u for u in urls if u]
+ limit = parse_limit(args.get("limit"), 50, 200)
+ sliced = cap_list(items, limit, max_cap=200)
+ total_all = None
+ if totals and total_key:
+ total_all = totals.get(total_key)
+ return {
+ "gap_type": gap_type,
+ "urls": sliced["items"],
+ "total": int(total_all or sliced["total"]),
+ "truncated": sliced["truncated"] or (int(total_all or 0) > limit),
+ }
+
+
+def list_indexation_submitted_not_indexed(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """URLs in sitemap but not successfully crawled (submitted, not indexed in crawl)."""
+ _, cov = _indexation_cov(conn, ctx, args)
+ if cov.get("error"):
+ return cov
+ assert isinstance(cov, dict)
+ lists = cov.get("lists") or {}
+ urls = lists.get("sitemap_only") or []
+ totals = cov.get("lists_total") or {}
+ return _cap_indexation_urls(urls, args, gap_type="sitemap_only", totals=totals, total_key="sitemap_only")
+
+
+def list_indexation_indexed_not_submitted(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """Crawled URLs not present in sitemap (found/indexable but not submitted)."""
+ _, cov = _indexation_cov(conn, ctx, args)
+ if cov.get("error"):
+ return cov
+ assert isinstance(cov, dict)
+ lists = cov.get("lists") or {}
+ urls = lists.get("crawled_not_in_sitemap") or []
+ totals = cov.get("lists_total") or {}
+ return _cap_indexation_urls(urls, args, gap_type="crawled_not_in_sitemap", totals=totals, total_key="crawled_not_in_sitemap")
+
+
+def list_sitemap_urls_not_in_crawl(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ _, cov = _indexation_cov(conn, ctx, args)
+ if cov.get("error"):
+ return cov
+ assert isinstance(cov, dict)
+ lists = cov.get("lists") or {}
+ urls = lists.get("sitemap_only") or []
+ totals = cov.get("lists_total") or {}
+ result = _cap_indexation_urls(urls, args, gap_type="sitemap_only", totals=totals, total_key="sitemap_only")
+ result["source"] = "indexation_coverage.lists.sitemap_only"
+ return result
+
+
+def list_crawl_urls_not_in_sitemap(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ _, cov = _indexation_cov(conn, ctx, args)
+ if cov.get("error"):
+ return cov
+ assert isinstance(cov, dict)
+ lists = cov.get("lists") or {}
+ urls = lists.get("crawled_not_in_sitemap") or []
+ totals = cov.get("lists_total") or {}
+ result = _cap_indexation_urls(urls, args, gap_type="crawled_not_in_sitemap", totals=totals, total_key="crawled_not_in_sitemap")
+ result["source"] = "indexation_coverage.lists.crawled_not_in_sitemap"
+ return result
+
+
+def _require_log(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any]]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return None, {"error": "property_id is required", "paths": [], "total": 0, "truncated": False}
+ row = _load_log_analysis(conn, int(scoped.property_id))
+ if not row:
+ return None, {"error": "no log uploads found", "missing": True, "paths": [], "total": 0, "truncated": False}
+ return row, {}
+
+
+def list_log_paths_by_hits(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ row, err = _require_log(conn, ctx, args)
+ if err:
+ return err
+ assert row is not None
+ analysis = row.get("analysis") or {}
+ paths = analysis.get("top_paths") or []
+ if not isinstance(paths, list):
+ paths = []
+ limit = parse_limit(args.get("limit"), 30, 100)
+ sliced = cap_list(paths, limit, max_cap=100)
+ return {
+ "paths": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "upload_id": row.get("upload_id"),
+ "filename": row.get("filename"),
+ }
+
+
+def list_log_5xx_paths(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ row, err = _require_log(conn, ctx, args)
+ if err:
+ return err
+ assert row is not None
+ analysis = row.get("analysis") or {}
+ paths = analysis.get("paths_5xx") or []
+ if not isinstance(paths, list):
+ paths = []
+ if not paths:
+ status_counts = analysis.get("status_counts") or {}
+ paths = [
+ {"path": f"status_{code}", "hits": hits, "aggregate": True}
+ for code, hits in status_counts.items()
+ if str(code).startswith("5")
+ ]
+ limit = parse_limit(args.get("limit"), 30, 100)
+ sliced = cap_list(paths, limit, max_cap=100)
+ return {
+ "paths": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "upload_id": row.get("upload_id"),
+ }
+
+
+def list_log_googlebot_low_crawl(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """High-traffic log paths with low or zero Googlebot hits (under-crawled)."""
+ scoped = ctx.with_args(args)
+ row, err = _require_log(conn, ctx, args)
+ if err:
+ return err
+ assert row is not None
+ analysis = row.get("analysis") or {}
+ top_paths = analysis.get("top_paths") or []
+ bot_paths = {
+ str(r.get("path") or ""): int(r.get("hits") or 0)
+ for r in (analysis.get("googlebot_paths") or [])
+ if isinstance(r, dict) and r.get("path")
+ }
+ try:
+ min_hits = int(args.get("min_hits") or 20)
+ max_bot_hits = int(args.get("max_googlebot_hits") or 0)
+ except (TypeError, ValueError):
+ min_hits, max_bot_hits = 20, 0
+ payload = scoped.load_payload(conn)
+ crawl_paths: set[str] = set()
+ if payload:
+ for link in payload.get("links") or []:
+ if isinstance(link, dict) and link.get("url"):
+ crawl_paths.add(_norm_path(str(link["url"])))
+ items: list[dict[str, Any]] = []
+ for row_data in top_paths if isinstance(top_paths, list) else []:
+ if not isinstance(row_data, dict):
+ continue
+ path = str(row_data.get("path") or "")
+ hits = int(row_data.get("hits") or 0)
+ bot_hits = bot_paths.get(path, 0)
+ if hits < min_hits:
+ continue
+ if bot_hits > max_bot_hits:
+ continue
+ if path in crawl_paths and bot_hits > 0:
+ continue
+ items.append({
+ "path": path,
+ "total_hits": hits,
+ "googlebot_hits": bot_hits,
+ "in_crawl": path in crawl_paths,
+ })
+ items.sort(key=lambda x: (-x["total_hits"], x["googlebot_hits"]))
+ limit = parse_limit(args.get("limit"), 30, 100)
+ sliced = cap_list(items, limit, max_cap=100)
+ return {"paths": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_log_orphan_high_traffic(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """Access-log paths with high hits that map to crawl orphan URLs."""
+ scoped = ctx.with_args(args)
+ row, err = _require_log(conn, ctx, args)
+ if err:
+ return err
+ assert row is not None
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "paths": [], "total": 0, "truncated": False}
+ orphan_path_set: set[str] = set()
+ for url in payload.get("orphan_urls") or []:
+ if not url:
+ continue
+ path = _norm_path(str(url))
+ orphan_path_set.add(path)
+ orphan_path_set.add(path.rstrip("/") or "/")
+ if not orphan_path_set:
+ return {"paths": [], "total": 0, "truncated": False, "note": "no orphan URLs in report"}
+ analysis = row.get("analysis") or {}
+ top_paths = analysis.get("top_paths") or []
+ try:
+ min_hits = int(args.get("min_hits") or 10)
+ except (TypeError, ValueError):
+ min_hits = 10
+ items: list[dict[str, Any]] = []
+ for row_data in top_paths if isinstance(top_paths, list) else []:
+ if not isinstance(row_data, dict):
+ continue
+ path = str(row_data.get("path") or "")
+ hits = int(row_data.get("hits") or 0)
+ if hits < min_hits:
+ continue
+ if path not in orphan_path_set and path.rstrip("/") not in orphan_path_set:
+ continue
+ items.append({"path": path, "hits": hits})
+ items.sort(key=lambda x: -x["hits"])
+ limit = parse_limit(args.get("limit"), 30, 100)
+ sliced = cap_list(items, limit, max_cap=100)
+ return {"paths": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_redirect_chains_by_length(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False}
+ try:
+ min_length = int(args.get("min_length") or args.get("chain_length") or _REDIRECT_CHAIN_MIN)
+ except (TypeError, ValueError):
+ min_length = _REDIRECT_CHAIN_MIN
+ pages: list[dict[str, Any]] = []
+ for _, row in df.iterrows():
+ rec = row.to_dict()
+ try:
+ chain_len = int(rec.get("redirect_chain_length") or 0)
+ except (TypeError, ValueError):
+ chain_len = 0
+ if chain_len < min_length:
+ continue
+ pages.append({
+ "url": str(rec.get("url") or ""),
+ "status": str(rec.get("status") or ""),
+ "redirect_chain_length": chain_len,
+ "final_url": str(rec.get("final_url") or ""),
+ })
+ pages.sort(key=lambda p: -int(p.get("redirect_chain_length") or 0))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {
+ "pages": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "min_length": min_length,
+ }
+
+
+def list_hreflang_reciprocal_gaps(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """Pages whose hreflang alternates do not link back reciprocally."""
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False, "missing": True}
+ href_map: dict[str, set[str]] = {}
+ url_by_norm: dict[str, str] = {}
+ for _, row in df.iterrows():
+ rec = row.to_dict()
+ if not str(rec.get("status") or "").startswith("2"):
+ continue
+ url = str(rec.get("url") or "").strip()
+ if not url:
+ continue
+ norm = normalize_url(url)
+ url_by_norm[norm] = url
+ pa = _parse_page_analysis(rec)
+ alts = pa.get("hreflang_alternates") or []
+ targets: set[str] = set()
+ for alt in alts if isinstance(alts, list) else []:
+ if not isinstance(alt, dict):
+ continue
+ href = str(alt.get("href") or "").strip()
+ if href:
+ targets.add(normalize_url(href))
+ if targets:
+ href_map[norm] = targets
+ gaps: list[dict[str, Any]] = []
+ for src_norm, targets in href_map.items():
+ src_url = url_by_norm.get(src_norm, src_norm)
+ missing_returns: list[str] = []
+ for tgt_norm in targets:
+ if tgt_norm == src_norm:
+ continue
+ tgt_targets = href_map.get(tgt_norm, set())
+ if src_norm not in tgt_targets:
+ missing_returns.append(url_by_norm.get(tgt_norm, tgt_norm))
+ if missing_returns:
+ gaps.append({
+ "url": src_url,
+ "missing_reciprocal_from": missing_returns[:10],
+ "gap_count": len(missing_returns),
+ })
+ gaps.sort(key=lambda g: -int(g.get("gap_count") or 0))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(gaps, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/insight_helpers.py b/src/website_profiling/tools/audit_tools/insight_helpers.py
new file mode 100644
index 00000000..e38d09ff
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/insight_helpers.py
@@ -0,0 +1,207 @@
+"""Shared helpers for cross-platform GSC + GA4 + audit insight tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from ...integrations.google.normalize import normalize_url, url_to_path
+
+
+def provenance_block(
+ sources: list[str],
+ fetched_at: str | None = None,
+ *,
+ confidence: str = "high",
+) -> dict[str, Any]:
+ return {
+ "sources": sources,
+ "fetched_at": fetched_at,
+ "confidence": confidence,
+ }
+
+
+def _num(val: Any, default: float = 0.0) -> float:
+ try:
+ if val is None:
+ return default
+ return float(val)
+ except (TypeError, ValueError):
+ return default
+
+
+def classify_opportunity_quadrant(
+ gsc_row: dict[str, Any] | None,
+ ga4_row: dict[str, Any] | None,
+ *,
+ site_median_sessions: float = 0.0,
+) -> str:
+ position = _num((gsc_row or {}).get("position"), 99)
+ impressions = _num((gsc_row or {}).get("impressions"))
+ sessions = _num((ga4_row or {}).get("sessions"))
+ engagement = _num((ga4_row or {}).get("engagementRate"))
+
+ rank_potential = impressions >= 100 and 4 <= position <= 20
+ convert_potential = sessions >= max(site_median_sessions * 0.5, 5) or engagement >= 0.5
+
+ if rank_potential and convert_potential:
+ return "high_impact"
+ if rank_potential:
+ return "worth_optimizing"
+ if convert_potential:
+ return "good_but_capped"
+ return "low_priority"
+
+
+def traffic_health_ratio(
+ gsc_summary: dict[str, Any] | None,
+ ga4_summary: dict[str, Any] | None,
+) -> dict[str, Any]:
+ clicks = _num((gsc_summary or {}).get("clicks"))
+ sessions = _num((ga4_summary or {}).get("sessions"))
+ if clicks <= 0 and sessions <= 0:
+ return {
+ "gsc_clicks": clicks,
+ "ga4_sessions": sessions,
+ "ratio": None,
+ "diagnosis": "no_data",
+ "note": "Connect GSC and GA4 and re-run the pipeline.",
+ }
+ ratio = sessions / clicks if clicks > 0 else None
+ diagnosis = "healthy"
+ note = "GSC clicks and GA4 sessions are in a plausible range."
+ if ratio is not None:
+ if ratio < 0.3:
+ diagnosis = "tracking_gap"
+ note = "GA4 sessions are much lower than GSC clicks — check filters, consent mode, or landing page tagging."
+ elif ratio > 3.0:
+ diagnosis = "filter_issue"
+ note = "GA4 sessions exceed GSC clicks — GA4 may include non-organic traffic or GSC date range differs."
+ return {
+ "gsc_clicks": clicks,
+ "ga4_sessions": sessions,
+ "ratio": round(ratio, 3) if ratio is not None else None,
+ "diagnosis": diagnosis,
+ "note": note,
+ }
+
+
+def blend_landing_pages(
+ gsc_by_page: dict[str, Any],
+ ga4_by_path: dict[str, Any],
+ *,
+ limit: int = 50,
+ min_impressions: int = 0,
+) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ ga4_by_norm: dict[str, dict[str, Any]] = {}
+ for path, val in (ga4_by_path or {}).items():
+ if not isinstance(val, dict):
+ continue
+ full = str(val.get("full_url") or path)
+ ga4_by_norm[normalize_url(full)] = val
+ ga4_by_norm[normalize_url(str(path))] = val
+
+ session_vals = [_num(v.get("sessions")) for v in ga4_by_norm.values() if isinstance(v, dict)]
+ session_vals.sort()
+ median_sessions = session_vals[len(session_vals) // 2] if session_vals else 0.0
+
+ for page_url, gsc_row in (gsc_by_page or {}).items():
+ if not isinstance(gsc_row, dict):
+ continue
+ impressions = _num(gsc_row.get("impressions"))
+ if impressions < min_impressions:
+ continue
+ norm = normalize_url(str(page_url))
+ ga4_row = ga4_by_norm.get(norm)
+ if ga4_row is None:
+ path = url_to_path(str(page_url))
+ ga4_row = ga4_by_norm.get(normalize_url(path))
+ quadrant = classify_opportunity_quadrant(
+ gsc_row, ga4_row if isinstance(ga4_row, dict) else None,
+ site_median_sessions=median_sessions,
+ )
+ rows.append({
+ "url": page_url,
+ "gsc_clicks": int(_num(gsc_row.get("clicks"))),
+ "gsc_impressions": int(impressions),
+ "gsc_position": round(_num(gsc_row.get("position")), 1),
+ "gsc_ctr": round(_num(gsc_row.get("ctr")), 4),
+ "ga4_sessions": int(_num((ga4_row or {}).get("sessions"))) if ga4_row else 0,
+ "ga4_engagement_rate": round(_num((ga4_row or {}).get("engagementRate")), 3) if ga4_row else None,
+ "quadrant": quadrant,
+ })
+
+ rows.sort(key=lambda r: (-r["gsc_clicks"], -r["gsc_impressions"]))
+ return rows[: max(1, min(limit, 100))]
+
+
+def page_issue_flags(url: str, payload: dict[str, Any]) -> list[dict[str, Any]]:
+ norm = normalize_url(url)
+ flags: list[dict[str, Any]] = []
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ for issue in cat.get("issues") or []:
+ if not isinstance(issue, dict):
+ continue
+ issue_url = str(issue.get("url") or "")
+ if issue_url and normalize_url(issue_url) != norm:
+ continue
+ flags.append({
+ "priority": issue.get("priority"),
+ "category_id": cat.get("id"),
+ "message": issue.get("message"),
+ "url": issue_url or url,
+ })
+ return flags[:30]
+
+
+def composite_page_score(
+ gsc_page: dict[str, Any] | None,
+ ga4_page: dict[str, Any] | None,
+ gsc_site: dict[str, Any] | None,
+ ga4_site: dict[str, Any] | None,
+ issue_flags: list[dict[str, Any]],
+ lighthouse: dict[str, Any] | None,
+) -> dict[str, Any]:
+ score = 75.0
+ flags_out: list[str] = []
+
+ site_pos = _num((gsc_site or {}).get("position"), 10)
+ page_pos = _num((gsc_page or {}).get("position"), site_pos)
+ if page_pos > site_pos + 5:
+ score -= 10
+ flags_out.append("below_avg_gsc_position")
+
+ site_eng = _num((ga4_site or {}).get("engagementRate"), 0.5)
+ page_eng = _num((ga4_page or {}).get("engagementRate"), site_eng)
+ if ga4_page and page_eng < site_eng * 0.7:
+ score -= 10
+ flags_out.append("low_engagement")
+
+ crit = sum(1 for f in issue_flags if str(f.get("priority")) == "Critical")
+ high = sum(1 for f in issue_flags if str(f.get("priority")) == "High")
+ if crit:
+ score -= min(20, crit * 10)
+ flags_out.append("critical_issues")
+ elif high:
+ score -= min(10, high * 5)
+ flags_out.append("high_issues")
+
+ if lighthouse:
+ perf = _num(lighthouse.get("performance"), 100)
+ seo = _num(lighthouse.get("seo"), 100)
+ if perf < 50:
+ score -= 8
+ flags_out.append("poor_lighthouse_performance")
+ if seo < 70:
+ score -= 5
+ flags_out.append("poor_lighthouse_seo")
+
+ score = max(0, min(100, round(score)))
+ if score >= 75:
+ band = "green"
+ elif score >= 50:
+ band = "amber"
+ else:
+ band = "red"
+ return {"score": score, "band": band, "flags": flags_out}
diff --git a/src/website_profiling/tools/audit_tools/insight_tools.py b/src/website_profiling/tools/audit_tools/insight_tools.py
new file mode 100644
index 00000000..1d9357b2
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/insight_tools.py
@@ -0,0 +1,166 @@
+"""Cross-platform insight audit tools (GSC + GA4 + crawl + issues)."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ...integrations.google.page_lookup import slice_from_google_row
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+from .insight_helpers import (
+ blend_landing_pages,
+ composite_page_score,
+ page_issue_flags,
+ provenance_block,
+ traffic_health_ratio,
+)
+from .report import list_issues
+
+
+def _gsc_ga4_blobs(raw: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
+ gsc = raw.get("gsc_full") if isinstance(raw.get("gsc_full"), dict) else raw.get("gsc") or {}
+ ga4 = raw.get("ga4_full") if isinstance(raw.get("ga4_full"), dict) else raw.get("ga4") or {}
+ return gsc if isinstance(gsc, dict) else {}, ga4 if isinstance(ga4, dict) else {}
+
+
+def get_landing_page_blended_table(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ raw = scoped.load_google_full(conn)
+ if not raw:
+ return {"error": "no google data found", "missing": True, "rows": []}
+ gsc, ga4 = _gsc_ga4_blobs(raw)
+ by_page = gsc.get("by_page") if isinstance(gsc.get("by_page"), dict) else {}
+ by_path = ga4.get("by_path") if isinstance(ga4.get("by_path"), dict) else {}
+ if not by_page and gsc.get("top_pages"):
+ by_page = {str(r.get("page")): r for r in (gsc.get("top_pages") or []) if isinstance(r, dict) and r.get("page")}
+ limit = parse_limit(args.get("limit"), 30, 100)
+ min_impressions = parse_limit(args.get("min_impressions"), 0, 1_000_000)
+ rows = blend_landing_pages(by_page, by_path, limit=limit, min_impressions=min_impressions)
+ return {
+ "rows": rows,
+ "total": len(rows),
+ "truncated": len(by_page) > limit,
+ "provenance": provenance_block(["gsc", "ga4"], raw.get("fetched_at")),
+ "insights": [
+ f"{sum(1 for r in rows if r['quadrant'] == 'high_impact')} high-impact landing pages",
+ f"{sum(1 for r in rows if r['quadrant'] == 'worth_optimizing')} worth optimizing for rank",
+ ],
+ }
+
+
+def get_opportunity_matrix(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ blended = get_landing_page_blended_table(conn, ctx, args)
+ if blended.get("error"):
+ return blended
+ quadrants: dict[str, list[dict[str, Any]]] = {
+ "high_impact": [],
+ "worth_optimizing": [],
+ "good_but_capped": [],
+ "low_priority": [],
+ }
+ for row in blended.get("rows") or []:
+ q = str(row.get("quadrant") or "low_priority")
+ quadrants.setdefault(q, []).append(row)
+ counts = {k: len(v) for k, v in quadrants.items()}
+ return {
+ "quadrants": quadrants,
+ "counts": counts,
+ "provenance": blended.get("provenance"),
+ "insights": [
+ f"Focus on {counts.get('high_impact', 0)} high-impact pages first.",
+ f"{counts.get('worth_optimizing', 0)} pages could rank higher with on-page work.",
+ ],
+ }
+
+
+def get_traffic_health_check(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ raw = scoped.load_google_full(conn) or scoped.load_google(conn)
+ if not raw:
+ return {"error": "no google data found", "missing": True}
+ gsc, ga4 = _gsc_ga4_blobs(raw)
+ health = traffic_health_ratio(
+ gsc.get("summary") if isinstance(gsc.get("summary"), dict) else {},
+ ga4.get("summary") if isinstance(ga4.get("summary"), dict) else {},
+ )
+ return {
+ **health,
+ "provenance": provenance_block(["gsc", "ga4"], raw.get("fetched_at")),
+ "insights": [health.get("note") or ""],
+ }
+
+
+def get_landing_page_full_diagnosis(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ url = str(args.get("url") or "").strip()
+ if not url:
+ return {"error": "url is required"}
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "missing": True}
+ raw = scoped.load_google_full(conn) or scoped.load_google(conn) or {}
+ slice_data = slice_from_google_row(raw, url)
+ gsc_page = (slice_data.get("gsc") or {}) if isinstance(slice_data.get("gsc"), dict) else None
+ ga4_page = (slice_data.get("ga4") or {}) if isinstance(slice_data.get("ga4"), dict) else None
+ benchmarks = slice_data.get("siteBenchmarks") or {}
+ flags = page_issue_flags(url, payload)
+ lh = (payload.get("lighthouse_by_url") or {}).get(url) or {}
+ if not lh:
+ norm_url = url.rstrip("/")
+ for k, v in (payload.get("lighthouse_by_url") or {}).items():
+ if str(k).rstrip("/") == norm_url:
+ lh = v
+ break
+ score = composite_page_score(
+ gsc_page, ga4_page,
+ (benchmarks.get("gsc") or {}) if isinstance(benchmarks.get("gsc"), dict) else {},
+ (benchmarks.get("ga4") or {}) if isinstance(benchmarks.get("ga4"), dict) else {},
+ flags,
+ lh if isinstance(lh, dict) else None,
+ )
+ crawl_row = None
+ for row in payload.get("top_pages") or []:
+ if isinstance(row, dict) and str(row.get("url") or "").rstrip("/") == url.rstrip("/"):
+ crawl_row = row
+ break
+ return {
+ "url": url,
+ "gsc_ga4": slice_data,
+ "issues": flags,
+ "lighthouse": lh or None,
+ "crawl": crawl_row,
+ "diagnosis": score,
+ "provenance": provenance_block(
+ ["gsc", "ga4", "crawl", "audit"],
+ raw.get("fetched_at") or payload.get("report_generated_at"),
+ ),
+ "insights": score.get("flags") or [],
+ }
+
+
+def get_issue_to_traffic_map(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ result = list_issues(conn, ctx, {**args, "sort": "impact"})
+ if result.get("error"):
+ return result
+ issues = result.get("issues") or []
+ rows = []
+ for issue in issues:
+ if not isinstance(issue, dict):
+ continue
+ rows.append({
+ "url": issue.get("url"),
+ "priority": issue.get("priority"),
+ "category": issue.get("category"),
+ "message": issue.get("message"),
+ "impact_score": issue.get("impact_score"),
+ "gsc_clicks": issue.get("gsc_clicks"),
+ "ga4_sessions": issue.get("ga4_sessions"),
+ })
+ return {
+ "issues": rows,
+ "total": result.get("total"),
+ "truncated": result.get("truncated"),
+ "provenance": provenance_block(["audit", "gsc", "ga4"]),
+ "insights": ["Issues sorted by traffic-weighted impact_score."],
+ }
diff --git a/src/website_profiling/tools/audit_tools/issue_lists.py b/src/website_profiling/tools/audit_tools/issue_lists.py
new file mode 100644
index 00000000..edd870f9
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/issue_lists.py
@@ -0,0 +1,526 @@
+"""Issue and gap list tools from report payload buckets and crawl columns."""
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any, Callable
+
+import pandas as pd
+from psycopg import Connection
+
+from ...reporting.categories._helpers import (
+ RESPONSE_TIME_SLOW_MS,
+ TITLE_LEN_MAX,
+ TITLE_LEN_MIN,
+ _hreflang_issues,
+ _orphan_hub_suggestions,
+)
+from ...reporting.categories.accessibility import contrast_issues_from_sources
+from ._slice import _parse_page_analysis, _row_schema_types_list, cap_list, parse_limit
+from .context import AuditToolContext
+
+_READING_LEVEL_HIGH = 12.0
+_VERY_THIN_WORDS = 100
+
+
+def _payload_url_bucket(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ bucket: str,
+ *,
+ item_key: str = "pages",
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", item_key: [], "total": 0, "truncated": False}
+ content_urls = payload.get("content_urls") or {}
+ if not isinstance(content_urls, dict):
+ return {"error": "content_urls not in report", "missing": True, item_key: [], "total": 0, "truncated": False}
+ items = content_urls.get(bucket) or []
+ if not isinstance(items, list):
+ items = []
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(items, limit, max_cap=50)
+ return {
+ "bucket": bucket,
+ item_key: sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ }
+
+
+def _payload_list_key(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ key: str,
+ *,
+ item_key: str = "pages",
+ nested: str = "",
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", item_key: [], "total": 0, "truncated": False}
+ raw = payload.get(key)
+ if nested and isinstance(raw, dict):
+ items = raw.get(nested) or []
+ elif isinstance(raw, list):
+ items = raw
+ else:
+ items = []
+ if not items:
+ return {"missing": True, item_key: [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(items if isinstance(items, list) else [], limit, max_cap=50)
+ return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _filter_crawl_df(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ *,
+ predicate: Callable[[dict[str, Any]], bool],
+ projection: Callable[[dict[str, Any]], dict[str, Any]],
+ only_2xx: bool = True,
+ item_key: str = "pages",
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {item_key: [], "total": 0, "truncated": False}
+ work = df
+ if only_2xx and "status" in df.columns:
+ work = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)]
+ pages: list[dict[str, Any]] = []
+ for _, row in work.iterrows():
+ rec = row.to_dict()
+ if predicate(rec):
+ pages.append(projection(rec))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _issues_by_type(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ issue_type: str,
+ *,
+ item_key: str = "issues",
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", item_key: [], "total": 0, "truncated": False}
+ issues_root = payload.get("issues") or {}
+ seo = issues_root.get("seo") if isinstance(issues_root, dict) else []
+ if not isinstance(seo, list):
+ seo = []
+ needle = issue_type.strip().lower()
+ filtered = [
+ x for x in seo
+ if isinstance(x, dict) and str(x.get("type") or "").lower() == needle
+ ]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(filtered, limit, max_cap=50)
+ return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _bucket_or_crawl(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ bucket: str,
+ *,
+ predicate: Callable[[dict[str, Any]], bool],
+ projection: Callable[[dict[str, Any]], dict[str, Any]],
+) -> dict[str, Any]:
+ result = _payload_url_bucket(conn, ctx, args, bucket)
+ if result.get("total", 0) > 0 or result.get("missing"):
+ return result
+ return _filter_crawl_df(conn, ctx, args, predicate=predicate, projection=projection)
+
+
+def list_pages_title_too_short(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ def _short(r: dict[str, Any]) -> bool:
+ try:
+ tl = int(r.get("title_length") or 0)
+ except (TypeError, ValueError):
+ tl = 0
+ return tl > 0 and tl < TITLE_LEN_MIN
+
+ return _bucket_or_crawl(
+ conn, ctx, args, "title_short",
+ predicate=_short,
+ projection=lambda r: {
+ "url": str(r.get("url") or ""),
+ "title": str(r.get("title") or ""),
+ "title_length": int(r.get("title_length") or 0),
+ },
+ )
+
+
+def list_pages_title_too_long(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ def _long(r: dict[str, Any]) -> bool:
+ try:
+ tl = int(r.get("title_length") or 0)
+ except (TypeError, ValueError):
+ tl = 0
+ return tl > TITLE_LEN_MAX
+
+ return _bucket_or_crawl(
+ conn, ctx, args, "title_long",
+ predicate=_long,
+ projection=lambda r: {
+ "url": str(r.get("url") or ""),
+ "title": str(r.get("title") or ""),
+ "title_length": int(r.get("title_length") or 0),
+ },
+ )
+
+
+def list_pages_slow_response(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ try:
+ threshold = int(args.get("threshold_ms") or RESPONSE_TIME_SLOW_MS)
+ except (TypeError, ValueError):
+ threshold = RESPONSE_TIME_SLOW_MS
+
+ def _slow(r: dict[str, Any]) -> bool:
+ try:
+ ms = float(r.get("response_time_ms") or 0)
+ except (TypeError, ValueError):
+ ms = 0
+ return ms >= threshold
+
+ return _bucket_or_crawl(
+ conn, ctx, args, "slow_response",
+ predicate=_slow,
+ projection=lambda r: {
+ "url": str(r.get("url") or ""),
+ "response_time_ms": float(r.get("response_time_ms") or 0),
+ "status": str(r.get("status") or ""),
+ },
+ )
+
+
+def list_pages_missing_html_lang(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ def _missing(r: dict[str, Any]) -> bool:
+ pa = _parse_page_analysis(r)
+ return not str(pa.get("html_lang") or "").strip()
+
+ return _bucket_or_crawl(
+ conn, ctx, args, "missing_html_lang",
+ predicate=_missing,
+ projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")},
+ )
+
+
+def list_pages_invalid_viewport(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ def _truthy(val: Any) -> bool:
+ return str(val or "").lower() in ("true", "1", "yes")
+
+ def _invalid(r: dict[str, Any]) -> bool:
+ if not _truthy(r.get("viewport_present")):
+ return False
+ content = str(r.get("viewport_content") or "").strip()
+ return not content or "width" not in content.lower() and "device-width" not in content.lower()
+
+ bucket = _payload_url_bucket(conn, ctx, args, "invalid_viewport")
+ if bucket.get("total", 0) > 0 or bucket.get("missing"):
+ return bucket
+ return _filter_crawl_df(conn, ctx, args, predicate=_invalid, projection=lambda r: {
+ "url": str(r.get("url") or ""),
+ "viewport_content": str(r.get("viewport_content") or ""),
+ })
+
+
+def list_pages_color_contrast_failures(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ df = scoped.load_crawl_df(conn)
+ lh = payload.get("lighthouse_by_url") if isinstance(payload.get("lighthouse_by_url"), dict) else {}
+ issues = contrast_issues_from_sources(df if df is not None else pd.DataFrame(), lh)
+ pages = [
+ {"url": str(i.get("url") or ""), "message": str(i.get("message") or "")}
+ for i in issues if isinstance(i, dict) and i.get("url")
+ ]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_pages_high_reading_level(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ try:
+ min_grade = float(args.get("min_reading_level") or _READING_LEVEL_HIGH)
+ except (TypeError, ValueError):
+ min_grade = _READING_LEVEL_HIGH
+
+ def _high(r: dict[str, Any]) -> bool:
+ try:
+ lvl = float(r.get("reading_level") or 0)
+ except (TypeError, ValueError):
+ lvl = 0
+ return lvl >= min_grade
+
+ return _bucket_or_crawl(
+ conn, ctx, args, "high_reading_level",
+ predicate=_high,
+ projection=lambda r: {
+ "url": str(r.get("url") or ""),
+ "reading_level": float(r.get("reading_level") or 0),
+ "title": str(r.get("title") or ""),
+ },
+ )
+
+
+def list_pages_very_thin_content(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ try:
+ max_words = int(args.get("max_word_count") or _VERY_THIN_WORDS)
+ except (TypeError, ValueError):
+ max_words = _VERY_THIN_WORDS
+
+ def _thin(r: dict[str, Any]) -> bool:
+ try:
+ wc = int(r.get("word_count") or 0)
+ except (TypeError, ValueError):
+ wc = 0
+ return 0 < wc < max_words
+
+ bucket = _payload_url_bucket(conn, ctx, args, "very_thin_content")
+ if bucket.get("total", 0) > 0 or bucket.get("missing"):
+ return bucket
+ return _filter_crawl_df(
+ conn, ctx, args,
+ predicate=_thin,
+ projection=lambda r: {
+ "url": str(r.get("url") or ""),
+ "word_count": int(r.get("word_count") or 0),
+ },
+ )
+
+
+def list_hreflang_issue_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ payload_result = _payload_list_key(conn, ctx, args, "hreflang_issue_urls")
+ if payload_result.get("total", 0) > 0:
+ return payload_result
+ scoped = ctx.with_args(args)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False, "missing": True}
+ success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df
+ issues = _hreflang_issues(success)
+ pages = [{"url": str(i.get("url") or ""), "message": str(i.get("message") or "")} for i in issues]
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_pages_missing_og_tags(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ social = payload.get("social_coverage") if isinstance(payload.get("social_coverage"), dict) else {}
+ urls = social.get("missing_og") or []
+ if isinstance(urls, list) and urls:
+ limit = parse_limit(args.get("limit"), 30, 50)
+ pages = [{"url": str(u)} for u in urls if u]
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+ return _filter_crawl_df(
+ conn, ctx, args,
+ predicate=lambda r: not str(r.get("og_title") or "").strip(),
+ projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")},
+ )
+
+
+def list_pages_missing_twitter_cards(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ social = payload.get("social_coverage") if isinstance(payload.get("social_coverage"), dict) else {}
+ urls = social.get("missing_twitter") or []
+ if isinstance(urls, list) and urls:
+ limit = parse_limit(args.get("limit"), 30, 50)
+ pages = [{"url": str(u)} for u in urls if u]
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+ return _filter_crawl_df(
+ conn, ctx, args,
+ predicate=lambda r: not str(r.get("twitter_card") or "").strip(),
+ projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")},
+ )
+
+
+def list_pages_invalid_json_ld(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ def _invalid(r: dict[str, Any]) -> bool:
+ has_schema = str(r.get("has_schema") or "").lower() in ("true", "1", "yes")
+ types = _row_schema_types_list(r)
+ return has_schema and not types
+
+ return _filter_crawl_df(
+ conn, ctx, args,
+ predicate=_invalid,
+ projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")},
+ )
+
+
+def list_pages_mixed_language(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ df = scoped.load_crawl_df(conn)
+ if df is None or df.empty:
+ return {"pages": [], "total": 0, "truncated": False, "missing": True}
+ lang_summary = payload.get("language_summary") if isinstance(payload.get("language_summary"), dict) else {}
+ counts = lang_summary.get("counts") if isinstance(lang_summary.get("counts"), dict) else {}
+ if counts:
+ dominant = max(counts.items(), key=lambda x: x[1])[0]
+ else:
+ dominant = ""
+ pages: list[dict[str, Any]] = []
+ for _, row in df.iterrows():
+ rec = row.to_dict()
+ if not str(rec.get("status") or "").startswith("2"):
+ continue
+ lang = str(rec.get("detected_language") or rec.get("language") or "").strip().lower()
+ if not lang or not dominant:
+ continue
+ if lang != str(dominant).lower():
+ pages.append({
+ "url": str(rec.get("url") or ""),
+ "language": lang,
+ "dominant_language": dominant,
+ })
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_orphan_hub_suggestions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "suggestions": [], "total": 0, "truncated": False}
+ orphans = payload.get("orphan_urls") or []
+ if not isinstance(orphans, list):
+ orphans = []
+ edges_raw = payload.get("graph_edges") or []
+ edges: list[tuple[str, str]] = []
+ nodes = payload.get("graph_nodes") or []
+ node_urls: dict[int, str] = {}
+ if isinstance(nodes, list):
+ for i, n in enumerate(nodes):
+ if isinstance(n, dict):
+ node_urls[i] = str(n.get("url") or n.get("id") or "")
+ elif isinstance(n, str):
+ node_urls[i] = n
+ for e in edges_raw:
+ if isinstance(e, (list, tuple)) and len(e) >= 2:
+ src = node_urls.get(int(e[0]), str(e[0]))
+ tgt = node_urls.get(int(e[1]), str(e[1]))
+ if src and tgt:
+ edges.append((src, tgt))
+ elif isinstance(e, dict):
+ src = str(e.get("source") or e.get("from") or "")
+ tgt = str(e.get("target") or e.get("to") or "")
+ if src and tgt:
+ edges.append((src, tgt))
+ issues = _orphan_hub_suggestions(edges, [str(u) for u in orphans if u])
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(issues, limit, max_cap=50)
+ return {"suggestions": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _lighthouse_failure_bucket(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ metric: str,
+) -> dict[str, Any]:
+ payload_result = _payload_list_key(
+ conn, ctx, args, "lighthouse_failure_urls", nested=metric, item_key="pages",
+ )
+ if payload_result.get("total", 0) > 0:
+ return payload_result
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ lh_by_url = payload.get("lighthouse_by_url") if isinstance(payload.get("lighthouse_by_url"), dict) else {}
+ pages: list[dict[str, Any]] = []
+ metric_key = metric.lower()
+ for url, summary in lh_by_url.items():
+ if not isinstance(summary, dict):
+ continue
+ audits = summary.get("audits") if isinstance(summary.get("audits"), dict) else {}
+ score_val = summary.get(metric_key) or summary.get(metric.upper())
+ failed = False
+ if metric_key in ("lcp", "inp", "cls"):
+ try:
+ failed = float(score_val or 0) > 0 and metric_key in str(summary.get("cwv_failures") or "").lower()
+ except (TypeError, ValueError):
+ failed = False
+ if not failed:
+ for fail in summary.get("top_failures") or []:
+ if isinstance(fail, dict) and metric_key in str(fail.get("id") or "").lower():
+ failed = True
+ break
+ elif metric_key == "seo":
+ try:
+ failed = float(summary.get("seo") or 100) < 70
+ except (TypeError, ValueError):
+ failed = False
+ if failed or (metric_key == "seo" and isinstance(score_val, (int, float)) and float(score_val) < 70):
+ pages.append({"url": str(url), "lighthouse": {metric: score_val}})
+ elif metric_key in audits and isinstance(audits[metric_key], dict):
+ audit = audits[metric_key]
+ if audit.get("score") is not None and float(audit.get("score") or 1) < 0.9:
+ pages.append({"url": str(url), "audit": audit.get("title") or metric})
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_lighthouse_failure_lcp(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _lighthouse_failure_bucket(conn, ctx, args, "lcp")
+
+
+def list_lighthouse_failure_inp(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _lighthouse_failure_bucket(conn, ctx, args, "inp")
+
+
+def list_lighthouse_failure_cls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _lighthouse_failure_bucket(conn, ctx, args, "cls")
+
+
+def list_lighthouse_failure_seo(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ try:
+ threshold = int(args.get("seo_threshold") or 70)
+ except (TypeError, ValueError):
+ threshold = 70
+ lh_by_url = payload.get("lighthouse_by_url") if isinstance(payload.get("lighthouse_by_url"), dict) else {}
+ pages: list[dict[str, Any]] = []
+ for url, summary in lh_by_url.items():
+ if not isinstance(summary, dict):
+ continue
+ try:
+ seo = float(summary.get("seo") or 100)
+ except (TypeError, ValueError):
+ seo = 100
+ if seo < threshold:
+ pages.append({"url": str(url), "seo_score": seo})
+ pages.sort(key=lambda p: float(p.get("seo_score") or 0))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(pages, limit, max_cap=50)
+ return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
diff --git a/src/website_profiling/tools/audit_tools/keyword_lists.py b/src/website_profiling/tools/audit_tools/keyword_lists.py
new file mode 100644
index 00000000..72ca6b4f
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/keyword_lists.py
@@ -0,0 +1,558 @@
+"""Keyword list and delta audit tools."""
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any, Callable
+
+from psycopg import Connection
+
+from ...integrations.google.keyword_enrich import opportunity_clicks
+from ...integrations.google.keyword_store import read_keyword_snapshots_for_property
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+from .insight_helpers import _num
+
+
+def _require_property(ctx: AuditToolContext) -> dict[str, Any] | None:
+ if ctx.property_id is None:
+ return {"error": "property_id is required for keyword data", "missing": True}
+ return None
+
+
+def _load_keywords(scoped: AuditToolContext, conn: Connection) -> dict[str, Any] | None:
+ return scoped.load_keywords(conn)
+
+
+def _keyword_rows(data: dict[str, Any] | None) -> list[dict[str, Any]]:
+ if not data:
+ return []
+ rows = data.get("rows") or []
+ return [r for r in rows if isinstance(r, dict)]
+
+
+def _index_keywords(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+ out: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ key = str(row.get("keyword") or row.get("normalized") or "").strip().lower()
+ if key:
+ out[key] = row
+ return out
+
+
+def _position(row: dict[str, Any]) -> float | None:
+ raw = row.get("gsc_position")
+ if raw is None:
+ return None
+ try:
+ pos = float(raw)
+ except (TypeError, ValueError):
+ return None
+ return pos if pos > 0 else None
+
+
+def _load_keyword_pair(
+ scoped: AuditToolContext,
+ conn: Connection,
+) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+ current = _load_keywords(scoped, conn)
+ snapshots = read_keyword_snapshots_for_property(conn, scoped.property_id, limit=2)
+ prior: dict[str, Any] | None = None
+ if len(snapshots) >= 2:
+ prior = snapshots[1]
+ elif len(snapshots) == 1 and current is None:
+ current = snapshots[0]
+ return current, prior
+
+
+def _rank_delta_rows(
+ current: dict[str, Any],
+ prior: dict[str, Any],
+ *,
+ improved: bool,
+) -> list[dict[str, Any]]:
+ curr = _index_keywords(_keyword_rows(current))
+ prev = _index_keywords(_keyword_rows(prior))
+ deltas: list[dict[str, Any]] = []
+ for key, row in curr.items():
+ old = prev.get(key)
+ if not old:
+ continue
+ cur_pos = _position(row)
+ old_pos = _position(old)
+ if cur_pos is None or old_pos is None:
+ continue
+ pos_delta = cur_pos - old_pos
+ if improved:
+ if pos_delta >= 0:
+ continue
+ elif pos_delta <= 0:
+ continue
+ entry = {
+ "keyword": row.get("keyword") or key,
+ "gsc_position": cur_pos,
+ "prior_position": old_pos,
+ "position_delta": round(pos_delta, 2),
+ "gsc_clicks": row.get("gsc_clicks"),
+ "gsc_impressions": row.get("gsc_impressions"),
+ "gsc_url": row.get("gsc_url"),
+ }
+ deltas.append(entry)
+ deltas.sort(key=lambda r: r.get("position_delta", 0), reverse=not improved)
+ return deltas
+
+
+def _top_ten_transitions(
+ current: dict[str, Any],
+ prior: dict[str, Any],
+ *,
+ entered: bool,
+) -> list[dict[str, Any]]:
+ curr = _index_keywords(_keyword_rows(current))
+ prev = _index_keywords(_keyword_rows(prior))
+ rows: list[dict[str, Any]] = []
+ if entered:
+ for key, row in curr.items():
+ cur_pos = _position(row)
+ if cur_pos is None or cur_pos > 10:
+ continue
+ old_pos = _position(prev.get(key, {}))
+ if old_pos is not None and old_pos <= 10:
+ continue
+ rows.append({
+ "keyword": row.get("keyword") or key,
+ "gsc_position": cur_pos,
+ "prior_position": old_pos,
+ "gsc_clicks": row.get("gsc_clicks"),
+ "gsc_impressions": row.get("gsc_impressions"),
+ })
+ else:
+ for key, old in prev.items():
+ old_pos = _position(old)
+ if old_pos is None or old_pos > 10:
+ continue
+ row = curr.get(key, {})
+ cur_pos = _position(row)
+ if cur_pos is not None and cur_pos <= 10:
+ continue
+ rows.append({
+ "keyword": old.get("keyword") or key,
+ "prior_position": old_pos,
+ "gsc_position": cur_pos,
+ "gsc_clicks": row.get("gsc_clicks") if row else old.get("gsc_clicks"),
+ "gsc_impressions": row.get("gsc_impressions") if row else old.get("gsc_impressions"),
+ })
+ rows.sort(key=lambda r: -_num(r.get("gsc_impressions")))
+ return rows
+
+
+def _keyword_bucket(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ *,
+ key: str,
+ item_key: str,
+ empty_error: str | None = None,
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ err = _require_property(scoped)
+ if err:
+ return {**err, item_key: [], "total": 0, "truncated": False}
+ data = _load_keywords(scoped, conn)
+ if not data:
+ return {"error": empty_error or "no keyword data found", "missing": True, item_key: [], "total": 0, "truncated": False}
+ items = data.get(key) or []
+ if key == "semantic_keyword_clusters":
+ payload = scoped.load_payload(conn)
+ items = payload.get("semantic_keyword_clusters") or items
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(items if isinstance(items, list) else [], limit, max_cap=50)
+ return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _filter_keywords(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ predicate: Callable[[dict[str, Any]], bool],
+ *,
+ sort_key: Callable[[dict[str, Any]], Any] | None = None,
+ reverse: bool = True,
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ err = _require_property(scoped)
+ if err:
+ return {**err, "keywords": [], "total": 0, "truncated": False}
+ data = _load_keywords(scoped, conn)
+ if not data:
+ return {"error": "no keyword data found", "missing": True, "keywords": [], "total": 0, "truncated": False}
+ matches = [r for r in _keyword_rows(data) if predicate(r)]
+ if sort_key:
+ matches.sort(key=sort_key, reverse=reverse)
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(matches, limit, max_cap=50)
+ return {"keywords": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _pair_delta_tool(
+ conn: Connection,
+ ctx: AuditToolContext,
+ args: dict[str, Any],
+ *,
+ builder: Callable[[dict[str, Any], dict[str, Any]], list[dict[str, Any]]],
+ item_key: str,
+) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ err = _require_property(scoped)
+ if err:
+ return {**err, item_key: [], "total": 0, "truncated": False}
+ current, prior = _load_keyword_pair(scoped, conn)
+ if not current:
+ return {"error": "no keyword data found", "missing": True, item_key: [], "total": 0, "truncated": False}
+ if not prior:
+ return {"error": "no prior keyword snapshot for comparison", "missing": True, item_key: [], "total": 0, "truncated": False}
+ rows = builder(current, prior)
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(rows, limit, max_cap=50)
+ return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def _serp_features(row: dict[str, Any]) -> list[str]:
+ raw = row.get("serp_features")
+ if isinstance(raw, list):
+ return [str(f).lower() for f in raw if f]
+ if isinstance(raw, str) and raw.strip():
+ return [raw.strip().lower()]
+ return []
+
+
+def _has_serp_feature(row: dict[str, Any], *needles: str) -> bool:
+ features = _serp_features(row)
+ return any(any(n in f for n in needles) for f in features)
+
+
+def list_keyword_rank_improvements(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _pair_delta_tool(
+ conn, ctx, args,
+ builder=lambda cur, prev: _rank_delta_rows(cur, prev, improved=True),
+ item_key="keywords",
+ )
+
+
+def list_keyword_rank_declines(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _pair_delta_tool(
+ conn, ctx, args,
+ builder=lambda cur, prev: _rank_delta_rows(cur, prev, improved=False),
+ item_key="keywords",
+ )
+
+
+def list_keywords_new_to_top_10(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _pair_delta_tool(
+ conn, ctx, args,
+ builder=lambda cur, prev: _top_ten_transitions(cur, prev, entered=True),
+ item_key="keywords",
+ )
+
+
+def list_keywords_fell_out_of_top_10(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _pair_delta_tool(
+ conn, ctx, args,
+ builder=lambda cur, prev: _top_ten_transitions(cur, prev, entered=False),
+ item_key="keywords",
+ )
+
+
+def list_cannibalisation_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _keyword_bucket(conn, ctx, args, key="cannibalisation", item_key="queries")
+
+
+def list_cannibalisation_urls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ err = _require_property(scoped)
+ if err:
+ return {**err, "urls": [], "total": 0, "truncated": False}
+ data = _load_keywords(scoped, conn)
+ if not data:
+ return {"error": "no keyword data found", "missing": True, "urls": [], "total": 0, "truncated": False}
+ by_url: dict[str, dict[str, Any]] = {}
+ for issue in data.get("cannibalisation") or []:
+ if not isinstance(issue, dict):
+ continue
+ query = str(issue.get("query") or "")
+ for page in issue.get("pages") or []:
+ if not isinstance(page, dict):
+ continue
+ url = str(page.get("url") or "").strip()
+ if not url:
+ continue
+ bucket = by_url.setdefault(url, {
+ "url": url,
+ "queries": [],
+ "query_count": 0,
+ "total_clicks": 0,
+ "total_impressions": 0,
+ })
+ bucket["queries"].append({
+ "query": query,
+ "position": page.get("position"),
+ "clicks": page.get("clicks"),
+ "impressions": page.get("impressions"),
+ })
+ bucket["query_count"] += 1
+ bucket["total_clicks"] += int(_num(page.get("clicks")))
+ bucket["total_impressions"] += int(_num(page.get("impressions")))
+ urls = sorted(by_url.values(), key=lambda r: (-r["query_count"], -r["total_impressions"]))
+ limit = parse_limit(args.get("limit"), 30, 50)
+ sliced = cap_list(urls, limit, max_cap=50)
+ return {"urls": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_misaligned_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _keyword_bucket(
+ conn, ctx, args,
+ key="query_page_misalignment",
+ item_key="misalignments",
+ )
+
+
+def list_keywords_by_recommended_action(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ action = str(args.get("recommended_action") or args.get("action") or "").strip().lower()
+ if not action:
+ return {"error": "recommended_action is required", "keywords": [], "total": 0, "truncated": False}
+ return _filter_keywords(
+ conn, ctx, args,
+ lambda r: action in str(r.get("recommended_action") or "").lower(),
+ sort_key=lambda r: _num(r.get("gsc_impressions")),
+ )
+
+
+def list_keywords_by_serp_feature(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ feature = str(args.get("serp_feature") or args.get("feature") or "").strip().lower()
+ if not feature:
+ return {"error": "serp_feature is required", "keywords": [], "total": 0, "truncated": False}
+ return _filter_keywords(
+ conn, ctx, args,
+ lambda r: _has_serp_feature(r, feature),
+ sort_key=lambda r: _num(r.get("gsc_impressions")),
+ )
+
+
+def _semantic_clusters(scoped: AuditToolContext, conn: Connection) -> list[dict[str, Any]]:
+ payload = scoped.load_payload(conn)
+ clusters = payload.get("semantic_keyword_clusters") if isinstance(payload, dict) else []
+ if isinstance(clusters, list) and clusters:
+ return [c for c in clusters if isinstance(c, dict)]
+ data = _load_keywords(scoped, conn)
+ if not data:
+ return []
+ fallback = data.get("semantic_keyword_clusters") or []
+ return [c for c in fallback if isinstance(c, dict)] if isinstance(fallback, list) else []
+
+
+def list_semantic_cluster_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ clusters = _semantic_clusters(scoped, conn)
+ if not clusters:
+ return {"missing": True, "clusters": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 20, 50)
+ sliced = cap_list(clusters, limit, max_cap=50)
+ return {"clusters": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_semantic_cluster_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ err = _require_property(scoped)
+ if err:
+ return {**err, "clusters": [], "total": 0, "truncated": False}
+ clusters = _semantic_clusters(scoped, conn)
+ if not clusters:
+ return {"missing": True, "clusters": [], "total": 0, "truncated": False}
+ kw_to_url: dict[str, str] = {}
+ for row in _keyword_rows(_load_keywords(scoped, conn)):
+ kw = str(row.get("keyword") or "").strip().lower()
+ url = str(row.get("gsc_url") or "").strip()
+ if kw and url:
+ kw_to_url[kw] = url
+ enriched: list[dict[str, Any]] = []
+ for cluster in clusters:
+ keywords = [str(k).strip().lower() for k in (cluster.get("keywords") or []) if k]
+ pages: dict[str, list[str]] = defaultdict(list)
+ for kw in keywords:
+ url = kw_to_url.get(kw)
+ if url:
+ pages[url].append(kw)
+ enriched.append({
+ "top_keyword": cluster.get("top_keyword") or cluster.get("representative"),
+ "cluster_score": cluster.get("cluster_score"),
+ "keywords": cluster.get("keywords") or [],
+ "pages": [
+ {"url": url, "keywords": kws, "keyword_count": len(kws)}
+ for url, kws in sorted(pages.items(), key=lambda x: -len(x[1]))
+ ],
+ "page_count": len(pages),
+ })
+ limit = parse_limit(args.get("limit"), 20, 50)
+ sliced = cap_list(enriched, limit, max_cap=50)
+ return {"clusters": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def get_keyword_opportunity_score(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ err = _require_property(scoped)
+ if err:
+ return err
+ keyword = str(args.get("keyword") or "").strip()
+ if not keyword:
+ return {"error": "keyword is required"}
+ data = _load_keywords(scoped, conn)
+ if not data:
+ return {"error": "no keyword data found", "missing": True}
+ needle = keyword.lower()
+ row = next(
+ (r for r in _keyword_rows(data) if str(r.get("keyword") or "").lower() == needle),
+ None,
+ )
+ if not row:
+ return {"error": "keyword not found", "keyword": keyword, "missing": True}
+ pos = _position(row) or 0.0
+ impressions = int(_num(row.get("gsc_impressions")))
+ opp_clicks = row.get("opportunity_clicks")
+ if opp_clicks is None and pos > 0:
+ opp_clicks = opportunity_clicks(impressions, pos, target_pos=3)
+ score = float(row.get("score") or 0)
+ traffic_potential = int(_num(row.get("traffic_potential")))
+ composite = round(
+ min(100.0, (float(opp_clicks or 0) * 2) + (traffic_potential / 50.0) + score),
+ 2,
+ )
+ return {
+ "keyword": row.get("keyword") or keyword,
+ "opportunity_score": composite,
+ "opportunity_clicks": opp_clicks,
+ "traffic_potential": traffic_potential,
+ "gsc_position": pos or None,
+ "gsc_impressions": impressions,
+ "gsc_clicks": row.get("gsc_clicks"),
+ "recommended_action": row.get("recommended_action"),
+ "fetched_at": data.get("fetched_at"),
+ }
+
+
+def list_keywords_near_page_one(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ try:
+ min_pos = float(args.get("min_position") or 4)
+ max_pos = float(args.get("max_position") or 20)
+ min_impressions = int(args.get("min_impressions") or 50)
+ except (TypeError, ValueError):
+ return {"error": "min_position, max_position, and min_impressions must be numbers"}
+
+ def _near(row: dict[str, Any]) -> bool:
+ pos = _position(row)
+ if pos is None:
+ return False
+ return min_pos <= pos <= max_pos and _num(row.get("gsc_impressions")) >= min_impressions
+
+ return _filter_keywords(
+ conn, ctx, args, _near,
+ sort_key=lambda r: (_num(r.get("gsc_impressions")), -(_position(r) or 99)),
+ )
+
+
+def list_keywords_high_impression_zero_click(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ min_impressions = parse_limit(args.get("min_impressions"), 100, 1_000_000)
+
+ def _zero_click(row: dict[str, Any]) -> bool:
+ return int(_num(row.get("gsc_clicks"))) == 0 and int(_num(row.get("gsc_impressions"))) >= min_impressions
+
+ return _filter_keywords(
+ conn, ctx, args, _zero_click,
+ sort_key=lambda r: _num(r.get("gsc_impressions")),
+ )
+
+
+def list_keywords_by_competition_band(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ try:
+ min_comp = float(args.get("min_competition") if args.get("min_competition") is not None else 0)
+ max_comp = float(args.get("max_competition") if args.get("max_competition") is not None else 100)
+ except (TypeError, ValueError):
+ return {"error": "min_competition and max_competition must be numbers"}
+
+ def _in_band(row: dict[str, Any]) -> bool:
+ raw = row.get("serp_estimated_competition")
+ if raw is None:
+ return False
+ try:
+ val = float(raw)
+ except (TypeError, ValueError):
+ return False
+ return min_comp <= val <= max_comp
+
+ return _filter_keywords(
+ conn, ctx, args, _in_band,
+ sort_key=lambda r: float(r.get("serp_estimated_competition") or 0),
+ reverse=False,
+ )
+
+
+def get_keyword_serp_snapshot(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ err = _require_property(scoped)
+ if err:
+ return err
+ keyword = str(args.get("keyword") or "").strip()
+ if not keyword:
+ return {"error": "keyword is required"}
+ data = _load_keywords(scoped, conn)
+ if not data:
+ return {"error": "no keyword data found", "missing": True}
+ needle = keyword.lower()
+ row = next(
+ (r for r in _keyword_rows(data) if str(r.get("keyword") or "").lower() == needle),
+ None,
+ )
+ if not row:
+ return {"error": "keyword not found", "keyword": keyword, "missing": True}
+ return {
+ "keyword": row.get("keyword") or keyword,
+ "serp_features": row.get("serp_features"),
+ "serp_estimated_competition": row.get("serp_estimated_competition"),
+ "serp_organic_count": row.get("serp_organic_count"),
+ "serp_provenance": row.get("serp_provenance") or "Estimated",
+ "gsc_position": row.get("gsc_position"),
+ "gsc_impressions": row.get("gsc_impressions"),
+ "fetched_at": data.get("fetched_at"),
+ }
+
+
+def list_keywords_with_ai_overview(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _filter_keywords(
+ conn, ctx, args,
+ lambda r: _has_serp_feature(r, "ai_overview", "answer_box", "featured_snippet", "knowledge_graph"),
+ sort_key=lambda r: _num(r.get("gsc_impressions")),
+ )
+
+
+def list_keywords_local_pack(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _filter_keywords(
+ conn, ctx, args,
+ lambda r: _has_serp_feature(r, "local_pack", "local", "map"),
+ sort_key=lambda r: _num(r.get("gsc_impressions")),
+ )
+
+
+def list_keywords_question_intent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ return _filter_keywords(
+ conn, ctx, args,
+ lambda r: bool(r.get("is_question")),
+ sort_key=lambda r: _num(r.get("gsc_impressions")),
+ )
+
+
+def list_keywords_commercial_intent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ commercial = {"commercial", "transactional"}
+ return _filter_keywords(
+ conn, ctx, args,
+ lambda r: str(r.get("intent") or "").lower() in commercial,
+ sort_key=lambda r: _num(r.get("gsc_impressions")),
+ )
diff --git a/src/website_profiling/tools/audit_tools/keywords.py b/src/website_profiling/tools/audit_tools/keywords.py
index 79b9a8ae..e6cd45c8 100644
--- a/src/website_profiling/tools/audit_tools/keywords.py
+++ b/src/website_profiling/tools/audit_tools/keywords.py
@@ -250,3 +250,33 @@ def _impressions(row: dict[str, Any]) -> int:
conn, ctx, args,
lambda r: _impressions(r) >= min_v,
)
+
+
+def get_brand_keyword_split(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ data = scoped.load_keywords(conn)
+ if not data:
+ return {"error": "no keyword data found", "missing": True}
+ rows = [r for r in (data.get("rows") or []) if isinstance(r, dict)]
+ branded = [r for r in rows if r.get("is_branded")]
+ non_branded = [r for r in rows if not r.get("is_branded")]
+ return {
+ "brand_name": data.get("brand_name"),
+ "branded_count": len(branded),
+ "non_branded_count": len(non_branded),
+ "branded_sample": branded[:10],
+ "non_branded_sample": non_branded[:10],
+ "provenance": "Keywords enrichment",
+ }
+
+
+def list_keywords_by_intent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ intent = str(args.get("intent") or "").strip().lower()
+ if not intent:
+ return {"error": "intent is required"}
+ return _filter_keyword_rows(
+ conn, ctx, args,
+ lambda r: str(r.get("intent") or "").lower() == intent,
+ )
diff --git a/src/website_profiling/tools/audit_tools/link_lists.py b/src/website_profiling/tools/audit_tools/link_lists.py
new file mode 100644
index 00000000..0d207896
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/link_lists.py
@@ -0,0 +1,208 @@
+"""Link graph list tools from payload link_edges, graph_edges, and PageRank."""
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urlparse
+
+from psycopg import Connection
+
+from ._slice import cap_list, parse_limit
+from .context import AuditToolContext
+
+
+def _norm_url(url: str) -> str:
+ return str(url or "").strip().rstrip("/").lower()
+
+
+def _load_link_edges(payload: dict[str, Any]) -> list[dict[str, Any]]:
+ edges = payload.get("link_edges") or []
+ if isinstance(edges, list) and edges:
+ return [e for e in edges if isinstance(e, dict)]
+ graph = payload.get("graph_edges") or []
+ if not isinstance(graph, list):
+ return []
+ converted: list[dict[str, Any]] = []
+ for edge in graph:
+ if isinstance(edge, dict):
+ converted.append({
+ "from_url": edge.get("from") or edge.get("source") or edge.get("from_url"),
+ "to_url": edge.get("to") or edge.get("target") or edge.get("to_url"),
+ "link_type": edge.get("link_type") or "internal",
+ "is_nofollow": bool(edge.get("is_nofollow")),
+ "rel": edge.get("rel"),
+ "anchor_text": edge.get("anchor_text") or edge.get("label"),
+ })
+ elif isinstance(edge, (list, tuple)) and len(edge) >= 2:
+ converted.append({
+ "from_url": edge[0],
+ "to_url": edge[1],
+ "link_type": "internal",
+ "is_nofollow": False,
+ })
+ return converted
+
+
+def _pagerank_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
+ candidates = payload.get("top_pages") or payload.get("links") or []
+ if not isinstance(candidates, list):
+ return []
+ ranked: list[dict[str, Any]] = []
+ for rec in candidates:
+ if not isinstance(rec, dict):
+ continue
+ pr = rec.get("pagerank")
+ if pr is None:
+ continue
+ try:
+ score = float(pr)
+ except (TypeError, ValueError):
+ continue
+ ranked.append({
+ "url": rec.get("url"),
+ "pagerank": round(score, 5),
+ "inlinks": rec.get("inlinks"),
+ "outlinks": rec.get("outlinks"),
+ })
+ ranked.sort(key=lambda x: float(x.get("pagerank") or 0))
+ return ranked
+
+
+def list_outbound_links(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "links": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 100)
+ edges = _load_link_edges(payload)
+ items = [
+ {
+ "from_url": e.get("from_url"),
+ "to_url": e.get("to_url"),
+ "anchor_text": e.get("anchor_text"),
+ "rel": e.get("rel"),
+ "is_nofollow": bool(e.get("is_nofollow")),
+ }
+ for e in edges
+ if str(e.get("link_type") or "") == "external"
+ ]
+ if not items:
+ start = str(payload.get("start_url") or payload.get("origin") or "").strip()
+ origin_host = urlparse(start).netloc.lower().lstrip("www.") if start else ""
+ for e in edges:
+ to_url = str(e.get("to_url") or "")
+ host = urlparse(to_url).netloc.lower().lstrip("www.")
+ if origin_host and host and host != origin_host:
+ items.append({
+ "from_url": e.get("from_url"),
+ "to_url": to_url,
+ "anchor_text": e.get("anchor_text"),
+ "rel": e.get("rel"),
+ "is_nofollow": bool(e.get("is_nofollow")),
+ })
+ sliced = cap_list(items, limit, max_cap=100)
+ return {"links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_internal_links_from_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ source = str(args.get("url") or args.get("from_url") or "").strip()
+ if not source:
+ return {"error": "url is required", "links": [], "total": 0, "truncated": False}
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "links": [], "total": 0, "truncated": False}
+ needle = _norm_url(source)
+ limit = parse_limit(args.get("limit"), 30, 100)
+ edges = _load_link_edges(payload)
+ items = [
+ {
+ "from_url": e.get("from_url"),
+ "to_url": e.get("to_url"),
+ "anchor_text": e.get("anchor_text"),
+ "rel": e.get("rel"),
+ "is_nofollow": bool(e.get("is_nofollow")),
+ }
+ for e in edges
+ if _norm_url(str(e.get("from_url") or "")) == needle
+ and str(e.get("link_type") or "internal") == "internal"
+ ]
+ sliced = cap_list(items, limit, max_cap=100)
+ return {"url": source, "links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_internal_links_to_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ target = str(args.get("url") or args.get("to_url") or args.get("target_url") or "").strip()
+ if not target:
+ return {"error": "url is required", "links": [], "total": 0, "truncated": False}
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "links": [], "total": 0, "truncated": False}
+ needle = _norm_url(target)
+ limit = parse_limit(args.get("limit"), 30, 100)
+ edges = _load_link_edges(payload)
+ items = [
+ {
+ "from_url": e.get("from_url"),
+ "to_url": e.get("to_url"),
+ "anchor_text": e.get("anchor_text"),
+ "rel": e.get("rel"),
+ "is_nofollow": bool(e.get("is_nofollow")),
+ }
+ for e in edges
+ if _norm_url(str(e.get("to_url") or "")) == needle
+ and str(e.get("link_type") or "internal") == "internal"
+ ]
+ sliced = cap_list(items, limit, max_cap=100)
+ return {"url": target, "links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]}
+
+
+def list_links_by_rel_nofollow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "links": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 100)
+ rel_filter = str(args.get("rel") or "nofollow").strip().lower()
+ edges = _load_link_edges(payload)
+ items = []
+ for e in edges:
+ rel = str(e.get("rel") or "").lower()
+ is_nf = bool(e.get("is_nofollow"))
+ if rel_filter == "nofollow" and not is_nf and "nofollow" not in rel:
+ continue
+ if rel_filter not in ("nofollow", "") and rel_filter not in rel:
+ continue
+ items.append({
+ "from_url": e.get("from_url"),
+ "to_url": e.get("to_url"),
+ "link_type": e.get("link_type"),
+ "rel": e.get("rel"),
+ "is_nofollow": is_nf,
+ "anchor_text": e.get("anchor_text"),
+ })
+ sliced = cap_list(items, limit, max_cap=100)
+ return {"links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "rel_filter": rel_filter}
+
+
+def list_pagerank_low_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ payload = scoped.load_payload(conn)
+ if not payload:
+ return {"error": "no report found", "pages": [], "total": 0, "truncated": False}
+ limit = parse_limit(args.get("limit"), 30, 50)
+ try:
+ max_pr = float(args.get("max_pagerank", 0.01))
+ except (TypeError, ValueError):
+ max_pr = 0.01
+ ranked = _pagerank_rows(payload)
+ if not ranked:
+ return {"missing": True, "pages": [], "total": 0, "truncated": False, "note": "pagerank not in report"}
+ low = [r for r in ranked if float(r.get("pagerank") or 0) <= max_pr]
+ sliced = cap_list(low, limit, max_cap=50)
+ return {
+ "pages": sliced["items"],
+ "total": sliced["total"],
+ "truncated": sliced["truncated"],
+ "max_pagerank": max_pr,
+ }
diff --git a/src/website_profiling/tools/audit_tools/registry.py b/src/website_profiling/tools/audit_tools/registry.py
index faa8da99..c1b472fa 100644
--- a/src/website_profiling/tools/audit_tools/registry.py
+++ b/src/website_profiling/tools/audit_tools/registry.py
@@ -6,6 +6,123 @@
from psycopg import Connection
from ...db.storage import db_session
+
+from .backlink_lists import (
+ get_anchor_text_distribution,
+ list_backlinks_by_anchor_text,
+ list_backlinks_from_domain,
+ list_backlinks_to_url,
+ list_referring_domains,
+)
+from .compare_list_tools import (
+ list_compare_lighthouse_regressions,
+ list_compare_new_issues,
+ list_compare_new_urls,
+ list_compare_removed_urls,
+ list_compare_resolved_issues,
+ list_compare_traffic_losers,
+)
+from .content_lists import (
+ get_text_content_analysis,
+ list_amp_validation_issues,
+ list_duplicate_content_pairs,
+ list_html_validation_issues,
+ list_pages_by_word_count_band,
+ list_pages_containing_keyword,
+ list_pages_missing_article_schema,
+ list_pagination_issues,
+ list_schema_errors_by_type,
+ list_spell_check_issues,
+)
+from .geo_list_tools import (
+ list_pages_ai_citation_signals,
+ list_pages_missing_howto_schema,
+ list_pages_missing_llms_txt_reference,
+ list_robots_blocked_ai_crawlers,
+)
+from .google_lists import (
+ compare_gsc_periods,
+ get_ga4_path_trend,
+ get_gsc_page_trend,
+ get_gsc_query_trend,
+ get_gsc_site_benchmarks,
+ list_ga4_landing_pages,
+ list_ga4_pages_by_bounce_rate,
+ list_ga4_pages_by_engagement_rate,
+ list_gsc_branded_queries,
+ list_gsc_ctr_underperformers,
+ list_gsc_decaying_pages,
+ list_gsc_decaying_queries,
+ list_gsc_ga4_mismatch_pages,
+ list_gsc_new_queries,
+ list_gsc_non_branded_queries,
+ list_gsc_pages_by_clicks,
+ list_gsc_pages_by_impressions,
+ list_gsc_pages_by_position_band,
+ list_gsc_queries_by_clicks,
+ list_gsc_queries_by_impressions,
+)
+from .indexation_lists import (
+ list_crawl_urls_not_in_sitemap,
+ list_hreflang_reciprocal_gaps,
+ list_indexation_indexed_not_submitted,
+ list_indexation_submitted_not_indexed,
+ list_log_5xx_paths,
+ list_log_googlebot_low_crawl,
+ list_log_orphan_high_traffic,
+ list_log_paths_by_hits,
+ list_redirect_chains_by_length,
+ list_sitemap_urls_not_in_crawl,
+)
+from .issue_lists import (
+ list_hreflang_issue_pages,
+ list_lighthouse_failure_cls,
+ list_lighthouse_failure_inp,
+ list_lighthouse_failure_lcp,
+ list_lighthouse_failure_seo,
+ list_orphan_hub_suggestions,
+ list_pages_color_contrast_failures,
+ list_pages_high_reading_level,
+ list_pages_invalid_json_ld,
+ list_pages_invalid_viewport,
+ list_pages_missing_html_lang,
+ list_pages_missing_og_tags,
+ list_pages_missing_twitter_cards,
+ list_pages_mixed_language,
+ list_pages_slow_response,
+ list_pages_title_too_long,
+ list_pages_title_too_short,
+ list_pages_very_thin_content,
+)
+from .keyword_lists import (
+ get_keyword_opportunity_score,
+ get_keyword_serp_snapshot,
+ list_cannibalisation_queries,
+ list_cannibalisation_urls,
+ list_keyword_rank_declines,
+ list_keyword_rank_improvements,
+ list_keywords_by_competition_band,
+ list_keywords_by_recommended_action,
+ list_keywords_by_serp_feature,
+ list_keywords_commercial_intent,
+ list_keywords_fell_out_of_top_10,
+ list_keywords_high_impression_zero_click,
+ list_keywords_local_pack,
+ list_keywords_near_page_one,
+ list_keywords_new_to_top_10,
+ list_keywords_question_intent,
+ list_keywords_with_ai_overview,
+ list_misaligned_queries,
+ list_semantic_cluster_pages,
+ list_semantic_cluster_queries,
+)
+from .link_lists import (
+ list_internal_links_from_url,
+ list_internal_links_to_url,
+ list_links_by_rel_nofollow,
+ list_outbound_links,
+ list_pagerank_low_pages,
+)
from .backlinks import (
get_backlinks_velocity,
get_bing_backlinks_summary,
@@ -25,6 +142,22 @@
get_title_length_distribution,
get_top_crawled_pages,
)
+from .data_coverage import get_data_coverage_report
+from .insight_tools import (
+ get_issue_to_traffic_map,
+ get_landing_page_blended_table,
+ get_landing_page_full_diagnosis,
+ get_opportunity_matrix,
+ get_traffic_health_check,
+)
+from .router_tools import (
+ list_tool_domains,
+ run_domain_agent,
+ run_insight_workflow,
+ run_keyword_workflow,
+ run_technical_workflow,
+ search_audit_tools,
+)
from .compare import compare_reports
from .compare_slices import (
compare_category_deltas,
@@ -101,6 +234,8 @@
list_broken_links,
list_pages_by_fetch_method,
list_pages_with_console_errors,
+ list_pages_console_errors_by_type,
+ list_pages_js_rendering_delta,
list_redirects,
list_status_4xx_pages,
list_status_5xx_pages,
@@ -108,10 +243,15 @@
search_pages_advanced,
)
from .google import (
+ get_ga4_by_channel,
+ get_ga4_by_device,
+ get_ga4_daily_trend,
get_ga4_page_metrics,
get_ga4_summary,
get_google_summary,
get_gsc_ctr_opportunity_pages,
+ get_gsc_daily_trend,
+ get_gsc_page_queries,
get_gsc_page_query_slice,
get_gsc_top_pages,
get_gsc_top_queries,
@@ -128,6 +268,7 @@
from .international import get_hreflang_summary, get_language_summary
from .issues import get_category_issues, list_issues_by_category
from .keywords import (
+ get_brand_keyword_split,
get_keyword_cannibalisation,
get_keyword_history,
get_keyword_serp_overlay,
@@ -137,6 +278,7 @@
get_striking_distance_keywords,
list_keywords_by_action,
list_keywords_by_impressions,
+ list_keywords_by_intent,
list_keywords_by_position,
list_keywords_ctr_opportunity,
search_keywords,
@@ -260,6 +402,16 @@
)
from .tech import get_tech_stack_summary, list_pages_by_technology
from .tool_catalog import TOOL_DEFINITIONS
+from .tool_domains import (
+ TIER_0_TOOLS,
+ build_tool_meta,
+ classify_tool_domain,
+ domains_catalog,
+ tool_names_for_domain as _meta_tool_names_for_domain,
+ tool_names_for_mcp_bundle,
+ tool_names_for_tier as _meta_tool_names_for_tier,
+ tools_by_domain,
+)
from .workflow import list_issue_workflow
ToolHandler = Callable[[Connection, AuditToolContext, dict[str, Any]], dict[str, Any]]
@@ -486,6 +638,125 @@
"get_bing_index_status": get_bing_index_status,
"get_serp_feature_overlay": get_serp_feature_overlay,
"check_ai_citation_presence": check_ai_citation_presence,
+ "search_audit_tools": search_audit_tools,
+ "list_tool_domains": list_tool_domains,
+ "get_data_coverage_report": get_data_coverage_report,
+ "run_insight_workflow": run_insight_workflow,
+ "run_technical_workflow": run_technical_workflow,
+ "run_keyword_workflow": run_keyword_workflow,
+ "run_domain_agent": run_domain_agent,
+ "get_landing_page_blended_table": get_landing_page_blended_table,
+ "get_opportunity_matrix": get_opportunity_matrix,
+ "get_traffic_health_check": get_traffic_health_check,
+ "get_landing_page_full_diagnosis": get_landing_page_full_diagnosis,
+ "get_issue_to_traffic_map": get_issue_to_traffic_map,
+ "get_gsc_daily_trend": get_gsc_daily_trend,
+ "get_ga4_daily_trend": get_ga4_daily_trend,
+ "get_ga4_by_device": get_ga4_by_device,
+ "get_ga4_by_channel": get_ga4_by_channel,
+ "get_gsc_page_queries": get_gsc_page_queries,
+ "get_brand_keyword_split": get_brand_keyword_split,
+ "list_keywords_by_intent": list_keywords_by_intent,
+ "list_pages_title_too_short": list_pages_title_too_short,
+ "list_pages_title_too_long": list_pages_title_too_long,
+ "list_pages_slow_response": list_pages_slow_response,
+ "list_pages_missing_html_lang": list_pages_missing_html_lang,
+ "list_pages_invalid_viewport": list_pages_invalid_viewport,
+ "list_pages_color_contrast_failures": list_pages_color_contrast_failures,
+ "list_pages_high_reading_level": list_pages_high_reading_level,
+ "list_pages_very_thin_content": list_pages_very_thin_content,
+ "list_hreflang_issue_pages": list_hreflang_issue_pages,
+ "list_pages_missing_og_tags": list_pages_missing_og_tags,
+ "list_pages_missing_twitter_cards": list_pages_missing_twitter_cards,
+ "list_pages_invalid_json_ld": list_pages_invalid_json_ld,
+ "list_pages_mixed_language": list_pages_mixed_language,
+ "list_orphan_hub_suggestions": list_orphan_hub_suggestions,
+ "list_lighthouse_failure_lcp": list_lighthouse_failure_lcp,
+ "list_lighthouse_failure_inp": list_lighthouse_failure_inp,
+ "list_lighthouse_failure_cls": list_lighthouse_failure_cls,
+ "list_lighthouse_failure_seo": list_lighthouse_failure_seo,
+ "list_gsc_pages_by_impressions": list_gsc_pages_by_impressions,
+ "list_gsc_pages_by_clicks": list_gsc_pages_by_clicks,
+ "list_gsc_queries_by_impressions": list_gsc_queries_by_impressions,
+ "list_gsc_queries_by_clicks": list_gsc_queries_by_clicks,
+ "list_gsc_ctr_underperformers": list_gsc_ctr_underperformers,
+ "list_gsc_decaying_pages": list_gsc_decaying_pages,
+ "list_gsc_decaying_queries": list_gsc_decaying_queries,
+ "list_gsc_new_queries": list_gsc_new_queries,
+ "list_ga4_landing_pages": list_ga4_landing_pages,
+ "list_ga4_pages_by_bounce_rate": list_ga4_pages_by_bounce_rate,
+ "list_ga4_pages_by_engagement_rate": list_ga4_pages_by_engagement_rate,
+ "get_gsc_query_trend": get_gsc_query_trend,
+ "get_gsc_page_trend": get_gsc_page_trend,
+ "get_ga4_path_trend": get_ga4_path_trend,
+ "list_gsc_ga4_mismatch_pages": list_gsc_ga4_mismatch_pages,
+ "list_gsc_pages_by_position_band": list_gsc_pages_by_position_band,
+ "get_gsc_site_benchmarks": get_gsc_site_benchmarks,
+ "list_gsc_branded_queries": list_gsc_branded_queries,
+ "list_gsc_non_branded_queries": list_gsc_non_branded_queries,
+ "compare_gsc_periods": compare_gsc_periods,
+ "list_keyword_rank_improvements": list_keyword_rank_improvements,
+ "list_keyword_rank_declines": list_keyword_rank_declines,
+ "list_keywords_new_to_top_10": list_keywords_new_to_top_10,
+ "list_keywords_fell_out_of_top_10": list_keywords_fell_out_of_top_10,
+ "list_cannibalisation_queries": list_cannibalisation_queries,
+ "list_cannibalisation_urls": list_cannibalisation_urls,
+ "list_misaligned_queries": list_misaligned_queries,
+ "list_keywords_by_recommended_action": list_keywords_by_recommended_action,
+ "list_keywords_by_serp_feature": list_keywords_by_serp_feature,
+ "list_semantic_cluster_queries": list_semantic_cluster_queries,
+ "list_semantic_cluster_pages": list_semantic_cluster_pages,
+ "get_keyword_opportunity_score": get_keyword_opportunity_score,
+ "list_keywords_near_page_one": list_keywords_near_page_one,
+ "list_keywords_high_impression_zero_click": list_keywords_high_impression_zero_click,
+ "list_keywords_by_competition_band": list_keywords_by_competition_band,
+ "get_keyword_serp_snapshot": get_keyword_serp_snapshot,
+ "list_keywords_with_ai_overview": list_keywords_with_ai_overview,
+ "list_keywords_local_pack": list_keywords_local_pack,
+ "list_keywords_question_intent": list_keywords_question_intent,
+ "list_keywords_commercial_intent": list_keywords_commercial_intent,
+ "list_referring_domains": list_referring_domains,
+ "list_backlinks_by_anchor_text": list_backlinks_by_anchor_text,
+ "list_backlinks_to_url": list_backlinks_to_url,
+ "list_backlinks_from_domain": list_backlinks_from_domain,
+ "get_anchor_text_distribution": get_anchor_text_distribution,
+ "get_text_content_analysis": get_text_content_analysis,
+ "list_pages_containing_keyword": list_pages_containing_keyword,
+ "list_pages_by_word_count_band": list_pages_by_word_count_band,
+ "list_duplicate_content_pairs": list_duplicate_content_pairs,
+ "list_spell_check_issues": list_spell_check_issues,
+ "list_html_validation_issues": list_html_validation_issues,
+ "list_amp_validation_issues": list_amp_validation_issues,
+ "list_pagination_issues": list_pagination_issues,
+ "list_schema_errors_by_type": list_schema_errors_by_type,
+ "list_pages_missing_article_schema": list_pages_missing_article_schema,
+ "list_outbound_links": list_outbound_links,
+ "list_internal_links_from_url": list_internal_links_from_url,
+ "list_internal_links_to_url": list_internal_links_to_url,
+ "list_links_by_rel_nofollow": list_links_by_rel_nofollow,
+ "list_pagerank_low_pages": list_pagerank_low_pages,
+ "list_indexation_submitted_not_indexed": list_indexation_submitted_not_indexed,
+ "list_indexation_indexed_not_submitted": list_indexation_indexed_not_submitted,
+ "list_sitemap_urls_not_in_crawl": list_sitemap_urls_not_in_crawl,
+ "list_crawl_urls_not_in_sitemap": list_crawl_urls_not_in_sitemap,
+ "list_log_paths_by_hits": list_log_paths_by_hits,
+ "list_log_5xx_paths": list_log_5xx_paths,
+ "list_log_googlebot_low_crawl": list_log_googlebot_low_crawl,
+ "list_log_orphan_high_traffic": list_log_orphan_high_traffic,
+ "list_redirect_chains_by_length": list_redirect_chains_by_length,
+ "list_hreflang_reciprocal_gaps": list_hreflang_reciprocal_gaps,
+ "list_compare_new_issues": list_compare_new_issues,
+ "list_compare_resolved_issues": list_compare_resolved_issues,
+ "list_compare_new_urls": list_compare_new_urls,
+ "list_compare_removed_urls": list_compare_removed_urls,
+ "list_compare_lighthouse_regressions": list_compare_lighthouse_regressions,
+ "list_compare_traffic_losers": list_compare_traffic_losers,
+ "list_pages_missing_howto_schema": list_pages_missing_howto_schema,
+ "list_pages_ai_citation_signals": list_pages_ai_citation_signals,
+ "list_pages_missing_llms_txt_reference": list_pages_missing_llms_txt_reference,
+ "list_robots_blocked_ai_crawlers": list_robots_blocked_ai_crawlers,
+ "list_pages_console_errors_by_type": list_pages_console_errors_by_type,
+ "list_pages_js_rendering_delta": list_pages_js_rendering_delta,
}
@@ -512,10 +783,85 @@ def dispatch_tool(
return handler(session, merged_ctx, payload_args)
-def openai_tools_schema() -> list[dict[str, Any]]:
- """Convert TOOL_DEFINITIONS to OpenAI function-calling format."""
+_TOOL_DEFINITIONS_BY_NAME: dict[str, dict[str, Any]] = {t["name"]: t for t in TOOL_DEFINITIONS}
+_TOOL_META: dict[str, dict[str, Any]] = build_tool_meta(set(_TOOL_HANDLERS.keys()))
+
+
+def tool_meta() -> dict[str, dict[str, Any]]:
+ return _TOOL_META
+
+
+def tool_definition(name: str) -> dict[str, Any] | None:
+ return _TOOL_DEFINITIONS_BY_NAME.get(name)
+
+
+def tool_names_for_domain(domain: str) -> list[str]:
+ return _meta_tool_names_for_domain(_TOOL_META, domain)
+
+
+def tool_names_for_tier(tier: int) -> list[str]:
+ return _meta_tool_names_for_tier(_TOOL_META, tier)
+
+
+def tier0_tool_names() -> set[str]:
+ return set(TIER_0_TOOLS) & set(_TOOL_HANDLERS.keys())
+
+
+def mcp_tool_names(bundle: str) -> set[str]:
+ return tool_names_for_mcp_bundle(_TOOL_META, bundle) & set(_TOOL_HANDLERS.keys())
+
+
+def tools_catalog_by_domain() -> dict[str, list[str]]:
+ return tools_by_domain(_TOOL_META)
+
+
+def list_domains_catalog() -> list[dict[str, Any]]:
+ return domains_catalog(_TOOL_META)
+
+
+def search_tools(query: str, limit: int = 10) -> list[dict[str, Any]]:
+ """Keyword search over tool name, description, tags, and domain."""
+ q = (query or "").strip().lower()
+ if not q:
+ return []
+ tokens = [t for t in q.replace("/", " ").split() if t]
+ scored: list[tuple[int, str, dict[str, Any]]] = []
+ for tool in TOOL_DEFINITIONS:
+ name = tool["name"]
+ desc = str(tool.get("description") or "").lower()
+ meta = _TOOL_META.get(name) or {}
+ domain = str(meta.get("domain") or classify_tool_domain(name))
+ tags = " ".join(str(t) for t in (meta.get("tags") or []))
+ haystack = f"{name} {desc} {domain} {tags}".lower()
+ score = 0
+ if q in name:
+ score += 100
+ if q in haystack:
+ score += 40
+ for tok in tokens:
+ if tok in name:
+ score += 30
+ elif tok in haystack:
+ score += 10
+ if score <= 0:
+ continue
+ scored.append((score, name, {
+ "name": name,
+ "description": tool.get("description", ""),
+ "domain": domain,
+ "tier": meta.get("tier", 1),
+ }))
+ scored.sort(key=lambda x: (-x[0], x[1]))
+ cap = max(1, min(int(limit or 10), 50))
+ return [row for _, _, row in scored[:cap]]
+
+
+def openai_tools_schema(names: set[str] | None = None) -> list[dict[str, Any]]:
+ """Convert TOOL_DEFINITIONS to OpenAI function-calling format (optional name filter)."""
out: list[dict[str, Any]] = []
for tool in TOOL_DEFINITIONS:
+ if names is not None and tool["name"] not in names:
+ continue
out.append({
"type": "function",
"function": {
@@ -529,3 +875,19 @@ def openai_tools_schema() -> list[dict[str, Any]]:
def tool_handler_names() -> set[str]:
return set(_TOOL_HANDLERS.keys())
+
+
+def validate_tool_registry() -> list[str]:
+ """Return validation errors for catalog/handler/meta parity."""
+ errors: list[str] = []
+ handler_names = tool_handler_names()
+ catalog_names = {t["name"] for t in TOOL_DEFINITIONS}
+ meta_names = set(_TOOL_META.keys())
+ if handler_names != catalog_names:
+ errors.append(f"handler/catalog mismatch: handlers={len(handler_names)} catalog={len(catalog_names)}")
+ if handler_names != meta_names:
+ errors.append(f"handler/meta mismatch: handlers={len(handler_names)} meta={len(meta_names)}")
+ missing_t0 = TIER_0_TOOLS - handler_names
+ if missing_t0:
+ errors.append(f"tier0 tools missing handlers: {sorted(missing_t0)}")
+ return errors
diff --git a/src/website_profiling/tools/audit_tools/router_tools.py b/src/website_profiling/tools/audit_tools/router_tools.py
new file mode 100644
index 00000000..6eb0a941
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/router_tools.py
@@ -0,0 +1,147 @@
+"""Router and workflow meta-tools (Tier 0)."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+
+from ._slice import parse_limit
+from .context import AuditToolContext
+from .tool_domains import classify_tool_domain
+
+
+def search_audit_tools(_conn: Connection, _ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ from .registry import search_tools
+
+ query = str(args.get("query") or args.get("q") or "").strip()
+ limit = parse_limit(args.get("limit"), 10, 50)
+ if not query:
+ return {"error": "query is required", "tools": [], "tool_names": []}
+ matches = search_tools(query, limit=limit)
+ return {
+ "query": query,
+ "tools": matches,
+ "tool_names": [m["name"] for m in matches],
+ "total": len(matches),
+ }
+
+
+def list_tool_domains(_conn: Connection, _ctx: AuditToolContext, _args: dict[str, Any]) -> dict[str, Any]:
+ from .registry import list_domains_catalog, tools_catalog_by_domain
+
+ catalog = list_domains_catalog()
+ by_domain = tools_catalog_by_domain()
+ return {
+ "domains": catalog,
+ "domain_tool_counts": {d: len(by_domain.get(d) or []) for d in by_domain},
+ }
+
+
+def _dispatch(name: str, conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ from .registry import dispatch_tool
+
+ return dispatch_tool(name, args, context=ctx, conn=conn)
+
+
+def run_insight_workflow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ wf_type = str(args.get("type") or "priorities").strip().lower()
+ base = {"property_id": scoped.property_id, "report_id": scoped.report_id}
+ steps: list[dict[str, Any]] = []
+
+ if wf_type in ("traffic", "health"):
+ r = _dispatch("get_traffic_health_check", conn, scoped, base)
+ steps.append({"tool": "get_traffic_health_check", "result": r})
+ elif wf_type in ("landing_pages", "landing"):
+ r = _dispatch("get_landing_page_blended_table", conn, scoped, {**base, "limit": args.get("limit") or 30})
+ steps.append({"tool": "get_landing_page_blended_table", "result": r})
+ r2 = _dispatch("get_opportunity_matrix", conn, scoped, {**base, "limit": args.get("limit") or 30})
+ steps.append({"tool": "get_opportunity_matrix", "result": r2})
+ else:
+ r = _dispatch("get_opportunity_matrix", conn, scoped, {**base, "limit": args.get("limit") or 30})
+ steps.append({"tool": "get_opportunity_matrix", "result": r})
+ r2 = _dispatch("get_issue_to_traffic_map", conn, scoped, {**base, "limit": args.get("limit") or 20})
+ steps.append({"tool": "get_issue_to_traffic_map", "result": r2})
+
+ return {"workflow": "insight", "type": wf_type, "steps": steps}
+
+
+def run_technical_workflow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ base = {"property_id": scoped.property_id, "report_id": scoped.report_id}
+ steps = [
+ {"tool": "get_report_summary", "result": _dispatch("get_report_summary", conn, scoped, base)},
+ {"tool": "get_critical_issues", "result": _dispatch("get_critical_issues", conn, scoped, base)},
+ {"tool": "get_issue_priority_breakdown", "result": _dispatch("get_issue_priority_breakdown", conn, scoped, base)},
+ ]
+ baseline = args.get("baseline_report_id")
+ if baseline is not None:
+ steps.append({
+ "tool": "compare_issue_deltas",
+ "result": _dispatch("compare_issue_deltas", conn, scoped, {
+ **base,
+ "baseline_report_id": baseline,
+ }),
+ })
+ return {"workflow": "technical", "steps": steps}
+
+
+def run_keyword_workflow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ scoped = ctx.with_args(args)
+ if scoped.property_id is None:
+ return {"error": "property_id is required"}
+ base = {"property_id": scoped.property_id, "limit": args.get("limit") or 20}
+ steps = []
+ for tool_name in ("get_brand_keyword_split", "get_striking_distance_keywords", "list_keywords_ctr_opportunity"):
+ steps.append({"tool": tool_name, "result": _dispatch(tool_name, conn, scoped, base)})
+ return {"workflow": "keyword", "steps": steps}
+
+
+def run_domain_agent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]:
+ """Run a short scripted sequence of tools within one domain (subagent-style)."""
+ from .registry import search_tools, tool_names_for_domain, tool_meta
+
+ scoped = ctx.with_args(args)
+ task = str(args.get("task") or "").strip()
+ domain = str(args.get("domain") or "").strip().lower()
+ max_steps = parse_limit(args.get("max_steps"), 5, 8)
+ if not task:
+ return {"error": "task is required"}
+
+ meta = tool_meta()
+ if domain:
+ pool = set(tool_names_for_domain(domain))
+ else:
+ pool = set(meta.keys())
+
+ matches = search_tools(task, limit=max_steps * 2)
+ picked: list[str] = []
+ for m in matches:
+ name = m["name"]
+ if name in pool and name not in picked:
+ picked.append(name)
+ if len(picked) >= max_steps:
+ break
+
+ if not picked:
+ for m in matches:
+ name = m["name"]
+ if name not in picked and name in meta:
+ picked.append(name)
+ if len(picked) >= max_steps:
+ break
+
+ if not picked and domain:
+ picked = tool_names_for_domain(domain)[:max_steps]
+
+ base = {"property_id": scoped.property_id, "report_id": scoped.report_id, "limit": 20}
+ steps = []
+ for name in picked:
+ steps.append({"tool": name, "result": _dispatch(name, conn, scoped, base)})
+
+ return {
+ "task": task,
+ "domain": domain or classify_tool_domain(picked[0]) if picked else "",
+ "steps": steps,
+ "tools_used": picked,
+ }
diff --git a/src/website_profiling/tools/audit_tools/tool_catalog.py b/src/website_profiling/tools/audit_tools/tool_catalog.py
index f3f106f8..facbcad9 100644
--- a/src/website_profiling/tools/audit_tools/tool_catalog.py
+++ b/src/website_profiling/tools/audit_tools/tool_catalog.py
@@ -378,4 +378,129 @@ def _tool(name: str, description: str, properties: dict[str, Any], required: lis
_tool("get_bing_index_status", "Bing Webmaster URL info (requires bing_webmaster_api_key).", {"url": _URL, "property_id": _PID}, ["url", "property_id"]),
_tool("get_serp_feature_overlay", "Keywords with SERP feature / competition overlay data.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
_tool("check_ai_citation_presence", "On-site citation readiness estimate for brand/query (no live LLM API).", {"property_id": _PID, "query": {"type": "string"}, "brand": {"type": "string"}}),
+ # Router / Tier 0 (Cursor-style)
+ _tool("search_audit_tools", "Search the audit tool catalog by keyword. Returns matching tool names to call next.", {"query": {"type": "string"}, "limit": {"type": "integer", "maximum": 50}}, ["query"]),
+ _tool("list_tool_domains", "List SEO tool domains with counts and example prompts.", {}),
+ _tool("get_data_coverage_report", "Report which integrations and optional audit data are populated for a property.", {"property_id": _PID}, ["property_id"]),
+ _tool("run_insight_workflow", "Run insight workflow: type=traffic|landing_pages|priorities (default).", {"property_id": _PID, "report_id": _RID, "type": {"type": "string"}, "limit": _LIMIT}),
+ _tool("run_technical_workflow", "Run technical workflow: report summary, critical issues, priority chart; optional baseline compare.", {"property_id": _PID, "report_id": _RID, "baseline_report_id": {"type": "integer"}}),
+ _tool("run_keyword_workflow", "Run keyword workflow: brand split, striking distance, CTR opportunities.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("run_domain_agent", "Subagent-style: run up to max_steps tools in one domain for a task description.", {"property_id": _PID, "report_id": _RID, "task": {"type": "string"}, "domain": {"type": "string"}, "max_steps": {"type": "integer", "maximum": 8}}, ["task"]),
+ # Cross-platform insight (Tier 0)
+ _tool("get_landing_page_blended_table", "GSC clicks + GA4 sessions per landing page with opportunity quadrant.", {"property_id": _PID, "limit": {"type": "integer", "maximum": 100}, "min_impressions": {"type": "integer"}}, ["property_id"]),
+ _tool("get_opportunity_matrix", "Landing pages grouped by rank vs conversion opportunity quadrants.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]),
+ _tool("get_traffic_health_check", "GSC clicks vs GA4 sessions ratio and tracking health diagnosis.", {"property_id": _PID}, ["property_id"]),
+ _tool("get_landing_page_full_diagnosis", "One URL: GSC+GA4 slice, crawl, issues, Lighthouse, composite score.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]),
+ _tool("get_issue_to_traffic_map", "Audit issues ranked by traffic-weighted impact with GSC/GA4 columns.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ # Google dimensions
+ _tool("get_gsc_daily_trend", "GSC daily clicks/impressions trend from latest fetch.", {"property_id": _PID}, ["property_id"]),
+ _tool("get_ga4_daily_trend", "GA4 daily sessions trend from latest fetch.", {"property_id": _PID}, ["property_id"]),
+ _tool("get_ga4_by_device", "GA4 sessions breakdown by device category.", {"property_id": _PID}, ["property_id"]),
+ _tool("get_ga4_by_channel", "GA4 sessions breakdown by channel.", {"property_id": _PID}, ["property_id"]),
+ _tool("get_gsc_page_queries", "GSC queries for a single page URL (full by_page blob).", {"url": _URL, "property_id": _PID, "limit": _LIMIT}, ["url"]),
+ # Keyword dimensions
+ _tool("get_brand_keyword_split", "Branded vs non-branded keyword counts and samples.", {"property_id": _PID}, ["property_id"]),
+ _tool("list_keywords_by_intent", "Keywords filtered by intent (informational, commercial, etc.).", {"property_id": _PID, "intent": {"type": "string"}, "limit": _LIMIT}, ["property_id", "intent"]),
+
+ # --- 100 new audit tools (240→340) ---
+ _tool("list_pages_title_too_short", "Pages with title shorter than SEO minimum.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_title_too_long", "Pages with title longer than SEO maximum.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_slow_response", "Pages with server response time above threshold.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_missing_html_lang", "Pages missing html lang attribute.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_invalid_viewport", "Pages missing or invalid viewport meta.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_color_contrast_failures", "Pages failing color contrast checks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_high_reading_level", "Pages with high reading grade level.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_very_thin_content", "Pages with very low word count.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_hreflang_issue_pages", "Pages with hreflang cluster issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_missing_og_tags", "Pages missing Open Graph tags.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_missing_twitter_cards", "Pages missing Twitter card tags.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_invalid_json_ld", "Pages with invalid JSON-LD schema.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_mixed_language", "Pages with mixed-language content signals.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_orphan_hub_suggestions", "Suggested hub pages to link orphan URLs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_lighthouse_failure_lcp", "URLs failing Lighthouse LCP audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_lighthouse_failure_inp", "URLs failing Lighthouse INP audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_lighthouse_failure_cls", "URLs failing Lighthouse CLS audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_lighthouse_failure_seo", "URLs failing Lighthouse SEO audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_pages_by_impressions", "GSC pages sorted by impressions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_pages_by_clicks", "GSC pages sorted by clicks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_queries_by_impressions", "GSC queries sorted by impressions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_queries_by_clicks", "GSC queries sorted by clicks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_ctr_underperformers", "Queries/pages with low CTR for position band.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_decaying_pages", "GSC pages with click decline vs prior period.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_decaying_queries", "GSC queries with click decline vs prior period.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_new_queries", "New GSC queries vs prior period.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_ga4_landing_pages", "GA4 landing pages by sessions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_ga4_pages_by_bounce_rate", "GA4 pages with high bounce rate.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_ga4_pages_by_engagement_rate", "GA4 pages sorted by engagement rate.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("get_gsc_query_trend", "Daily GSC clicks/impressions for one query.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "query": {'type': 'string'}}, ['query']),
+ _tool("get_gsc_page_trend", "Daily GSC clicks/impressions for one URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Page URL'}}, ['url']),
+ _tool("get_ga4_path_trend", "GA4 sessions trend for one path.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "path": {'type': 'string'}}, ['path']),
+ _tool("list_gsc_ga4_mismatch_pages", "High GSC clicks but low GA4 sessions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_pages_by_position_band", "GSC pages in position range.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_position": {'type': 'number'}, "max_position": {'type': 'number'}}),
+ _tool("get_gsc_site_benchmarks", "Sitewide median CTR and position benchmarks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_branded_queries", "Branded GSC queries.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_gsc_non_branded_queries", "Non-branded GSC queries.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("compare_gsc_periods", "GSC summary delta vs prior Google snapshot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_keyword_rank_improvements", "Keywords with improved position vs prior snapshot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keyword_rank_declines", "Keywords with declined position vs prior snapshot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_new_to_top_10", "Keywords newly in top 10.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_fell_out_of_top_10", "Keywords that fell out of top 10.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_cannibalisation_queries", "Queries with multiple ranking URLs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_cannibalisation_urls", "URLs involved in keyword cannibalisation.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_misaligned_queries", "Queries landing on misaligned URLs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_by_recommended_action", "Keywords filtered by recommended_action.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "recommended_action": {'type': 'string'}}, ['property_id']),
+ _tool("list_keywords_by_serp_feature", "Keywords with given SERP feature.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "serp_feature": {'type': 'string'}}, ['property_id']),
+ _tool("list_semantic_cluster_queries", "Queries in a semantic keyword cluster.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "cluster_id": {'type': 'string'}}),
+ _tool("list_semantic_cluster_pages", "Pages in a semantic keyword cluster.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "cluster_id": {'type': 'string'}}),
+ _tool("get_keyword_opportunity_score", "Composite opportunity score for one keyword.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "keyword": {'type': 'string'}}, ['keyword', 'property_id']),
+ _tool("list_keywords_near_page_one", "Keywords in striking-distance position band.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_high_impression_zero_click", "High-impression keywords with zero clicks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_by_competition_band", "Keywords by SERP competition band.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_competition": {'type': 'number'}, "max_competition": {'type': 'number'}}, ['property_id']),
+ _tool("get_keyword_serp_snapshot", "SERP overlay fields for one keyword.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "keyword": {'type': 'string'}}, ['keyword', 'property_id']),
+ _tool("list_keywords_with_ai_overview", "Keywords triggering AI overview SERP features.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_local_pack", "Keywords with local pack SERP feature.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_question_intent", "Question-intent keywords.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_keywords_commercial_intent", "Commercial-intent keywords.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_referring_domains", "Top referring domains from GSC Links.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("list_backlinks_by_anchor_text", "Backlinks filtered by anchor text.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "anchor_text": {'type': 'string'}}, ['property_id']),
+ _tool("list_backlinks_to_url", "Inbound links to target URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Target URL'}}, ['url', 'property_id']),
+ _tool("list_backlinks_from_domain", "Backlinks from referring domain.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "domain": {'type': 'string'}}, ['domain', 'property_id']),
+ _tool("get_anchor_text_distribution", "Sitewide anchor text frequency from GSC Links.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']),
+ _tool("get_text_content_analysis", "Sitewide text content analysis summary.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_containing_keyword", "Pages containing keyword in body.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "keyword": {'type': 'string'}}, ['keyword']),
+ _tool("list_pages_by_word_count_band", "Pages filtered by word count range.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_words": {'type': 'integer'}, "max_words": {'type': 'integer'}}),
+ _tool("list_duplicate_content_pairs", "Near-duplicate content pairs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_spell_check_issues", "Spell-check optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_html_validation_issues", "HTML validation optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_amp_validation_issues", "AMP validation optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pagination_issues", "Pagination optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_schema_errors_by_type", "Schema validation errors by type.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "schema_type": {'type': 'string'}}),
+ _tool("list_pages_missing_article_schema", "Article/blog URLs missing Article schema.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_outbound_links", "Outbound links from crawl graph.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_internal_links_from_url", "Internal outlinks from source URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Source URL'}}, ['url']),
+ _tool("list_internal_links_to_url", "Internal inlinks to target URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Target URL'}}, ['url']),
+ _tool("list_links_by_rel_nofollow", "Links filtered by rel attribute.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "rel": {'type': 'string'}}),
+ _tool("list_pagerank_low_pages", "Pages with lowest internal PageRank.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "max_pagerank": {'type': 'number'}}),
+ _tool("list_indexation_submitted_not_indexed", "URLs submitted but not indexed.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_indexation_indexed_not_submitted", "Indexed URLs not in sitemap.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_sitemap_urls_not_in_crawl", "Sitemap URLs missing from crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_crawl_urls_not_in_sitemap", "Crawled URLs missing from sitemap.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_log_paths_by_hits", "Access log paths ranked by hits.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_log_5xx_paths", "Access log paths with 5xx responses.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_log_googlebot_low_crawl", "High-value paths under-crawled by Googlebot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_log_orphan_high_traffic", "Orphan URLs with high log traffic.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_redirect_chains_by_length", "Redirect chains above min_length.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_length": {'type': 'integer'}}),
+ _tool("list_hreflang_reciprocal_gaps", "Hreflang alternates missing reciprocal return tags.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_compare_new_issues", "Issues new since baseline report.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']),
+ _tool("list_compare_resolved_issues", "Issues resolved since baseline report.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']),
+ _tool("list_compare_new_urls", "URLs new since baseline crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']),
+ _tool("list_compare_removed_urls", "URLs removed since baseline crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']),
+ _tool("list_compare_lighthouse_regressions", "Lighthouse score regressions vs baseline.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']),
+ _tool("list_compare_traffic_losers", "GSC click losers vs baseline report.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']),
+ _tool("list_pages_missing_howto_schema", "How-to URLs missing HowTo schema.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_ai_citation_signals", "Pages with AEO/AI citation readiness signals.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_missing_llms_txt_reference", "Pages not referenced in llms.txt.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_robots_blocked_ai_crawlers", "Pages blocking AI crawler user-agents.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
+ _tool("list_pages_console_errors_by_type", "Console errors filtered by error_type.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "error_type": {'type': 'string'}}),
+ _tool("list_pages_js_rendering_delta", "URLs with static vs rendered content delta.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}),
]
diff --git a/src/website_profiling/tools/audit_tools/tool_domains.py b/src/website_profiling/tools/audit_tools/tool_domains.py
new file mode 100644
index 00000000..6f69d672
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/tool_domains.py
@@ -0,0 +1,339 @@
+"""Explicit tool domain/tier metadata for routing and MCP domain servers."""
+from __future__ import annotations
+
+from typing import Any
+
+CANONICAL_DOMAINS: tuple[str, ...] = (
+ "core",
+ "portfolio",
+ "issues",
+ "crawl",
+ "onpage",
+ "schema",
+ "links",
+ "indexation",
+ "content",
+ "keywords",
+ "google",
+ "backlinks",
+ "performance",
+ "drift",
+ "security",
+ "ops",
+ "export",
+ "images",
+ "geo",
+ "accessibility",
+ "assets",
+ "ctr",
+ "integrations",
+ "insight",
+)
+
+# Tier 0 — always included in chat dynamic routing (router + top insight tools).
+TIER_0_TOOLS: frozenset[str] = frozenset({
+ "search_audit_tools",
+ "list_tool_domains",
+ "get_data_coverage_report",
+ "run_insight_workflow",
+ "run_technical_workflow",
+ "run_keyword_workflow",
+ "run_domain_agent",
+ "get_report_summary",
+ "list_top_impact_issues",
+ "prioritize_fix_roadmap",
+ "get_landing_page_blended_table",
+ "get_opportunity_matrix",
+ "get_traffic_health_check",
+ "get_landing_page_full_diagnosis",
+ "get_issue_to_traffic_map",
+ "get_google_summary",
+})
+
+# Explicit domain overrides (name -> domain).
+_DOMAIN_OVERRIDES: dict[str, str] = {
+ "search_audit_tools": "core",
+ "list_tool_domains": "core",
+ "get_data_coverage_report": "core",
+ "run_insight_workflow": "core",
+ "run_technical_workflow": "core",
+ "run_keyword_workflow": "core",
+ "run_domain_agent": "core",
+ "get_landing_page_blended_table": "insight",
+ "get_opportunity_matrix": "insight",
+ "get_traffic_health_check": "insight",
+ "get_landing_page_full_diagnosis": "insight",
+ "get_issue_to_traffic_map": "insight",
+ "get_gsc_daily_trend": "google",
+ "get_ga4_daily_trend": "google",
+ "get_ga4_by_device": "google",
+ "get_ga4_by_channel": "google",
+ "get_brand_keyword_split": "keywords",
+ "list_keywords_by_intent": "keywords",
+ "get_gsc_page_queries": "google",
+ "list_broken_links": "links",
+ "list_broken_link_sources": "links",
+ "get_gsc_sample_links": "backlinks",
+ "get_gsc_latest_links": "backlinks",
+ "get_gsc_links_summary": "backlinks",
+ "get_gsc_links_import_status": "backlinks",
+ "list_seo_onpage_issues": "onpage",
+ "list_content_url_issues": "onpage",
+ "list_pages_missing_title": "onpage",
+ "list_pages_missing_h1": "onpage",
+ "list_pages_multiple_h1": "onpage",
+ "list_pages_missing_meta_description": "onpage",
+ "list_pages_meta_desc_too_short": "onpage",
+ "list_pages_meta_desc_too_long": "onpage",
+ "list_pages_noindex": "onpage",
+ "list_pages_missing_canonical": "onpage",
+ "list_canonical_mismatch": "onpage",
+ "list_pages_with_missing_alt": "onpage",
+ "list_pages_skipped_headings": "onpage",
+ "list_pages_missing_viewport": "onpage",
+ "list_pages_missing_og_image": "onpage",
+ "get_report_summary": "portfolio",
+ "get_critical_issues": "issues",
+ "get_issue_priority_breakdown": "issues",
+ "list_top_impact_issues": "issues",
+ "prioritize_fix_roadmap": "issues",
+ "get_google_summary": "google",
+ "get_gsc_ctr_opportunity_pages": "ctr",
+ "list_keywords_ctr_opportunity": "ctr",
+ "analyze_serp_snippet_for_url": "ctr",
+ "compare_reports": "drift",
+ "compare_gsc_periods": "google",
+ "list_pages_title_too_short": "onpage",
+ "list_pages_title_too_long": "onpage",
+ "list_pages_slow_response": "performance",
+ "list_pages_color_contrast_failures": "accessibility",
+ "list_pages_high_reading_level": "content",
+ "list_pages_very_thin_content": "content",
+ "list_hreflang_issue_pages": "indexation",
+ "list_pages_mixed_language": "content",
+ "list_misaligned_queries": "keywords",
+ "list_referring_domains": "backlinks",
+ "get_anchor_text_distribution": "backlinks",
+ "list_backlinks_by_anchor_text": "backlinks",
+ "list_backlinks_to_url": "backlinks",
+ "list_backlinks_from_domain": "backlinks",
+ "get_keyword_opportunity_score": "keywords",
+ "list_sitemap_urls_not_in_crawl": "indexation",
+ "list_crawl_urls_not_in_sitemap": "indexation",
+ "list_log_googlebot_low_crawl": "ops",
+ "list_redirect_chains_by_length": "crawl",
+ "list_compare_new_issues": "drift",
+ "list_compare_resolved_issues": "drift",
+ "list_compare_lighthouse_regressions": "drift",
+ "list_pages_ai_citation_signals": "geo",
+ "list_pages_missing_llms_txt_reference": "geo",
+ "list_robots_blocked_ai_crawlers": "geo",
+ "list_pages_missing_howto_schema": "geo",
+ "list_pages_missing_article_schema": "geo",
+ "list_gsc_ctr_underperformers": "google",
+}
+
+_ONPAGE_PREFIXES = (
+ "list_pages_missing_",
+ "list_pages_meta_desc_",
+ "list_pages_multiple_h1",
+ "list_pages_noindex",
+ "list_seo_onpage",
+ "list_content_url",
+)
+
+_INSIGHT_PREFIXES = (
+ "get_landing_page_",
+ "get_opportunity_",
+ "get_traffic_health",
+ "get_issue_to_traffic",
+)
+
+# MCP server bundles (WP_MCP_DOMAIN env).
+MCP_DOMAIN_BUNDLES: dict[str, frozenset[str]] = {
+ "core": frozenset({"core", "insight"}),
+ "crawl": frozenset({"crawl", "onpage", "schema", "accessibility", "assets"}),
+ "google": frozenset({"google", "insight", "ctr", "keywords", "integrations"}),
+ "links": frozenset({"links", "backlinks", "indexation"}),
+ "full": frozenset(CANONICAL_DOMAINS),
+}
+
+DOMAIN_EXAMPLE_PROMPTS: dict[str, str] = {
+ "core": "What data do we have? Search for a specialized tool.",
+ "portfolio": "Give me an audit overview and health score.",
+ "issues": "What are the top critical issues to fix first?",
+ "crawl": "List 4xx pages and redirect chains.",
+ "onpage": "Which pages are missing title tags or meta descriptions?",
+ "google": "Top GSC queries and GA4 landing page performance.",
+ "insight": "High-click pages with low engagement — opportunity matrix.",
+ "drift": "Compare this audit to the previous report.",
+ "export": "Export the audit as PDF.",
+ "keywords": "Striking-distance keywords and CTR opportunities.",
+ "performance": "Lighthouse summary and Core Web Vitals failures.",
+ "links": "Orphan pages and broken internal links.",
+ "backlinks": "GSC backlinks sample and velocity.",
+ "images": "Image audit summary and largest unoptimized images.",
+ "geo": "GEO readiness score and llms.txt status.",
+}
+
+
+def classify_tool_domain(name: str) -> str:
+ """Return canonical domain for a tool name."""
+ if name in _DOMAIN_OVERRIDES:
+ return _DOMAIN_OVERRIDES[name]
+ if name.startswith("compare_"):
+ return "drift"
+ if name.startswith("list_compare_"):
+ return "drift"
+ if name in TIER_0_TOOLS:
+ return _DOMAIN_OVERRIDES.get(name, "core")
+
+ if name.startswith("export_") or name in ("compose_custom_report", "list_export_formats"):
+ return "export"
+ if name.startswith((
+ "get_image_", "list_pages_without_lazy", "list_pages_with_images_missing",
+ "list_site_image", "list_lighthouse_image", "list_largest_images",
+ "list_unoptimized_images", "list_images_needing",
+ )):
+ return "images"
+ if name.startswith((
+ "get_landing_page_", "get_opportunity_", "get_traffic_health", "get_issue_to_traffic",
+ )):
+ return "insight"
+ if name.startswith(("get_geo_", "get_aeo_", "get_llms_", "get_eeat_", "get_faq_",
+ "list_pages_missing_faq", "draft_llms", "check_ai_citation")):
+ return "geo"
+ if "axe" in name or "mixed_content" in name or name == "get_heading_outline_for_url":
+ return "accessibility"
+ if name in (
+ "get_asset_weight_summary", "get_readability_summary", "list_heavy_pages_by_bytes",
+ "list_pages_poor_cache_headers", "list_pages_low_content_ratio",
+ ):
+ return "assets"
+ if "ctr" in name or name in ("list_keywords_ctr_opportunity", "analyze_serp_snippet_for_url"):
+ return "ctr"
+ if name in ("get_gsc_url_inspection", "get_gsc_index_coverage", "get_bing_index_status", "get_serp_feature_overlay"):
+ return "integrations"
+ if any(name.startswith(p) for p in _ONPAGE_PREFIXES):
+ return "onpage"
+ if name.startswith((
+ "list_propert", "get_propert", "get_report", "get_executive", "get_site", "list_report",
+ "get_portfolio",
+ )) or name in (
+ "get_ads_txt_status", "get_security_txt_status", "get_contact_intelligence",
+ "get_rich_results_summary", "list_rich_results_failures", "get_competitor_keyword_gap",
+ "get_pagination_audit_summary", "get_portfolio_benchmark",
+ ):
+ return "portfolio"
+ if name in (
+ "list_top_impact_issues", "prioritize_fix_roadmap", "generate_issue_fix",
+ "summarize_category_for_client",
+ ) or "issue" in name or "category" in name or "workflow" in name:
+ return "issues"
+ if name.startswith((
+ "list_pages_", "list_canonical", "list_long_", "list_robots_", "get_top_pages_by",
+ "search_pages", "get_page_", "list_redirects", "list_broken", "list_status_",
+ "get_status_code", "get_response_time", "get_depth", "get_crawl_", "get_browser",
+ "list_pages_with", "list_pages_by", "list_pages_soft", "list_pages_poor",
+ "list_dead_end", "list_duplicate_title", "list_heavy_pages",
+ )):
+ return "crawl"
+ if "schema" in name or name == "get_seo_health":
+ return "schema"
+ if "orphan" in name or "link" in name or "fingerprint" in name or "pagerank" in name:
+ return "links"
+ if "indexation" in name or "hreflang" in name or "language" in name or name == "list_subdomains":
+ return "indexation"
+ if "content" in name or "social" in name or "ner" in name or "thin" in name or "opportunit" in name or "duplicate" in name:
+ return "content"
+ if "keyword" in name or "cannibal" in name or "misalignment" in name or "striking" in name or "semantic" in name or name in ("expand_keywords", "generate_content_brief"):
+ return "keywords"
+ if "google" in name or "gsc" in name or "ga4" in name:
+ return "google"
+ if "backlink" in name or "competitor" in name or "bing" in name or "gsc_links" in name:
+ return "backlinks"
+ if "lighthouse" in name or "crux" in name or "slow" in name or "cwv" in name:
+ return "performance"
+ if "health" in name or "compare" in name or "alert" in name or "tech_stack" in name or name == "list_pages_by_technology":
+ return "drift"
+ if "security" in name:
+ return "security"
+ if "log" in name or name in ("get_property_ops", "list_crawl_runs", "list_log_uploads", "get_page_coach"):
+ return "ops"
+ return "portfolio"
+
+
+def _tags_for_tool(name: str, domain: str) -> list[str]:
+ tags = [domain]
+ parts = name.replace("_", " ").split()
+ tags.extend(p for p in parts if len(p) > 2 and p not in tags)
+ return tags[:8]
+
+
+def build_tool_meta(tool_names: set[str] | frozenset[str]) -> dict[str, dict[str, Any]]:
+ """Build TOOL_META for all registered tool names."""
+ meta: dict[str, dict[str, Any]] = {}
+ for name in sorted(tool_names):
+ domain = classify_tool_domain(name)
+ tier = 0 if name in TIER_0_TOOLS else 1
+ meta[name] = {
+ "domain": domain,
+ "tier": tier,
+ "tags": _tags_for_tool(name, domain),
+ }
+ return meta
+
+
+def tools_by_domain(meta: dict[str, dict[str, Any]]) -> dict[str, list[str]]:
+ out: dict[str, list[str]] = {d: [] for d in CANONICAL_DOMAINS}
+ for name, info in meta.items():
+ domain = str(info.get("domain") or "portfolio")
+ if domain not in out:
+ out[domain] = []
+ out[domain].append(name)
+ for domain in out:
+ out[domain].sort()
+ return out
+
+
+def tool_names_for_domain(meta: dict[str, dict[str, Any]], domain: str) -> list[str]:
+ by_domain = tools_by_domain(meta)
+ return list(by_domain.get(domain) or [])
+
+
+def tool_names_for_tier(meta: dict[str, dict[str, Any]], tier: int) -> list[str]:
+ return sorted(name for name, info in meta.items() if info.get("tier") == tier)
+
+
+def tool_names_for_mcp_bundle(meta: dict[str, dict[str, Any]], bundle: str) -> set[str]:
+ """Return tool names exposed for an MCP domain bundle."""
+ bundle_key = (bundle or "core").strip().lower()
+ allowed_domains = MCP_DOMAIN_BUNDLES.get(bundle_key, MCP_DOMAIN_BUNDLES["core"])
+ if bundle_key == "full":
+ return set(meta.keys())
+ names: set[str] = set()
+ by_domain = tools_by_domain(meta)
+ for domain in allowed_domains:
+ names.update(by_domain.get(domain) or [])
+ if bundle_key == "core":
+ names.update(TIER_0_TOOLS & set(meta.keys()))
+ return names
+
+
+def domains_catalog(meta: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
+ by_domain = tools_by_domain(meta)
+ rows: list[dict[str, Any]] = []
+ for domain in CANONICAL_DOMAINS:
+ tools = by_domain.get(domain) or []
+ if not tools:
+ continue
+ rows.append({
+ "domain": domain,
+ "tool_count": len(tools),
+ "example_prompt": DOMAIN_EXAMPLE_PROMPTS.get(domain, ""),
+ "mcp_bundle": next(
+ (b for b, domains in MCP_DOMAIN_BUNDLES.items() if domain in domains and b != "full"),
+ "full",
+ ),
+ })
+ return rows
diff --git a/src/website_profiling/tools/audit_tools/tool_selector.py b/src/website_profiling/tools/audit_tools/tool_selector.py
new file mode 100644
index 00000000..949ff7e4
--- /dev/null
+++ b/src/website_profiling/tools/audit_tools/tool_selector.py
@@ -0,0 +1,151 @@
+"""Dynamic tool selection for chat agent (Cursor-style subset loading)."""
+from __future__ import annotations
+
+import os
+import re
+from typing import Any
+
+from .tool_domains import TIER_0_TOOLS
+from .registry import tier0_tool_names, tool_meta, tool_names_for_domain
+
+
+DOMAIN_KEYWORDS: dict[str, tuple[str, ...]] = {
+ "issues": ("issue", "issues", "critical issues", "fix", "priority", "roadmap", "impact"),
+ "crawl": ("crawl", "404", "500", "redirect", "status code", "orphan", "soft 404", "robots"),
+ "onpage": ("title tag", "meta description", "h1", "canonical", "noindex", "on-page", "onpage"),
+ "google": ("gsc", "search console", "ga4", "analytics", "clicks", "impressions", "queries"),
+ "insight": ("opportunity", "engagement", "landing page", "blended", "traffic health", "diagnosis"),
+ "keywords": ("keyword", "striking", "cannibal", "brand", "intent"),
+ "performance": ("lighthouse", "cwv", "core web vitals", "slow page", "crux", "page speed"),
+ "links": ("broken link", "internal link", "inlink", "outlink", "anchor text", "pagerank"),
+ "backlinks": ("backlink", "referring domain", "gsc links", "moz", "majestic"),
+ "drift": ("compare", "baseline", "delta", "history", "trend", "drift"),
+ "export": ("export", "pdf", "csv", "download"),
+ "images": ("image", "alt text", "lazy load", "webp", "lcp image"),
+ "geo": ("geo", "aeo", "llms.txt", "faq schema", "eeat"),
+ "accessibility": ("axe", "accessibility", "a11y", "mixed content"),
+ "security": ("security", "tls", "hsts", "ssl"),
+ "indexation": ("indexation", "sitemap", "hreflang", "indexed"),
+ "content": ("duplicate content", "thin content", "word count", "readability"),
+ "ops": ("access log", "log analysis", "log upload", "crawl run", "integration status", "5xx", "googlebot"),
+ "portfolio": ("overview", "health score", "category scores", "executive", "portfolio", "audit summary"),
+ "ctr": ("ctr", "snippet", "title meta ctr"),
+}
+
+# High-value tools referenced in the chat system prompt playbooks — pinned when domain matches.
+PLAYBOOK_ANCHORS: dict[str, tuple[str, ...]] = {
+ "images": ("get_image_audit_summary",),
+ "export": ("export_audit_report", "export_list_as_csv"),
+ "issues": ("get_critical_issues", "get_issue_priority_breakdown", "list_issues"),
+ "portfolio": ("get_category_scores", "list_audit_categories"),
+ "performance": ("get_lighthouse_summary", "list_pages_slow_response", "list_lighthouse_failure_lcp"),
+ "drift": ("compare_reports", "compare_issue_deltas", "list_compare_traffic_losers"),
+ "google": ("get_gsc_top_queries", "get_ga4_page_metrics", "list_gsc_decaying_queries", "list_gsc_decaying_pages"),
+ "keywords": ("get_striking_distance_keywords", "get_keyword_cannibalisation", "list_keyword_rank_declines"),
+ "indexation": ("list_hreflang_issue_pages", "list_indexation_gaps"),
+ "backlinks": ("list_referring_domains", "list_backlinks_by_anchor_text"),
+ "ops": ("list_log_paths_by_hits", "list_log_5xx_paths"),
+}
+
+
+def chat_tool_mode() -> str:
+ return (os.environ.get("CHAT_TOOL_MODE") or "dynamic").strip().lower()
+
+
+def chat_tool_max() -> int:
+ floor = len(TIER_0_TOOLS) + 1
+ try:
+ return max(floor, min(int(os.environ.get("CHAT_TOOL_MAX") or 45), 120))
+ except (TypeError, ValueError):
+ return max(floor, 45)
+
+
+def chat_tool_search_cap() -> int:
+ """Soft cap after search/domain-agent expansion (Tier 0 + pinned results)."""
+ return min(chat_tool_max() + 15, 75)
+
+
+def _keyword_in_text(keyword: str, text: str) -> bool:
+ if " " in keyword:
+ return keyword in text
+ return re.search(rf"\b{re.escape(keyword)}\b", text) is not None
+
+
+def _score_domains(text: str) -> list[tuple[int, str]]:
+ lower = text.lower()
+ scores: list[tuple[int, str]] = []
+ for domain, keywords in DOMAIN_KEYWORDS.items():
+ score = sum(3 if _keyword_in_text(kw, lower) else 0 for kw in keywords)
+ if score > 0:
+ scores.append((score, domain))
+ scores.sort(key=lambda x: (-x[0], x[1]))
+ return scores
+
+
+def apply_tool_cap(
+ selected: set[str],
+ cap: int,
+ *,
+ pinned: set[str] | None = None,
+ max_pinned: int = 12,
+) -> set[str]:
+ """Trim tool set while preserving Tier 0 and optionally pinned names."""
+ pinned = pinned or set()
+ tier0 = set(TIER_0_TOOLS) & selected
+ pinned_keep = sorted(pinned & selected)[: max(0, max_pinned)]
+ must_keep = tier0 | set(pinned_keep)
+ if len(selected) <= cap:
+ return selected
+ rest = sorted(selected - must_keep)
+ room = max(0, cap - len(must_keep))
+ return must_keep | set(rest[:room])
+
+
+def select_tools_for_turn(
+ user_message: str,
+ history: list[dict[str, Any]] | None = None,
+ *,
+ max_tools: int | None = None,
+ extra_names: set[str] | None = None,
+) -> set[str]:
+ """Return tool names to expose to the LLM this turn (Tier 0 + relevant Tier 1)."""
+ if chat_tool_mode() == "full":
+ from .registry import tool_handler_names
+ return tool_handler_names()
+
+ cap = max_tools if max_tools is not None else chat_tool_max()
+ selected: set[str] = set(tier0_tool_names())
+ if extra_names:
+ selected |= extra_names
+
+ texts = [user_message or ""]
+ if history:
+ for msg in reversed(history):
+ if msg.get("role") == "user":
+ prior = str(msg.get("content") or "")
+ if prior and prior != (user_message or ""):
+ texts.append(prior)
+ break
+ combined = " ".join(texts)
+ domain_scores = _score_domains(combined)
+ scored_domains = [domain for _, domain in domain_scores[:4]]
+
+ meta = tool_meta()
+ if not domain_scores:
+ for fallback in ("portfolio", "issues", "insight"):
+ selected.update(tool_names_for_domain(fallback))
+ else:
+ for domain in scored_domains:
+ selected.update(tool_names_for_domain(domain))
+ for anchor in PLAYBOOK_ANCHORS.get(domain, ()):
+ if anchor in meta:
+ selected.add(anchor)
+
+ selected = apply_tool_cap(selected, cap)
+ selected = {n for n in selected if n in meta or n in tier0_tool_names()}
+ return selected
+
+
+def compact_tool_list(names: set[str]) -> str:
+ lines = sorted(names)
+ return "\n".join(f"- {n}" for n in lines)
diff --git a/src/website_profiling/tools/export_audit.py b/src/website_profiling/tools/export_audit.py
index 599a4c78..03cfe76b 100644
--- a/src/website_profiling/tools/export_audit.py
+++ b/src/website_profiling/tools/export_audit.py
@@ -6,26 +6,36 @@
import io
import json
from datetime import datetime, timezone
-from typing import Any, Optional
+from typing import Optional
from ..db import db_session, read_report_payload
from ..reporting.terminology import category_display_name
-
-_GLOSSARY_ROWS: list[tuple[str, str]] = [
- ("Crawl", "URLs fetched by the site spider (status codes, titles, inlinks)."),
- ("Lighthouse", "Lab Core Web Vitals audit (LCP, CLS, TBT, and category scores)."),
- ("Google Search Console", "Queries, pages, clicks, impressions, and average position from GSC."),
- ("Google Analytics 4", "Sessions, users, and engagement from GA4."),
- ("Estimated", "Derived from crawl text only — not Google search volume or rankings."),
- ("AI insights", "Optional LLM summaries — verify before client delivery."),
-]
-
-_ISSUE_LIMIT_HTML = 200
-_ISSUE_LIMIT_PDF = 80
-_LINK_LIMIT = 50
-
-
-def _load_payload(report_id: Optional[int] = None) -> dict[str, Any]:
+from .export_audit_data import (
+ _GLOSSARY_ROWS,
+ _ISSUE_LIMIT_HTML,
+ _ISSUE_LIMIT_PDF,
+ _LINK_LIMIT,
+ _executive_export_data,
+ _executive_source_label,
+ _format_report_date,
+ _issue_priority_counts,
+ _issue_recommendation,
+ _issues_rows,
+ _overall_score,
+ _priority_sort_key,
+ _score_band,
+ _summary_lines,
+)
+from .export_audit_html import (
+ _category_cards_html,
+ _executive_summary_html,
+ _priority_stats_html,
+ _report_html_styles,
+)
+
+
+def _load_payload(report_id: Optional[int] = None) -> dict:
+ """Load report payload from DB (uses module-level db_session for test patches)."""
with db_session() as conn:
payload = read_report_payload(conn, report_id)
if not payload:
@@ -33,522 +43,6 @@ def _load_payload(report_id: Optional[int] = None) -> dict[str, Any]:
return payload
-def _issue_recommendation(issue: dict[str, Any]) -> tuple[str, str]:
- """Return (display recommendation, llm_recommendation if distinct)."""
- rule = str(issue.get("recommendation") or "").strip()
- llm = str(issue.get("llm_recommendation") or "").strip()
- if llm and llm != rule:
- display = llm if llm else rule
- return display, llm
- return llm or rule, llm
-
-
-def _issues_rows(payload: dict[str, Any]) -> list[dict[str, str]]:
- rows: list[dict[str, str]] = []
- for cat in payload.get("categories") or []:
- if not isinstance(cat, dict):
- continue
- cat_name = str(cat.get("name") or "")
- ui_name = category_display_name(cat_name)
- for issue in cat.get("issues") or []:
- if not isinstance(issue, dict):
- continue
- rec, llm_rec = _issue_recommendation(issue)
- rows.append({
- "category": ui_name,
- "priority": str(issue.get("priority") or ""),
- "message": str(issue.get("message") or ""),
- "url": str(issue.get("url") or ""),
- "recommendation": rec,
- "llm_recommendation": llm_rec,
- })
- return rows
-
-
-def _executive_export_data(payload: dict[str, Any]) -> dict[str, Any]:
- """Normalize executive_summary and legacy recommendations for export."""
- exec_sum = payload.get("executive_summary")
- summary = ""
- priorities: list[str] = []
- top_issues: list[dict[str, Any]] = []
- source = ""
- if isinstance(exec_sum, dict):
- summary = str(exec_sum.get("summary") or "").strip()
- source = str(exec_sum.get("source") or "").strip()
- raw_pri = exec_sum.get("priorities") or []
- if isinstance(raw_pri, list):
- priorities = [str(p).strip() for p in raw_pri if str(p).strip()]
- raw_top = exec_sum.get("top_issues") or []
- if isinstance(raw_top, list):
- top_issues = [i for i in raw_top if isinstance(i, dict)][:8]
-
- legacy_recs = payload.get("recommendations") or []
- legacy_list: list[str] = []
- if isinstance(legacy_recs, list):
- legacy_list = [str(r).strip() for r in legacy_recs if str(r).strip()]
-
- if not summary and legacy_list:
- summary = "\n".join(f"• {r}" for r in legacy_list[:12])
-
- return {
- "summary": summary,
- "priorities": priorities,
- "top_issues": top_issues,
- "source": source,
- "legacy_recommendations": legacy_list,
- }
-
-
-def _executive_source_label(source: str) -> str:
- if source == "ai_insights":
- return "AI insights"
- if source == "deterministic":
- return "Measured + Search Console"
- return source or "Audit data"
-
-
-def _executive_summary_html(payload: dict[str, Any]) -> str:
- data = _executive_export_data(payload)
- if not data["summary"] and not data["priorities"] and not data["top_issues"]:
- return ""
-
- parts: list[str] = ['Executive summary ']
- if data["source"]:
- parts.append(
- f'Source: {html.escape(_executive_source_label(data["source"]))}
'
- )
- if data["summary"]:
- summary_html = html.escape(data["summary"]).replace("\n", "{html.escape(p)} " for p in data["priorities"][:8])
- parts.append(f"Priorities ")
-
- if data["top_issues"]:
- rows = ""
- for iss in data["top_issues"]:
- pri = str(iss.get("priority") or "").lower()
- badge_cls = f"badge-{pri}" if pri in {"critical", "high", "medium", "low"} else "badge-low"
- clicks = iss.get("gsc_clicks")
- clicks_txt = ""
- if clicks is not None:
- try:
- if float(clicks) > 0:
- clicks_txt = f' · {int(float(clicks))} GSC clicks'
- except (TypeError, ValueError):
- pass
- rows += (
- ""
- f"{html.escape(str(iss.get('priority') or ''))} {html.escape(str(iss.get('message') or ''))} "
- f"{html.escape(str(iss.get('url') or ''))} "
- f"{html.escape(clicks_txt.lstrip(' · ') if clicks_txt else '—')} "
- " "
- )
- parts.append(
- "Top traffic-impacting issues "
- ''
- "Priority Issue URL GSC clicks "
- f" {rows}
"
- )
-
- parts.append("'
- f'{score_txt}
'
- f'{name}
'
- f'{issue_n} issue{"s" if issue_n != 1 else ""}
'
- f" "
- )
- return "".join(cards) or 'No category scores available.
'
-
-
-def _priority_stats_html(counts: dict[str, int]) -> str:
- labels = (
- ("critical", "Critical"),
- ("high", "High"),
- ("medium", "Medium"),
- ("low", "Low"),
- )
- parts: list[str] = []
- for key, label in labels:
- n = counts.get(key, 0)
- parts.append(
- f''
- f'{n} '
- f'{label} '
- f"
"
- )
- return "".join(parts)
-
-
-def _report_html_styles() -> str:
- return """
- :root {
- --ink: #0f172a;
- --muted: #64748b;
- --line: #e2e8f0;
- --surface: #ffffff;
- --surface-muted: #f8fafc;
- --brand: #0b0f19;
- --brand-accent: #2563eb;
- --good: #059669;
- --good-bg: #ecfdf5;
- --fair: #d97706;
- --fair-bg: #fffbeb;
- --poor: #dc2626;
- --poor-bg: #fef2f2;
- --critical: #991b1b;
- --high: #c2410c;
- --medium: #a16207;
- --low: #475569;
- }
- * { box-sizing: border-box; }
- body {
- margin: 0;
- background: #eef2f7;
- color: var(--ink);
- font: 400 15px/1.55 "Segoe UI", system-ui, -apple-system, sans-serif;
- }
- .report { max-width: 920px; margin: 0 auto; background: var(--surface); }
- .cover {
- background: linear-gradient(135deg, #0b0f19 0%, #111827 55%, #1e3a5f 100%);
- color: #f8fafc;
- padding: 2.5rem 2.75rem 2rem;
- }
- .cover-brand {
- font-size: 0.72rem;
- letter-spacing: 0.14em;
- text-transform: uppercase;
- color: #93c5fd;
- font-weight: 700;
- margin-bottom: 1rem;
- }
- .cover h1 {
- margin: 0;
- font-size: clamp(1.6rem, 4vw, 2.1rem);
- font-weight: 700;
- line-height: 1.15;
- }
- .cover-subtitle {
- margin: 0.5rem 0 0;
- color: #cbd5e1;
- font-size: 1rem;
- }
- .cover-meta {
- display: grid;
- grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
- gap: 0.75rem 1.5rem;
- margin-top: 1.75rem;
- padding-top: 1.25rem;
- border-top: 1px solid rgba(255,255,255,0.12);
- font-size: 0.82rem;
- }
- .cover-meta dt { color: #94a3b8; margin: 0 0 0.15rem; font-weight: 500; }
- .cover-meta dd { margin: 0; color: #f1f5f9; font-weight: 600; }
- .content { padding: 2rem 2.75rem 2.5rem; }
- .hero-score {
- display: flex;
- flex-wrap: wrap;
- align-items: center;
- gap: 1.25rem 2rem;
- padding: 1.25rem 1.5rem;
- border: 1px solid var(--line);
- border-radius: 12px;
- background: var(--surface-muted);
- margin-bottom: 1.75rem;
- }
- .hero-score-ring {
- width: 88px;
- height: 88px;
- border-radius: 50%;
- display: grid;
- place-items: center;
- font-size: 1.65rem;
- font-weight: 800;
- border: 4px solid currentColor;
- flex-shrink: 0;
- }
- .hero-score-ring.score-good { color: var(--good); background: var(--good-bg); }
- .hero-score-ring.score-fair { color: var(--fair); background: var(--fair-bg); }
- .hero-score-ring.score-poor { color: var(--poor); background: var(--poor-bg); }
- .hero-score-ring.score-na { color: var(--muted); background: #f1f5f9; border-color: #cbd5e1; }
- .hero-score-copy h2 { margin: 0 0 0.35rem; font-size: 1.05rem; }
- .hero-score-copy p { margin: 0; color: var(--muted); font-size: 0.92rem; }
- .stats-row {
- display: grid;
- grid-template-columns: repeat(4, minmax(0, 1fr));
- gap: 0.75rem;
- margin-bottom: 1.75rem;
- }
- .stat {
- border: 1px solid var(--line);
- border-radius: 10px;
- padding: 0.85rem 0.75rem;
- text-align: center;
- background: var(--surface);
- }
- .stat-value { display: block; font-size: 1.35rem; font-weight: 800; line-height: 1.1; }
- .stat-label {
- display: block;
- margin-top: 0.25rem;
- font-size: 0.72rem;
- text-transform: uppercase;
- letter-spacing: 0.06em;
- color: var(--muted);
- font-weight: 600;
- }
- .stat-critical .stat-value { color: var(--critical); }
- .stat-high .stat-value { color: var(--high); }
- .stat-medium .stat-value { color: var(--medium); }
- .stat-low .stat-value { color: var(--low); }
- .score-grid {
- display: grid;
- grid-template-columns: repeat(auto-fill, minmax(140px, 1fr));
- gap: 0.75rem;
- margin-bottom: 1.75rem;
- }
- .score-card {
- border: 1px solid var(--line);
- border-radius: 10px;
- padding: 0.9rem 0.75rem;
- background: var(--surface);
- }
- .score-card .score-value { font-size: 1.5rem; font-weight: 800; line-height: 1; }
- .score-card .score-name { margin-top: 0.45rem; font-size: 0.78rem; font-weight: 600; line-height: 1.25; }
- .score-card .score-meta { margin-top: 0.25rem; font-size: 0.72rem; color: var(--muted); }
- .score-card.score-good .score-value { color: var(--good); }
- .score-card.score-fair .score-value { color: var(--fair); }
- .score-card.score-poor .score-value { color: var(--poor); }
- .score-card.score-na .score-value { color: var(--muted); }
- section { margin-bottom: 2rem; page-break-inside: avoid; }
- section h2 {
- margin: 0 0 0.85rem;
- font-size: 1rem;
- font-weight: 700;
- letter-spacing: 0.02em;
- text-transform: uppercase;
- color: var(--ink);
- padding-bottom: 0.45rem;
- border-bottom: 2px solid var(--brand);
- }
- .callout {
- border-left: 4px solid var(--brand-accent);
- background: #eff6ff;
- padding: 1rem 1.15rem;
- border-radius: 0 10px 10px 0;
- margin-bottom: 0.5rem;
- }
- .callout ul { margin: 0; padding-left: 1.15rem; }
- .callout li { margin: 0.35rem 0; }
- table.data {
- width: 100%;
- border-collapse: collapse;
- font-size: 0.84rem;
- border: 1px solid var(--line);
- border-radius: 10px;
- overflow: hidden;
- }
- table.data th,
- table.data td {
- padding: 0.55rem 0.65rem;
- text-align: left;
- vertical-align: top;
- border-bottom: 1px solid var(--line);
- }
- table.data th {
- background: var(--surface-muted);
- font-size: 0.72rem;
- text-transform: uppercase;
- letter-spacing: 0.05em;
- color: var(--muted);
- font-weight: 700;
- }
- table.data tbody tr:last-child td { border-bottom: none; }
- table.data tbody tr:nth-child(even) td { background: #fcfdff; }
- table.kv th {
- width: 34%;
- font-weight: 600;
- color: var(--ink);
- background: var(--surface-muted);
- }
- .url { word-break: break-all; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.78rem; }
- .badge {
- display: inline-block;
- padding: 0.15rem 0.5rem;
- border-radius: 999px;
- font-size: 0.68rem;
- font-weight: 700;
- text-transform: uppercase;
- letter-spacing: 0.04em;
- }
- .badge-critical { background: #fee2e2; color: var(--critical); }
- .badge-high { background: #ffedd5; color: var(--high); }
- .badge-medium { background: #fef3c7; color: var(--medium); }
- .badge-low { background: #f1f5f9; color: var(--low); }
- .muted { color: var(--muted); font-size: 0.86rem; margin: 0.35rem 0 0.75rem; }
- .report-footer {
- border-top: 1px solid var(--line);
- padding: 1.25rem 2.75rem 2rem;
- color: var(--muted);
- font-size: 0.78rem;
- line-height: 1.5;
- }
- @media print {
- body { background: #fff; }
- .report { max-width: none; }
- .cover { -webkit-print-color-adjust: exact; print-color-adjust: exact; }
- .content { padding: 1.2cm 1.4cm; }
- section { page-break-inside: auto; }
- table.data { page-break-inside: auto; }
- table.data tr { page-break-inside: avoid; }
- .report-footer { padding-left: 1.4cm; padding-right: 1.4cm; }
- }
- @media (max-width: 640px) {
- .cover, .content, .report-footer { padding-left: 1.25rem; padding-right: 1.25rem; }
- .stats-row { grid-template-columns: repeat(2, minmax(0, 1fr)); }
- }
-"""
-
-
def export_audit_csv(report_id: Optional[int] = None) -> str:
payload = _load_payload(report_id)
buf = io.StringIO()
diff --git a/src/website_profiling/tools/export_audit_data.py b/src/website_profiling/tools/export_audit_data.py
new file mode 100644
index 00000000..56974a47
--- /dev/null
+++ b/src/website_profiling/tools/export_audit_data.py
@@ -0,0 +1,216 @@
+"""Audit export data helpers."""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from ..reporting.terminology import category_display_name
+
+_GLOSSARY_ROWS: list[tuple[str, str]] = [
+ ("Crawl", "URLs fetched by the site spider (status codes, titles, inlinks)."),
+ ("Lighthouse", "Lab Core Web Vitals audit (LCP, CLS, TBT, and category scores)."),
+ ("Google Search Console", "Queries, pages, clicks, impressions, and average position from GSC."),
+ ("Google Analytics 4", "Sessions, users, and engagement from GA4."),
+ ("Estimated", "Derived from crawl text only — not Google search volume or rankings."),
+ ("AI insights", "Optional LLM summaries — verify before client delivery."),
+]
+
+_ISSUE_LIMIT_HTML = 200
+_ISSUE_LIMIT_PDF = 80
+_LINK_LIMIT = 50
+
+
+def _issue_recommendation(issue: dict[str, Any]) -> tuple[str, str]:
+ """Return (display recommendation, llm_recommendation if distinct)."""
+ rule = str(issue.get("recommendation") or "").strip()
+ llm = str(issue.get("llm_recommendation") or "").strip()
+ if llm and llm != rule:
+ display = llm if llm else rule
+ return display, llm
+ return llm or rule, llm
+
+
+def _issues_rows(payload: dict[str, Any]) -> list[dict[str, str]]:
+ rows: list[dict[str, str]] = []
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ cat_name = str(cat.get("name") or "")
+ ui_name = category_display_name(cat_name)
+ for issue in cat.get("issues") or []:
+ if not isinstance(issue, dict):
+ continue
+ rec, llm_rec = _issue_recommendation(issue)
+ rows.append({
+ "category": ui_name,
+ "priority": str(issue.get("priority") or ""),
+ "message": str(issue.get("message") or ""),
+ "url": str(issue.get("url") or ""),
+ "recommendation": rec,
+ "llm_recommendation": llm_rec,
+ })
+ return rows
+
+
+def _executive_export_data(payload: dict[str, Any]) -> dict[str, Any]:
+ """Normalize executive_summary and legacy recommendations for export."""
+ exec_sum = payload.get("executive_summary")
+ summary = ""
+ priorities: list[str] = []
+ top_issues: list[dict[str, Any]] = []
+ source = ""
+ if isinstance(exec_sum, dict):
+ summary = str(exec_sum.get("summary") or "").strip()
+ source = str(exec_sum.get("source") or "").strip()
+ raw_pri = exec_sum.get("priorities") or []
+ if isinstance(raw_pri, list):
+ priorities = [str(p).strip() for p in raw_pri if str(p).strip()]
+ raw_top = exec_sum.get("top_issues") or []
+ if isinstance(raw_top, list):
+ top_issues = [i for i in raw_top if isinstance(i, dict)][:8]
+
+ legacy_recs = payload.get("recommendations") or []
+ legacy_list: list[str] = []
+ if isinstance(legacy_recs, list):
+ legacy_list = [str(r).strip() for r in legacy_recs if str(r).strip()]
+
+ if not summary and legacy_list:
+ summary = "\n".join(f"• {r}" for r in legacy_list[:12])
+
+ return {
+ "summary": summary,
+ "priorities": priorities,
+ "top_issues": top_issues,
+ "source": source,
+ "legacy_recommendations": legacy_list,
+ }
+
+
+def _executive_source_label(source: str) -> str:
+ if source == "ai_insights":
+ return "AI insights"
+ if source == "deterministic":
+ return "Measured + Search Console"
+ return source or "Audit data"
+
+def _priority_sort_key(row: dict[str, str]) -> int:
+ order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+ return order.get(row["priority"].lower(), 9)
+
+
+def _summary_lines(payload: dict[str, Any]) -> list[tuple[str, str]]:
+ lines: list[tuple[str, str]] = []
+ site = str(payload.get("site_name") or "Site")
+ lines.append(("Property", site))
+ if payload.get("report_generated_at"):
+ lines.append(("Report generated", str(payload["report_generated_at"])))
+ meta = payload.get("report_meta") or {}
+ if isinstance(meta, dict):
+ sources = meta.get("data_sources") or []
+ if sources:
+ lines.append(("Data sources", ", ".join(str(s) for s in sources)))
+ scope = meta.get("crawl_scope") or {}
+ if isinstance(scope, dict) and scope.get("pages_crawled") is not None:
+ pages = scope.get("pages_crawled")
+ max_p = scope.get("max_pages_configured")
+ scope_txt = f"{pages} pages crawled"
+ if max_p:
+ scope_txt += f" (limit {max_p})"
+ if scope.get("crawl_limited"):
+ scope_txt += " — crawl limit reached"
+ render_mode = scope.get("render_mode")
+ if render_mode == "javascript":
+ js_c = scope.get("js_concurrency")
+ scope_txt += " — JavaScript rendering"
+ if js_c:
+ scope_txt += f" ({js_c} parallel pages)"
+ elif render_mode == "auto":
+ scope_txt += " — auto rendering (static + JS fallback)"
+ ps = scope.get("pages_static")
+ pr = scope.get("pages_rendered")
+ if ps is not None and pr is not None:
+ scope_txt += f" ({ps} static, {pr} JavaScript-rendered)"
+ elif scope.get("static_html_only"):
+ scope_txt += " — static HTML only"
+ lines.append(("Crawl scope", scope_txt))
+ browser_diag = scope.get("browser_diagnostics")
+ if isinstance(browser_diag, dict):
+ pce = browser_diag.get("pages_with_console_errors")
+ tce = browser_diag.get("total_console_errors")
+ ppe = browser_diag.get("pages_with_page_errors")
+ if pce or ppe:
+ parts = []
+ if pce:
+ parts.append(f"{pce} page(s) with console errors ({tce or 0} total)")
+ if ppe:
+ parts.append(f"{ppe} page(s) with uncaught JS errors")
+ lines.append(("Browser diagnostics", "; ".join(parts)))
+ if meta.get("google_fetched_at"):
+ lines.append(("Google data fetched", str(meta["google_fetched_at"])))
+ summary = payload.get("summary") or {}
+ if isinstance(summary, dict):
+ for key, label in (
+ ("total_urls", "URLs in crawl"),
+ ("indexable", "Indexable URLs"),
+ ("issues_count", "Total issues"),
+ ("critical_issues", "Critical issues"),
+ ):
+ if summary.get(key) is not None:
+ lines.append((label, str(summary[key])))
+ status = payload.get("status_counts") or {}
+ if isinstance(status, dict) and status:
+ parts = [f"{k}: {v}" for k, v in sorted(status.items(), key=lambda x: -int(x[1] or 0))[:8]]
+ lines.append(("HTTP status mix", ", ".join(parts)))
+ return lines
+
+
+def _format_report_date(value: str) -> str:
+ if not value:
+ return "—"
+ try:
+ dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=timezone.utc)
+ return dt.astimezone(timezone.utc).strftime("%d %B %Y, %H:%M UTC")
+ except ValueError:
+ return value
+
+
+def _overall_score(payload: dict[str, Any]) -> Optional[int]:
+ scores: list[float] = []
+ for cat in payload.get("categories") or []:
+ if not isinstance(cat, dict):
+ continue
+ raw = cat.get("score")
+ if raw is None:
+ continue
+ try:
+ scores.append(float(raw))
+ except (TypeError, ValueError):
+ continue
+ if not scores:
+ return None
+ return int(round(sum(scores) / len(scores)))
+
+
+def _score_band(score: Optional[float]) -> tuple[str, str]:
+ if score is None:
+ return "—", "score-na"
+ rounded = int(round(score))
+ if rounded >= 80:
+ return str(rounded), "score-good"
+ if rounded >= 60:
+ return str(rounded), "score-fair"
+ return str(rounded), "score-poor"
+
+
+def _issue_priority_counts(rows: list[dict[str, str]]) -> dict[str, int]:
+ counts = {"critical": 0, "high": 0, "medium": 0, "low": 0}
+ for row in rows:
+ key = row["priority"].lower()
+ if key in counts:
+ counts[key] += 1
+ return counts
+
+
diff --git a/src/website_profiling/tools/export_audit_html.py b/src/website_profiling/tools/export_audit_html.py
new file mode 100644
index 00000000..2232e6b2
--- /dev/null
+++ b/src/website_profiling/tools/export_audit_html.py
@@ -0,0 +1,343 @@
+"""Audit export HTML generation."""
+from __future__ import annotations
+
+import html
+from typing import Any, Optional
+
+from ..reporting.terminology import category_display_name
+from .export_audit_data import (
+ _GLOSSARY_ROWS,
+ _ISSUE_LIMIT_HTML,
+ _ISSUE_LIMIT_PDF,
+ _LINK_LIMIT,
+ _executive_export_data,
+ _executive_source_label,
+ _format_report_date,
+ _issue_priority_counts,
+ _issues_rows,
+ _overall_score,
+ _priority_sort_key,
+ _score_band,
+ _summary_lines,
+)
+
+def _executive_summary_html(payload: dict[str, Any]) -> str:
+ data = _executive_export_data(payload)
+ if not data["summary"] and not data["priorities"] and not data["top_issues"]:
+ return ""
+
+ parts: list[str] = ['Executive summary ']
+ if data["source"]:
+ parts.append(
+ f'Source: {html.escape(_executive_source_label(data["source"]))}
'
+ )
+ if data["summary"]:
+ summary_html = html.escape(data["summary"]).replace("\n", "{html.escape(p)} " for p in data["priorities"][:8])
+ parts.append(f"Priorities ")
+
+ if data["top_issues"]:
+ rows = ""
+ for iss in data["top_issues"]:
+ pri = str(iss.get("priority") or "").lower()
+ badge_cls = f"badge-{pri}" if pri in {"critical", "high", "medium", "low"} else "badge-low"
+ clicks = iss.get("gsc_clicks")
+ clicks_txt = ""
+ if clicks is not None:
+ try:
+ if float(clicks) > 0:
+ clicks_txt = f' · {int(float(clicks))} GSC clicks'
+ except (TypeError, ValueError):
+ pass
+ rows += (
+ ""
+ f"{html.escape(str(iss.get('priority') or ''))} {html.escape(str(iss.get('message') or ''))} "
+ f"{html.escape(str(iss.get('url') or ''))} "
+ f"{html.escape(clicks_txt.lstrip(' · ') if clicks_txt else '—')} "
+ " "
+ )
+ parts.append(
+ "Top traffic-impacting issues "
+ ''
+ "Priority Issue URL GSC clicks "
+ f" {rows}
"
+ )
+
+ parts.append("'
+ f'{score_txt}
'
+ f'{name}
'
+ f'{issue_n} issue{"s" if issue_n != 1 else ""}
'
+ f" "
+ )
+ return "".join(cards) or 'No category scores available.
'
+
+
+def _priority_stats_html(counts: dict[str, int]) -> str:
+ labels = (
+ ("critical", "Critical"),
+ ("high", "High"),
+ ("medium", "Medium"),
+ ("low", "Low"),
+ )
+ parts: list[str] = []
+ for key, label in labels:
+ n = counts.get(key, 0)
+ parts.append(
+ f''
+ f'{n} '
+ f'{label} '
+ f"
"
+ )
+ return "".join(parts)
+
+
+def _report_html_styles() -> str:
+ return """
+ :root {
+ --ink: #0f172a;
+ --muted: #64748b;
+ --line: #e2e8f0;
+ --surface: #ffffff;
+ --surface-muted: #f8fafc;
+ --brand: #0b0f19;
+ --brand-accent: #2563eb;
+ --good: #059669;
+ --good-bg: #ecfdf5;
+ --fair: #d97706;
+ --fair-bg: #fffbeb;
+ --poor: #dc2626;
+ --poor-bg: #fef2f2;
+ --critical: #991b1b;
+ --high: #c2410c;
+ --medium: #a16207;
+ --low: #475569;
+ }
+ * { box-sizing: border-box; }
+ body {
+ margin: 0;
+ background: #eef2f7;
+ color: var(--ink);
+ font: 400 15px/1.55 "Segoe UI", system-ui, -apple-system, sans-serif;
+ }
+ .report { max-width: 920px; margin: 0 auto; background: var(--surface); }
+ .cover {
+ background: linear-gradient(135deg, #0b0f19 0%, #111827 55%, #1e3a5f 100%);
+ color: #f8fafc;
+ padding: 2.5rem 2.75rem 2rem;
+ }
+ .cover-brand {
+ font-size: 0.72rem;
+ letter-spacing: 0.14em;
+ text-transform: uppercase;
+ color: #93c5fd;
+ font-weight: 700;
+ margin-bottom: 1rem;
+ }
+ .cover h1 {
+ margin: 0;
+ font-size: clamp(1.6rem, 4vw, 2.1rem);
+ font-weight: 700;
+ line-height: 1.15;
+ }
+ .cover-subtitle {
+ margin: 0.5rem 0 0;
+ color: #cbd5e1;
+ font-size: 1rem;
+ }
+ .cover-meta {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+ gap: 0.75rem 1.5rem;
+ margin-top: 1.75rem;
+ padding-top: 1.25rem;
+ border-top: 1px solid rgba(255,255,255,0.12);
+ font-size: 0.82rem;
+ }
+ .cover-meta dt { color: #94a3b8; margin: 0 0 0.15rem; font-weight: 500; }
+ .cover-meta dd { margin: 0; color: #f1f5f9; font-weight: 600; }
+ .content { padding: 2rem 2.75rem 2.5rem; }
+ .hero-score {
+ display: flex;
+ flex-wrap: wrap;
+ align-items: center;
+ gap: 1.25rem 2rem;
+ padding: 1.25rem 1.5rem;
+ border: 1px solid var(--line);
+ border-radius: 12px;
+ background: var(--surface-muted);
+ margin-bottom: 1.75rem;
+ }
+ .hero-score-ring {
+ width: 88px;
+ height: 88px;
+ border-radius: 50%;
+ display: grid;
+ place-items: center;
+ font-size: 1.65rem;
+ font-weight: 800;
+ border: 4px solid currentColor;
+ flex-shrink: 0;
+ }
+ .hero-score-ring.score-good { color: var(--good); background: var(--good-bg); }
+ .hero-score-ring.score-fair { color: var(--fair); background: var(--fair-bg); }
+ .hero-score-ring.score-poor { color: var(--poor); background: var(--poor-bg); }
+ .hero-score-ring.score-na { color: var(--muted); background: #f1f5f9; border-color: #cbd5e1; }
+ .hero-score-copy h2 { margin: 0 0 0.35rem; font-size: 1.05rem; }
+ .hero-score-copy p { margin: 0; color: var(--muted); font-size: 0.92rem; }
+ .stats-row {
+ display: grid;
+ grid-template-columns: repeat(4, minmax(0, 1fr));
+ gap: 0.75rem;
+ margin-bottom: 1.75rem;
+ }
+ .stat {
+ border: 1px solid var(--line);
+ border-radius: 10px;
+ padding: 0.85rem 0.75rem;
+ text-align: center;
+ background: var(--surface);
+ }
+ .stat-value { display: block; font-size: 1.35rem; font-weight: 800; line-height: 1.1; }
+ .stat-label {
+ display: block;
+ margin-top: 0.25rem;
+ font-size: 0.72rem;
+ text-transform: uppercase;
+ letter-spacing: 0.06em;
+ color: var(--muted);
+ font-weight: 600;
+ }
+ .stat-critical .stat-value { color: var(--critical); }
+ .stat-high .stat-value { color: var(--high); }
+ .stat-medium .stat-value { color: var(--medium); }
+ .stat-low .stat-value { color: var(--low); }
+ .score-grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fill, minmax(140px, 1fr));
+ gap: 0.75rem;
+ margin-bottom: 1.75rem;
+ }
+ .score-card {
+ border: 1px solid var(--line);
+ border-radius: 10px;
+ padding: 0.9rem 0.75rem;
+ background: var(--surface);
+ }
+ .score-card .score-value { font-size: 1.5rem; font-weight: 800; line-height: 1; }
+ .score-card .score-name { margin-top: 0.45rem; font-size: 0.78rem; font-weight: 600; line-height: 1.25; }
+ .score-card .score-meta { margin-top: 0.25rem; font-size: 0.72rem; color: var(--muted); }
+ .score-card.score-good .score-value { color: var(--good); }
+ .score-card.score-fair .score-value { color: var(--fair); }
+ .score-card.score-poor .score-value { color: var(--poor); }
+ .score-card.score-na .score-value { color: var(--muted); }
+ section { margin-bottom: 2rem; page-break-inside: avoid; }
+ section h2 {
+ margin: 0 0 0.85rem;
+ font-size: 1rem;
+ font-weight: 700;
+ letter-spacing: 0.02em;
+ text-transform: uppercase;
+ color: var(--ink);
+ padding-bottom: 0.45rem;
+ border-bottom: 2px solid var(--brand);
+ }
+ .callout {
+ border-left: 4px solid var(--brand-accent);
+ background: #eff6ff;
+ padding: 1rem 1.15rem;
+ border-radius: 0 10px 10px 0;
+ margin-bottom: 0.5rem;
+ }
+ .callout ul { margin: 0; padding-left: 1.15rem; }
+ .callout li { margin: 0.35rem 0; }
+ table.data {
+ width: 100%;
+ border-collapse: collapse;
+ font-size: 0.84rem;
+ border: 1px solid var(--line);
+ border-radius: 10px;
+ overflow: hidden;
+ }
+ table.data th,
+ table.data td {
+ padding: 0.55rem 0.65rem;
+ text-align: left;
+ vertical-align: top;
+ border-bottom: 1px solid var(--line);
+ }
+ table.data th {
+ background: var(--surface-muted);
+ font-size: 0.72rem;
+ text-transform: uppercase;
+ letter-spacing: 0.05em;
+ color: var(--muted);
+ font-weight: 700;
+ }
+ table.data tbody tr:last-child td { border-bottom: none; }
+ table.data tbody tr:nth-child(even) td { background: #fcfdff; }
+ table.kv th {
+ width: 34%;
+ font-weight: 600;
+ color: var(--ink);
+ background: var(--surface-muted);
+ }
+ .url { word-break: break-all; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.78rem; }
+ .badge {
+ display: inline-block;
+ padding: 0.15rem 0.5rem;
+ border-radius: 999px;
+ font-size: 0.68rem;
+ font-weight: 700;
+ text-transform: uppercase;
+ letter-spacing: 0.04em;
+ }
+ .badge-critical { background: #fee2e2; color: var(--critical); }
+ .badge-high { background: #ffedd5; color: var(--high); }
+ .badge-medium { background: #fef3c7; color: var(--medium); }
+ .badge-low { background: #f1f5f9; color: var(--low); }
+ .muted { color: var(--muted); font-size: 0.86rem; margin: 0.35rem 0 0.75rem; }
+ .report-footer {
+ border-top: 1px solid var(--line);
+ padding: 1.25rem 2.75rem 2rem;
+ color: var(--muted);
+ font-size: 0.78rem;
+ line-height: 1.5;
+ }
+ @media print {
+ body { background: #fff; }
+ .report { max-width: none; }
+ .cover { -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+ .content { padding: 1.2cm 1.4cm; }
+ section { page-break-inside: auto; }
+ table.data { page-break-inside: auto; }
+ table.data tr { page-break-inside: avoid; }
+ .report-footer { padding-left: 1.4cm; padding-right: 1.4cm; }
+ }
+ @media (max-width: 640px) {
+ .cover, .content, .report-footer { padding-left: 1.25rem; padding-right: 1.25rem; }
+ .stats-row { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+ }
+"""
diff --git a/tests/test_audit_tools_batch100_coverage.py b/tests/test_audit_tools_batch100_coverage.py
new file mode 100644
index 00000000..27572834
--- /dev/null
+++ b/tests/test_audit_tools_batch100_coverage.py
@@ -0,0 +1,1686 @@
+"""100% line-coverage tests for batch-100 audit list tool modules."""
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+import requests
+
+from website_profiling.tools.audit_tools import crawl as crawl_mod
+from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx
+from website_profiling.tools.audit_tools import issue_lists as issue_mod
+from website_profiling.tools.audit_tools import google_lists as google_mod
+from website_profiling.tools.audit_tools import keyword_lists as kw_mod
+from website_profiling.tools.audit_tools import backlink_lists as bl_mod
+from website_profiling.tools.audit_tools import content_lists as content_mod
+from website_profiling.tools.audit_tools import link_lists as link_mod
+from website_profiling.tools.audit_tools import indexation_lists as idx_mod
+from website_profiling.tools.audit_tools import compare_list_tools as cmp_mod
+from website_profiling.tools.audit_tools import geo_list_tools as geo_list_mod
+
+
+@pytest.fixture
+def conn() -> MagicMock:
+ return MagicMock()
+
+
+@pytest.fixture
+def ctx() -> Ctx:
+ return Ctx(property_id=1, report_id=1)
+
+
+def _payload() -> dict:
+ return {
+ "start_url": "https://ex.com/",
+ "origin": "https://ex.com/",
+ "orphan_urls": ["https://ex.com/orphan", "https://ex.com/hub-target"],
+ "top_pages": [
+ {"url": "https://ex.com/", "inlinks": 5, "pagerank": 0.05, "outlinks": 2},
+ {"url": "https://ex.com/blog/post", "inlinks": 1, "pagerank": 0.001, "outlinks": 1},
+ ],
+ "links": [{"url": "https://ex.com/orphan", "inlinks": 0}],
+ "graph_nodes": [
+ {"url": "https://ex.com/hub", "id": "hub"},
+ "https://ex.com/orphan",
+ ],
+ "graph_edges": [
+ [0, 1],
+ {"source": "https://ex.com/hub", "target": "https://ex.com/orphan"},
+ ],
+ "link_edges": [
+ {
+ "from_url": "https://ex.com/",
+ "to_url": "https://external.com/page",
+ "link_type": "external",
+ "anchor_text": "ext",
+ "rel": "nofollow",
+ "is_nofollow": True,
+ },
+ {
+ "from_url": "https://ex.com/",
+ "to_url": "https://ex.com/about",
+ "link_type": "internal",
+ "anchor_text": "about",
+ "rel": "",
+ "is_nofollow": False,
+ },
+ {
+ "from_url": "https://ex.com/about",
+ "to_url": "https://ex.com/",
+ "link_type": "internal",
+ "anchor_text": "home",
+ "rel": "ugc nofollow",
+ "is_nofollow": True,
+ },
+ ],
+ "content_urls": {
+ "title_short": [{"url": "https://ex.com/short", "title": "Hi", "title_length": 2}],
+ "title_long": [{"url": "https://ex.com/long", "title": "x" * 80, "title_length": 80}],
+ "slow_response": [{"url": "https://ex.com/slow", "response_time_ms": 5000, "status": "200"}],
+ "missing_html_lang": [{"url": "https://ex.com/nolang"}],
+ "invalid_viewport": [{"url": "https://ex.com/badvp"}],
+ "high_reading_level": [{"url": "https://ex.com/hard", "reading_level": 14}],
+ "very_thin_content": [{"url": "https://ex.com/thin", "word_count": 50}],
+ },
+ "hreflang_issue_urls": [{"url": "https://ex.com/href", "message": "missing return"}],
+ "lighthouse_failure_urls": {
+ "lcp": ["https://ex.com/lcp-fail"],
+ "inp": ["https://ex.com/inp-fail"],
+ "cls": ["https://ex.com/cls-fail"],
+ "seo": ["https://ex.com/seo-fail"],
+ },
+ "optional_audit_urls": {
+ "spell": [{"url": "https://ex.com/spell", "message": "typo"}],
+ "html": [{"url": "https://ex.com/html", "message": "invalid html"}],
+ "amp": [{"url": "https://ex.com/amp", "message": "amp error"}],
+ "pagination": [{"url": "https://ex.com/pag", "message": "pagination rel=next"}],
+ },
+ "indexation_coverage": {
+ "lists": {
+ "sitemap_only": ["https://ex.com/sitemap-only", "https://ex.com/s2"],
+ "crawled_not_in_sitemap": ["https://ex.com/crawl-only"],
+ },
+ "lists_total": {"sitemap_only": 2, "crawled_not_in_sitemap": 1},
+ },
+ "social_coverage": {
+ "missing_og": ["https://ex.com/no-og"],
+ "missing_twitter": ["https://ex.com/no-tw"],
+ },
+ "language_summary": {"counts": {"en": 8, "fr": 2}},
+ "text_content_analysis": {
+ "keyword_index": [
+ {
+ "word": "widgets",
+ "top_pages": [
+ {"url": "https://ex.com/", "count": 3},
+ ["https://ex.com/about", 2],
+ "skip",
+ ],
+ }
+ ],
+ },
+ "content_analytics": {"keyword_index": [{"word": "fallback", "top_pages": []}]},
+ "content_duplicates": [
+ {
+ "id": "c1",
+ "representative_url": "https://ex.com/a",
+ "member_urls": ["https://ex.com/a", "https://ex.com/b"],
+ "similarity": 0.95,
+ },
+ "skip",
+ ],
+ "rich_results_validation": [
+ {"url": "https://ex.com/", "status": "pass", "type": "Organization"},
+ {"url": "https://ex.com/bad", "status": "fail", "schema_type": "FAQPage", "message": "invalid"},
+ ],
+ "semantic_keyword_clusters": [
+ {"top_keyword": "widgets", "keywords": ["widgets", "widget repair"], "cluster_score": 0.8},
+ ],
+ "issues": {
+ "seo": [
+ {"type": "missing_title", "url": "https://ex.com/x", "message": "Missing title"},
+ {"type": "OTHER", "url": "https://ex.com/y"},
+ "skip",
+ ],
+ },
+ "categories": [
+ {
+ "issues": [
+ {"message": "Spell check found typo on page"},
+ {"message": "HTML markup validation failed"},
+ {"message": "AMP validation issue detected"},
+ {"message": "pagination rel=prev missing on series"},
+ {"message": "custom audit needle found"},
+ ],
+ },
+ "skip",
+ ],
+ "lighthouse_by_url": {
+ "https://ex.com/lcp-live": {
+ "lcp": 4.5,
+ "cwv_failures": "lcp",
+ "top_failures": [{"id": "largest-contentful-paint"}],
+ "audits": {"lcp": {"score": 0.5, "title": "LCP slow"}},
+ },
+ "https://ex.com/inp-live": {
+ "inp": 500,
+ "top_failures": [{"id": "interaction-to-next-paint"}],
+ },
+ "https://ex.com/cls-live": {"cls": 0.3, "cwv_failures": "cls"},
+ "https://ex.com/seo-live": {"seo": 55, "audits": {"seo": {"score": 0.6, "title": "SEO"}}},
+ "https://ex.com/contrast": {
+ "audits": {
+ "color-contrast": {"score": 0.5, "title": "Contrast"},
+ },
+ },
+ },
+ "google": {
+ "fetched_at": "2026-06-01",
+ "gsc": {
+ "daily": [
+ {"query": "widgets", "date": "2026-06-01", "clicks": 1},
+ {"page": "https://ex.com/", "date": "2026-06-01", "clicks": 2},
+ ],
+ },
+ "gsc_full": {
+ "summary": {"clicks": 100, "impressions": 1000, "ctr": 0.1, "position": 5},
+ "pages": [
+ {"page": "https://ex.com/", "clicks": 50, "impressions": 500, "ctr": "0.5%", "position": 5},
+ {"page": "https://ex.com/low-ctr", "clicks": 1, "impressions": 800, "ctr": "0.1%", "position": 4},
+ {"page": "https://ex.com/high-ctr", "clicks": 100, "impressions": 500, "ctr": "20%", "position": 3},
+ {"page": "https://ex.com/band", "clicks": 5, "impressions": 100, "ctr": "5%", "position": 12},
+ ],
+ "queries": [
+ {"query": "widgets", "clicks": 10, "impressions": 200, "position": 6},
+ {"query": "new query", "clicks": 2, "impressions": 50, "position": 8},
+ ],
+ "daily": [
+ {"query": "widgets", "date": "2026-06-01", "clicks": 1},
+ {"page": "https://ex.com/", "date": "2026-06-01", "clicks": 2},
+ ],
+ },
+ "ga4_full": {
+ "summary": {"sessions": 200, "users": 150},
+ "top_pages": [
+ {"path": "/", "sessions": 100, "bounceRate": 0.8, "engagementRate": 0.2},
+ {"path": "/mismatch", "sessions": 50, "bounceRate": 0.9, "engagementRate": 0.1},
+ ],
+ "by_path": {
+ "/alt": {"sessions": 30, "bounceRate": 0.7, "engagementRate": 0.3},
+ },
+ "daily": [{"path": "/", "date": "2026-06-01", "sessions": 10}],
+ },
+ },
+ }
+
+
+def _prior_google() -> dict:
+ return {
+ "fetched_at": "2026-05-01",
+ "gsc_full": {
+ "summary": {"clicks": 80, "impressions": 900, "ctr": 0.09, "position": 6},
+ "pages": [
+ {"page": "https://ex.com/", "clicks": 60, "impressions": 400, "position": 4},
+ {"page": "https://ex.com/loser", "clicks": 30, "impressions": 200, "position": 5},
+ ],
+ "queries": [
+ {"query": "widgets", "clicks": 15, "impressions": 180, "position": 4},
+ {"query": "old query", "clicks": 5, "impressions": 40, "position": 12},
+ ],
+ },
+ "ga4_full": {"summary": {"sessions": 180, "users": 140}},
+ }
+
+
+def _google_mismatch() -> dict:
+ data = _payload()["google"].copy()
+ data["gsc"] = {
+ "daily": [
+ {"page": "https://ex.com/", "date": "2026-06-01", "clicks": 2},
+ {"page": "https://ex.com/mismatch-gsc", "date": "2026-06-01", "clicks": 1},
+ ],
+ }
+ gsc = dict(data["gsc_full"])
+ ga4 = dict(data["ga4_full"])
+ gsc["by_page"] = {
+ "https://ex.com/mismatch-gsc": {"clicks": 20, "impressions": 100},
+ "https://ex.com/ga4-only": {"clicks": 0, "impressions": 0},
+ }
+ ga4["by_path"] = {
+ "/mismatch-gsc": {"sessions": 0},
+ "/ga4-only": {"sessions": 25},
+ "/ratio-high": {"sessions": 60},
+ }
+ gsc["pages"] = list(gsc["pages"]) + [
+ {"page": "https://ex.com/mismatch-gsc", "clicks": 20, "impressions": 100, "position": 5},
+ {"page": "https://ex.com/ga4-only", "clicks": 0, "impressions": 0, "position": 10},
+ {"page": "https://ex.com/ratio-high", "clicks": 10, "impressions": 50, "position": 6},
+ ]
+ data["gsc_full"] = gsc
+ data["ga4_full"] = ga4
+ return data
+
+
+def _keyword_data() -> dict:
+ return {
+ "fetched_at": "2026-06-01",
+ "rows": [
+ {
+ "keyword": "widgets",
+ "gsc_position": 8,
+ "gsc_clicks": 0,
+ "gsc_impressions": 500,
+ "gsc_url": "https://ex.com/",
+ "recommended_action": "Improve CTR",
+ "serp_features": ["ai_overview", "faq"],
+ "serp_estimated_competition": 45,
+ "is_branded": True,
+ "is_question": True,
+ "intent": "commercial",
+ "score": 10,
+ "traffic_potential": 200,
+ },
+ {
+ "keyword": "repair service",
+ "gsc_position": 15,
+ "gsc_clicks": 0,
+ "gsc_impressions": 200,
+ "gsc_url": "https://ex.com/about",
+ "recommended_action": "Create content",
+ "serp_features": "local_pack",
+ "serp_estimated_competition": 70,
+ "is_branded": False,
+ "is_question": False,
+ "intent": "transactional",
+ },
+ {
+ "keyword": "near page one",
+ "gsc_position": 11,
+ "gsc_clicks": 2,
+ "gsc_impressions": 100,
+ "gsc_url": "https://ex.com/near",
+ },
+ {
+ "keyword": "bad position",
+ "gsc_position": "bad",
+ "gsc_clicks": 1,
+ "gsc_impressions": 10,
+ },
+ ],
+ "cannibalisation": [
+ {
+ "query": "widgets",
+ "pages": [
+ {"url": "https://ex.com/a", "position": 5, "clicks": 3, "impressions": 50},
+ {"url": "https://ex.com/b", "position": 8, "clicks": 1, "impressions": 20},
+ "skip",
+ ],
+ },
+ "skip",
+ ],
+ "query_page_misalignment": [{"keyword": "buy widgets", "url": "https://ex.com/wrong"}],
+ "semantic_keyword_clusters": [{"top_keyword": "widgets", "keywords": ["widgets"]}],
+ }
+
+
+def _prior_keywords() -> dict:
+ return {
+ "rows": [
+ {"keyword": "widgets", "gsc_position": 12, "gsc_impressions": 150},
+ {"keyword": "repair service", "gsc_position": 8, "gsc_impressions": 100},
+ {"keyword": "old top ten", "gsc_position": 9, "gsc_impressions": 80},
+ {"keyword": "fell out", "gsc_position": 5, "gsc_impressions": 60},
+ ],
+ }
+
+
+def _gsc_links() -> dict:
+ return {
+ "top_linking_sites": [{"site": "partner.com", "link_count": 5}],
+ "top_linking_text": [{"anchor_text": "widgets", "link_count": 3}],
+ "sample_links": [
+ {
+ "linking_site": "partner.com",
+ "source_page": "https://partner.com/page",
+ "target_page": "https://ex.com/",
+ "target_url_on_linking_page": "https://ex.com/",
+ "anchor_text": "Best widgets",
+ },
+ {
+ "source_page": "https://www.other.org/ref",
+ "target_page": "https://ex.com/about",
+ "anchor_text": "About us",
+ },
+ ],
+ "latest_links": [
+ {
+ "linking_site": "fresh.com",
+ "target_page": "https://ex.com/new",
+ "anchor_text": "fresh link",
+ },
+ ],
+ }
+
+
+def _crawl_df() -> pd.DataFrame:
+ return pd.DataFrame([
+ {
+ "url": "https://ex.com/",
+ "status": "200",
+ "title": "Home page with widgets",
+ "title_length": 25,
+ "meta_description": "desc",
+ "h1": "Home",
+ "word_count": 400,
+ "reading_level": 8,
+ "response_time_ms": 100,
+ "viewport_present": "true",
+ "viewport_content": "width=device-width",
+ "og_title": "OG Home",
+ "twitter_card": "summary",
+ "has_schema": "true",
+ "detected_language": "en",
+ "fetch_method": "static",
+ "content_excerpt": "Widgets are tools that means something useful.",
+ "html": " item ",
+ "page_analysis": json.dumps({
+ "html_lang": "en",
+ "json_ld_types": ["Organization", "FAQPage"],
+ "hreflang_alternates": [
+ {"hreflang": "en", "href": "https://ex.com/other"},
+ {"hreflang": "fr", "href": "https://ex.com/fr"},
+ ],
+ "browser": {
+ "console": [{"type": "error", "text": "Uncaught Error"}],
+ "page_errors": [{"message": "ReferenceError"}],
+ "failed_requests": [{"url": "https://ex.com/missing.js"}],
+ },
+ }),
+ },
+ {
+ "url": "https://ex.com/",
+ "status": "200",
+ "title": "Home rendered",
+ "h1": "Home R",
+ "word_count": 500,
+ "fetch_method": "rendered",
+ "page_analysis": "{}",
+ },
+ {
+ "url": "https://ex.com/fr",
+ "status": "200",
+ "title": "French",
+ "detected_language": "fr",
+ "word_count": 300,
+ "page_analysis": json.dumps({
+ "hreflang_alternates": [{"hreflang": "fr", "href": "https://ex.com/fr"}],
+ }),
+ },
+ {
+ "url": "https://ex.com/other",
+ "status": "200",
+ "title": "Other EN",
+ "detected_language": "en",
+ "word_count": 250,
+ "page_analysis": json.dumps({
+ "hreflang_alternates": [{"hreflang": "en", "href": "https://ex.com/other"}],
+ }),
+ },
+ {
+ "url": "https://ex.com/short-crawl",
+ "status": "200",
+ "title": "Short",
+ "title_length": 5,
+ "word_count": 80,
+ "reading_level": "bad",
+ "response_time_ms": "bad",
+ "viewport_present": "true",
+ "viewport_content": "broken",
+ "og_title": "",
+ "twitter_card": "",
+ "has_schema": "true",
+ "page_analysis": json.dumps({"json_ld_types": []}),
+ },
+ {
+ "url": "https://ex.com/slow-crawl",
+ "status": "200",
+ "title": "Slow page title here",
+ "title_length": 35,
+ "response_time_ms": 3000,
+ "word_count": 60,
+ "reading_level": 13,
+ "viewport_present": "false",
+ "page_analysis": json.dumps({"html_lang": ""}),
+ },
+ {
+ "url": "https://ex.com/blog/how-to-fix-widgets",
+ "status": "200",
+ "title": "How to fix widgets step-by-step",
+ "h1": "How to fix widgets",
+ "word_count": 350,
+ "content_excerpt": "Posted by author on Monday. " + ("word " * 220),
+ "page_analysis": json.dumps({"json_ld_types": ["WebPage"]}),
+ },
+ {
+ "url": "https://ex.com/guide/tutorial",
+ "status": "200",
+ "title": "Tutorial guide",
+ "word_count": 280,
+ "content_excerpt": "- step one\n- step two\nWidgets are devices.",
+ "html": "",
+ "page_analysis": json.dumps({"json_ld_types": ["HowTo"]}),
+ },
+ {
+ "url": "https://ex.com/redirect",
+ "status": "301",
+ "redirect_chain_length": 4,
+ "final_url": "https://ex.com/final",
+ "page_analysis": "{}",
+ },
+ {
+ "url": "https://ex.com/href-dup",
+ "status": "200",
+ "page_analysis": json.dumps({
+ "hreflang_alternates": [
+ {"hreflang": "en", "href": "https://ex.com/href-dup"},
+ {"hreflang": "en", "href": "https://ex.com/href-dup-2"},
+ ],
+ }),
+ },
+ {
+ "url": "https://ex.com/href-self",
+ "status": "200",
+ "page_analysis": json.dumps({
+ "hreflang_alternates": [{"hreflang": "de", "href": "https://ex.com/de-only"}],
+ }),
+ },
+ {
+ "url": "https://ex.com/console",
+ "status": "200",
+ "page_analysis": json.dumps({
+ "console_errors": ["raw error", {"type": "warning", "message": "warn"}],
+ "js_errors": [{"level": "error", "text": "js fail"}],
+ }),
+ },
+ {
+ "url": "https://ex.com/404",
+ "status": "404",
+ "word_count": 10,
+ "page_analysis": "{}",
+ },
+ ])
+
+
+def _log_row() -> dict:
+ return {
+ "upload_id": 7,
+ "filename": "access.log",
+ "analysis": {
+ "top_paths": [
+ {"path": "/orphan", "hits": 50},
+ {"path": "/popular", "hits": 100},
+ {"path": "/quiet", "hits": 5},
+ ],
+ "googlebot_paths": [{"path": "/popular", "hits": 2}],
+ "paths_5xx": [],
+ "status_counts": {"500": 12, "503": 3},
+ },
+ }
+
+
+def _compare_current() -> dict:
+ return {
+ "report_generated_at": "2026-06-07",
+ "google": _google_mismatch(),
+ "categories": [{"issues": [{"message": "New issue", "url": "https://ex.com/new"}]}],
+ "lighthouse_by_url": {
+ "https://ex.com/": {"performance": 70, "seo": 80},
+ "https://ex.com/regressed": {"performance": 50, "seo": 60},
+ },
+ "links": [{"url": "https://ex.com/"}, {"url": "https://ex.com/new"}],
+ }
+
+
+def _compare_baseline() -> dict:
+ return {
+ "report_generated_at": "2026-05-01",
+ "google": _prior_google(),
+ "categories": [{"issues": [{"message": "Old issue", "url": "https://ex.com/old"}]}],
+ "lighthouse_by_url": {
+ "https://ex.com/": {"performance": 80, "seo": 85},
+ "https://ex.com/regressed": {"performance": 70, "seo": 75},
+ },
+ "links": [{"url": "https://ex.com/"}, {"url": "https://ex.com/removed"}],
+ }
+
+
+def test_issue_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ empty_df = pd.DataFrame()
+ with patch.object(Ctx, "load_payload", return_value=None), patch.object(Ctx, "load_crawl_df", return_value=empty_df):
+ assert issue_mod.list_pages_title_too_short(conn, ctx, {})["total"] == 0
+ assert issue_mod.list_pages_color_contrast_failures(conn, ctx, {})["error"]
+ assert issue_mod.list_orphan_hub_suggestions(conn, ctx, {})["error"]
+ assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["error"]
+
+ with patch.object(Ctx, "load_payload", return_value={"content_urls": "bad"}):
+ assert issue_mod.list_pages_title_too_short(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_payload", return_value={"content_urls": {"title_short": "bad"}}):
+ out = issue_mod.list_pages_title_too_short(conn, ctx, {})
+ assert out["total"] == 0
+
+ payload = _payload()
+ df = _crawl_df()
+ with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert issue_mod.list_pages_title_too_short(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_title_too_long(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_slow_response(conn, ctx, {"threshold_ms": "bad"})["total"] >= 1
+ assert issue_mod.list_pages_missing_html_lang(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_invalid_viewport(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_high_reading_level(conn, ctx, {"min_reading_level": "bad"})["total"] >= 1
+ assert issue_mod.list_pages_very_thin_content(conn, ctx, {"max_word_count": "bad"})["total"] >= 1
+ assert issue_mod.list_hreflang_issue_pages(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_missing_og_tags(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_missing_twitter_cards(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_invalid_json_ld(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_mixed_language(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_orphan_hub_suggestions(conn, ctx, {})["total"] >= 0
+ assert issue_mod.list_lighthouse_failure_lcp(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_lighthouse_failure_inp(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_lighthouse_failure_cls(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {"seo_threshold": "bad"})["total"] >= 1
+ contrast = issue_mod.list_pages_color_contrast_failures(conn, ctx, {})
+ assert contrast["total"] >= 0
+
+ empty_payload = {"hreflang_issue_urls": [], "issues": {"seo": "bad"}}
+ with patch.object(Ctx, "load_payload", return_value=empty_payload), patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert issue_mod.list_hreflang_issue_pages(conn, ctx, {})["total"] >= 1
+
+ no_bucket = {"content_urls": {}, "social_coverage": {}}
+ with patch.object(Ctx, "load_payload", return_value=no_bucket), patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert issue_mod.list_pages_title_too_short(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_missing_og_tags(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_missing_twitter_cards(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_very_thin_content(conn, ctx, {})["total"] >= 1
+
+ lh_payload = {
+ "lighthouse_failure_urls": {},
+ "lighthouse_by_url": {
+ "https://ex.com/x": {"seo": 50, "audits": {"seo": {"score": 0.5, "title": "SEO"}}},
+ "bad": "skip",
+ },
+ }
+ with patch.object(Ctx, "load_payload", return_value=lh_payload):
+ assert issue_mod.list_lighthouse_failure_lcp(conn, ctx, {})["total"] == 0
+ assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["total"] >= 1
+
+ no_lang = {"language_summary": {}}
+ with patch.object(Ctx, "load_payload", return_value=no_lang), patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert issue_mod.list_pages_mixed_language(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_payload", return_value={"orphan_urls": "bad", "graph_edges": []}), patch.object(
+ Ctx, "load_crawl_df", return_value=df,
+ ):
+ assert issue_mod.list_orphan_hub_suggestions(conn, ctx, {})["total"] == 0
+
+
+def test_google_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ with patch.object(Ctx, "load_google_full", return_value=None), patch.object(Ctx, "load_google", return_value=None):
+ assert google_mod.list_gsc_pages_by_impressions(conn, ctx, {})["missing"] is True
+ assert google_mod.get_gsc_query_trend(conn, ctx, {"query": "x"})["missing"] is True
+ assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com"})["missing"] is True
+ assert google_mod.get_ga4_path_trend(conn, ctx, {"path": "/"})["missing"] is True
+
+ assert google_mod.get_gsc_query_trend(conn, ctx, {})["error"] == "query is required"
+ assert google_mod.get_gsc_page_trend(conn, ctx, {})["error"] == "url is required"
+ assert google_mod.get_ga4_path_trend(conn, ctx, {})["error"] == "path is required"
+
+ google = _google_mismatch()
+ with patch.object(Ctx, "load_google_full", return_value=google), patch.object(Ctx, "load_google", return_value=google):
+ assert google_mod.list_gsc_pages_by_impressions(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_gsc_pages_by_clicks(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_gsc_queries_by_impressions(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_gsc_queries_by_clicks(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_gsc_ctr_underperformers(conn, ctx, {})["total"] >= 0
+ assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_ga4_pages_by_bounce_rate(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_ga4_pages_by_engagement_rate(conn, ctx, {})["total"] >= 1
+ trend = google_mod.get_gsc_query_trend(conn, ctx, {"query": "widgets"})
+ assert trend.get("daily") or trend.get("snapshot")
+ assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com/"})["daily"]
+ assert google_mod.get_ga4_path_trend(conn, ctx, {"url": "https://ex.com/alt"})["path"] == "/alt"
+ assert google_mod.list_gsc_ga4_mismatch_pages(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_gsc_pages_by_position_band(conn, ctx, {"min_position": "bad"})["total"] >= 1
+ assert google_mod.get_gsc_site_benchmarks(conn, ctx, {})["page_count"] >= 1
+
+ top_key = {"gsc_full": {"top_pages": [{"page": "https://ex.com/top", "impressions": 99}]}}
+ with patch.object(Ctx, "load_google_full", return_value=top_key):
+ assert google_mod.list_gsc_pages_by_impressions(conn, ctx, {})["total"] == 1
+
+ no_daily = {"gsc_full": {"queries": [{"query": "only", "clicks": 1}]}}
+ with patch.object(Ctx, "load_google_full", return_value=no_daily):
+ snap = google_mod.get_gsc_query_trend(conn, ctx, {"query": "only"})
+ assert snap.get("snapshot")
+
+ with patch.object(Ctx, "load_keywords", return_value=_keyword_data()):
+ assert google_mod.list_gsc_branded_queries(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_gsc_non_branded_queries(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert google_mod.list_gsc_branded_queries(conn, ctx, {})["missing"] is True
+
+ current, prior = _google_mismatch(), _prior_google()
+ with patch.object(google_mod, "_load_google_pair", return_value=(current, prior)):
+ assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["total"] >= 0
+ assert google_mod.list_gsc_decaying_queries(conn, ctx, {})["total"] >= 0
+ assert google_mod.list_gsc_new_queries(conn, ctx, {})["total"] >= 1
+ assert google_mod.compare_gsc_periods(conn, ctx, {})["gsc"]["clicks"]["delta"] != 0
+
+ with patch.object(google_mod, "_load_google_pair", return_value=(current, None)):
+ assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["missing"] is True
+ assert google_mod.compare_gsc_periods(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_google_pair", return_value=(current, None)), patch(
+ "website_profiling.integrations.google.store.read_prior_google_snapshot",
+ return_value=prior,
+ ):
+ cur, pr = google_mod._load_google_pair(ctx, conn)
+ assert pr is not None
+
+ with patch.object(Ctx, "load_google_pair", return_value=(current, None)), patch(
+ "website_profiling.integrations.google.store.read_prior_google_snapshot",
+ side_effect=RuntimeError("fail"),
+ ):
+ conn.execute.return_value.fetchall.return_value = [{"data": prior}, {"data": prior}]
+ cur2, pr2 = google_mod._load_google_pair(ctx, conn)
+ assert pr2 is not None
+
+ with patch.object(Ctx, "load_google_pair", return_value=(current, None)), patch(
+ "website_profiling.integrations.google.store.read_prior_google_snapshot",
+ side_effect=RuntimeError("fail"),
+ ), patch.object(conn, "execute", side_effect=RuntimeError("db fail")):
+ cur3, pr3 = google_mod._load_google_pair(ctx, conn)
+ assert pr3 is None
+
+ no_prop = Ctx(property_id=None, report_id=1)
+ with patch.object(Ctx, "load_google_pair", return_value=(current, prior)):
+ cur4, pr4 = google_mod._load_google_pair(no_prop, conn)
+ assert pr4 is prior
+
+
+def test_keyword_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ no_prop = Ctx(property_id=None, report_id=1)
+ assert kw_mod.list_keyword_rank_improvements(conn, no_prop, {})["missing"] is True
+ assert kw_mod.list_keywords_by_recommended_action(conn, ctx, {})["error"]
+ assert kw_mod.list_keywords_by_serp_feature(conn, ctx, {})["error"]
+ assert kw_mod.get_keyword_opportunity_score(conn, ctx, {})["error"] == "keyword is required"
+ assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {})["error"] == "keyword is required"
+ assert kw_mod.list_keywords_near_page_one(conn, ctx, {"min_position": "bad"})["error"]
+
+ kw = _keyword_data()
+ prior = _prior_keywords()
+ payload = _payload()
+ with patch.object(Ctx, "load_keywords", return_value=kw), patch.object(Ctx, "load_payload", return_value=payload), patch(
+ "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property",
+ return_value=[kw, prior],
+ ):
+ assert kw_mod.list_keyword_rank_improvements(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keyword_rank_declines(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keywords_new_to_top_10(conn, ctx, {})["total"] >= 0
+ assert kw_mod.list_keywords_fell_out_of_top_10(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_cannibalisation_queries(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_cannibalisation_urls(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_misaligned_queries(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keywords_by_recommended_action(conn, ctx, {"recommended_action": "Improve"})["total"] >= 1
+ assert kw_mod.list_keywords_by_serp_feature(conn, ctx, {"serp_feature": "local"})["total"] >= 1
+ assert kw_mod.list_semantic_cluster_queries(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_semantic_cluster_pages(conn, ctx, {})["total"] >= 1
+ assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "widgets"})["opportunity_score"] > 0
+ assert kw_mod.list_keywords_near_page_one(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keywords_high_impression_zero_click(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keywords_by_competition_band(conn, ctx, {})["total"] >= 1
+ assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {"keyword": "widgets"})["keyword"] == "widgets"
+ assert kw_mod.list_keywords_with_ai_overview(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keywords_local_pack(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keywords_question_intent(conn, ctx, {})["total"] >= 1
+ assert kw_mod.list_keywords_commercial_intent(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert kw_mod.list_cannibalisation_queries(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_keywords", return_value={"rows": []}), patch(
+ "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property",
+ return_value=[{"rows": []}],
+ ):
+ assert kw_mod.list_keyword_rank_improvements(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_keywords", return_value=kw), patch(
+ "website_profiling.integrations.google.keyword_store.read_keyword_snapshots_for_property",
+ return_value=[prior],
+ ):
+ assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "missing"})["missing"] is True
+
+ with patch.object(Ctx, "load_payload", return_value={}), patch.object(Ctx, "load_keywords", return_value={"semantic_keyword_clusters": []}):
+ assert kw_mod.list_semantic_cluster_queries(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "x", "serp_estimated_competition": "bad"}]}):
+ assert kw_mod.list_keywords_by_competition_band(conn, ctx, {})["total"] == 0
+
+
+def test_backlink_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ no_prop = Ctx(property_id=None, report_id=1)
+ assert bl_mod.list_referring_domains(conn, no_prop, {})["error"]
+ assert bl_mod.list_backlinks_to_url(conn, ctx, {})["error"] == "url is required"
+ assert bl_mod.list_backlinks_from_domain(conn, ctx, {})["error"] == "domain is required"
+
+ links = _gsc_links()
+ with patch.object(Ctx, "load_gsc_links", return_value=links):
+ assert bl_mod.list_referring_domains(conn, ctx, {})["total"] >= 1
+ assert bl_mod.list_backlinks_by_anchor_text(conn, ctx, {"anchor_text": "widget"})["total"] >= 1
+ assert bl_mod.list_backlinks_to_url(conn, ctx, {"url": "https://ex.com/"})["total"] >= 1
+ assert bl_mod.list_backlinks_from_domain(conn, ctx, {"domain": "partner.com"})["total"] >= 1
+ assert bl_mod.get_anchor_text_distribution(conn, ctx, {})["source"] == "top_linking_text"
+
+ derived = {
+ "sample_links": [
+ {"source_page": "https://www.example.com/x", "anchor_text": ""},
+ {"source_page": "bad://", "anchor_text": "x"},
+ ],
+ }
+ with patch.object(Ctx, "load_gsc_links", return_value=derived):
+ assert bl_mod.list_referring_domains(conn, ctx, {})["total"] >= 1
+ assert bl_mod.get_anchor_text_distribution(conn, ctx, {})["source"] == "sample_links"
+
+ with patch.object(Ctx, "load_gsc_links", return_value=None):
+ assert bl_mod.list_referring_domains(conn, ctx, {})["missing"] is True
+
+
+def test_content_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ payload = _payload()
+ df = _crawl_df()
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert content_mod.get_text_content_analysis(conn, ctx, {})["missing"] is True
+ assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "x"})["error"]
+
+ assert content_mod.list_pages_containing_keyword(conn, ctx, {})["error"] == "keyword is required"
+
+ with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert content_mod.get_text_content_analysis(conn, ctx, {})["missing"] is False
+ assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "widgets"})["total"] >= 1
+ assert content_mod.list_pages_by_word_count_band(conn, ctx, {"min_word_count": "bad"})["band"]["min_word_count"] == 0
+ assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_spell_check_issues(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_html_validation_issues(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_amp_validation_issues(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_pagination_issues(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_schema_errors_by_type(conn, ctx, {"schema_type": "faq"})["total"] >= 1
+ assert content_mod.list_pages_missing_article_schema(conn, ctx, {})["total"] >= 1
+
+ fallback_payload = {"content_analytics": {"keyword_index": [{"word": "repair", "top_pages": []}]}}
+ with patch.object(Ctx, "load_payload", return_value=fallback_payload):
+ assert content_mod.get_text_content_analysis(conn, ctx, {})["note"]
+
+ cat_payload = {
+ "optional_audit_urls": {},
+ "categories": [{"issues": [{"message": "custom audit needle found on page"}]}],
+ }
+ with patch.object(Ctx, "load_payload", return_value=cat_payload):
+ assert content_mod.list_spell_check_issues(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_payload", return_value={"categories": []}), patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "widgets"})["total"] >= 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert content_mod.list_pages_by_word_count_band(conn, ctx, {})["missing"] is True
+ assert content_mod.list_pages_missing_article_schema(conn, ctx, {})["missing"] is True
+
+
+def test_link_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert link_mod.list_outbound_links(conn, ctx, {})["error"]
+ assert link_mod.list_internal_links_from_url(conn, ctx, {"url": "https://ex.com/"})["error"]
+ assert link_mod.list_internal_links_to_url(conn, ctx, {"url": "https://ex.com/"})["error"]
+ assert link_mod.list_links_by_rel_nofollow(conn, ctx, {})["error"]
+ assert link_mod.list_pagerank_low_pages(conn, ctx, {})["error"]
+
+ assert link_mod.list_internal_links_from_url(conn, ctx, {})["error"] == "url is required"
+ assert link_mod.list_internal_links_to_url(conn, ctx, {})["error"] == "url is required"
+
+ payload = _payload()
+ with patch.object(Ctx, "load_payload", return_value=payload):
+ assert link_mod.list_outbound_links(conn, ctx, {})["total"] >= 1
+ assert link_mod.list_internal_links_from_url(conn, ctx, {"url": "https://ex.com/"})["total"] >= 1
+ assert link_mod.list_internal_links_to_url(conn, ctx, {"url": "https://ex.com/"})["total"] >= 1
+ assert link_mod.list_links_by_rel_nofollow(conn, ctx, {})["total"] >= 1
+ assert link_mod.list_links_by_rel_nofollow(conn, ctx, {"rel": "ugc"})["total"] >= 1
+ assert link_mod.list_pagerank_low_pages(conn, ctx, {"max_pagerank": "bad"})["total"] >= 1
+
+ graph_payload = {
+ "start_url": "https://ex.com/",
+ "graph_edges": [[0, 1], {"from": "https://ex.com/a", "to": "https://other.com/x"}],
+ "top_pages": [{"url": "https://ex.com/x", "pagerank": "bad"}, {"url": "https://ex.com/y", "pagerank": 0.001}],
+ }
+ with patch.object(Ctx, "load_payload", return_value=graph_payload):
+ assert link_mod.list_outbound_links(conn, ctx, {})["total"] >= 1
+ assert link_mod.list_pagerank_low_pages(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_payload", return_value={"top_pages": []}):
+ assert link_mod.list_pagerank_low_pages(conn, ctx, {})["missing"] is True
+
+
+def test_indexation_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ payload = _payload()
+ df = _crawl_df()
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert idx_mod.list_indexation_submitted_not_indexed(conn, ctx, {})["error"]
+ assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["error"]
+
+ with patch.object(Ctx, "load_payload", return_value={"indexation_coverage": "bad"}):
+ assert idx_mod.list_sitemap_urls_not_in_crawl(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_payload", return_value=payload):
+ assert idx_mod.list_indexation_submitted_not_indexed(conn, ctx, {})["total"] >= 1
+ assert idx_mod.list_indexation_indexed_not_submitted(conn, ctx, {})["total"] >= 1
+ assert idx_mod.list_sitemap_urls_not_in_crawl(conn, ctx, {})["source"]
+ assert idx_mod.list_crawl_urls_not_in_sitemap(conn, ctx, {})["source"]
+
+ no_prop = Ctx(property_id=None, report_id=1)
+ assert idx_mod.list_log_paths_by_hits(conn, no_prop, {})["error"]
+
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=None):
+ assert idx_mod.list_log_paths_by_hits(conn, ctx, {})["missing"] is True
+
+ log = _log_row()
+ orphan_payload = {**payload, "orphan_urls": ["https://ex.com/orphan"]}
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log), patch.object(
+ Ctx, "load_payload", return_value=orphan_payload,
+ ):
+ assert idx_mod.list_log_paths_by_hits(conn, ctx, {})["total"] >= 1
+ assert idx_mod.list_log_5xx_paths(conn, ctx, {})["total"] >= 1
+ assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": "bad"})["total"] >= 0
+ assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {"min_hits": "bad"})["total"] >= 0
+
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log), patch.object(
+ Ctx, "load_payload", return_value={"orphan_urls": []},
+ ):
+ assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["note"]
+
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert idx_mod.list_redirect_chains_by_length(conn, ctx, {"min_length": "bad"})["total"] >= 1
+ assert idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})["missing"] is True
+
+
+def test_compare_list_tools_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ err = {"error": "baseline required"}
+ with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(None, None, None, None, err)):
+ assert cmp_mod.list_compare_new_issues(conn, ctx, {})["error"]
+ assert cmp_mod.list_compare_resolved_issues(conn, ctx, {})["error"]
+ assert cmp_mod.list_compare_new_urls(conn, ctx, {})["error"]
+ assert cmp_mod.list_compare_removed_urls(conn, ctx, {})["error"]
+ assert cmp_mod.list_compare_lighthouse_regressions(conn, ctx, {})["error"]
+ assert cmp_mod.list_compare_traffic_losers(conn, ctx, {})["error"]
+
+ current, baseline = _compare_current(), _compare_baseline()
+ with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(current, baseline, 2, 1, None)):
+ assert cmp_mod.list_compare_new_issues(conn, ctx, {})["total"] >= 0
+ assert cmp_mod.list_compare_resolved_issues(conn, ctx, {})["total"] >= 0
+ assert cmp_mod.list_compare_new_urls(conn, ctx, {})["total"] >= 1
+ assert cmp_mod.list_compare_removed_urls(conn, ctx, {})["total"] >= 1
+ assert cmp_mod.list_compare_lighthouse_regressions(conn, ctx, {"min_regression": "bad"})["total"] >= 1
+ assert cmp_mod.list_compare_traffic_losers(conn, ctx, {})["total"] >= 1
+
+ no_google = dict(current)
+ no_google.pop("google")
+ base_no_google = dict(baseline)
+ base_no_google.pop("google")
+ with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(no_google, base_no_google, 2, 1, None)), patch.object(
+ Ctx, "load_google_full", return_value=None,
+ ), patch.object(Ctx, "load_google", return_value=None):
+ assert cmp_mod.list_compare_traffic_losers(conn, ctx, {})["missing"] is True
+
+
+def test_geo_list_tools_all_paths(conn: MagicMock, ctx: Ctx) -> None:
+ df = _crawl_df()
+ payload = _payload()
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert geo_list_mod.list_pages_missing_howto_schema(conn, ctx, {})["missing"] is True
+ assert geo_list_mod.list_pages_ai_citation_signals(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ assert geo_list_mod.list_pages_missing_howto_schema(conn, ctx, {})["total"] >= 1
+ assert geo_list_mod.list_pages_ai_citation_signals(conn, ctx, {"min_score": "bad"})["total"] >= 1
+
+ with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch(
+ "website_profiling.tools.audit_tools.geo_list_tools._fetch_llms_txt",
+ return_value={"found": False},
+ ):
+ assert geo_list_mod.list_pages_missing_llms_txt_reference(conn, ctx, {})["missing"] is True
+
+ llms = {"found": True, "url": "https://ex.com/llms.txt", "preview": "https://ex.com/\nMore docs"}
+ with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch(
+ "website_profiling.tools.audit_tools.geo_list_tools._fetch_llms_txt", return_value=llms,
+ ), patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df):
+ missing = geo_list_mod.list_pages_missing_llms_txt_reference(conn, ctx, {})
+ assert missing["total"] >= 1
+
+ with patch.object(Ctx, "resolve_property_domain", return_value=""):
+ assert geo_list_mod.list_robots_blocked_ai_crawlers(conn, ctx, {})["error"]
+
+ robots = "User-agent: GPTBot\nDisallow: /\nUser-agent: *\nDisallow: /private"
+ with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch.object(
+ geo_list_mod, "_parse_robots_txt", return_value=robots,
+ ):
+ blocked = geo_list_mod.list_robots_blocked_ai_crawlers(conn, ctx, {})
+ assert blocked["total"] >= 1
+
+ with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch.object(
+ geo_list_mod, "_parse_robots_txt", return_value="",
+ ):
+ assert geo_list_mod.list_robots_blocked_ai_crawlers(conn, ctx, {})["missing"] is True
+
+ with patch("website_profiling.tools.audit_tools.geo_list_tools.requests.get", side_effect=requests.RequestException("fail")):
+ assert geo_list_mod._parse_robots_txt("ex.com") == ""
+
+ mock_resp = MagicMock(status_code=404, text="")
+ with patch("website_profiling.tools.audit_tools.geo_list_tools.requests.get", return_value=mock_resp):
+ assert geo_list_mod._parse_robots_txt("ex.com") == ""
+
+
+def test_crawl_console_and_js_handlers(conn: MagicMock, ctx: Ctx) -> None:
+ assert crawl_mod.list_pages_console_errors_by_type(conn, ctx, {})["error"] == "error_type is required"
+
+ df = _crawl_df()
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert crawl_mod.list_pages_console_errors_by_type(conn, ctx, {"error_type": "error"})["total"] == 0
+ assert crawl_mod.list_pages_js_rendering_delta(conn, ctx, {})["note"]
+
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ console = crawl_mod.list_pages_console_errors_by_type(conn, ctx, {"error_type": "error"})
+ assert console["total"] >= 1
+ page_err = crawl_mod.list_pages_console_errors_by_type(conn, ctx, {"error_type": "page_error"})
+ assert page_err["total"] >= 1
+ js = crawl_mod.list_pages_js_rendering_delta(conn, ctx, {})
+ assert js["total"] >= 1
+ assert js["provenance"] == "Crawl"
+
+ js_only = pd.DataFrame([
+ {"url": "https://ex.com/js", "status": "200", "fetch_method": "static", "title": "A", "word_count": 10, "h1": "A"},
+ {"url": "https://ex.com/js", "status": "200", "fetch_method": "javascript", "title": "B", "word_count": 100, "h1": "B"},
+ ])
+ with patch.object(Ctx, "load_crawl_df", return_value=js_only):
+ assert crawl_mod.list_pages_js_rendering_delta(conn, ctx, {})["total"] == 1
+
+
+def test_batch100_coverage_gaps(conn: MagicMock, ctx: Ctx) -> None:
+ """Hit remaining branches for 100% line coverage on batch-100 modules."""
+ empty = pd.DataFrame()
+ df = _crawl_df()
+
+ # issue_lists helpers and dead paths
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert issue_mod._payload_list_key(conn, ctx, {}, "missing_key")["error"]
+ with patch.object(Ctx, "load_payload", return_value={"flat_list": [{"url": "x"}]}):
+ assert issue_mod._payload_list_key(conn, ctx, {}, "flat_list")["total"] == 1
+ with patch.object(Ctx, "load_payload", return_value={"scalar_key": 123}):
+ assert issue_mod._payload_list_key(conn, ctx, {}, "scalar_key")["missing"] is True
+ with patch.object(Ctx, "load_payload", return_value={"issues": {"seo": [{"type": "missing_title", "url": "u"}]}}):
+ assert issue_mod._issues_by_type(conn, ctx, {}, "missing_title")["total"] == 1
+ with patch.object(Ctx, "load_payload", return_value={"issues": "bad"}):
+ assert issue_mod._issues_by_type(conn, ctx, {}, "x")["total"] == 0
+
+ crawl_only = pd.DataFrame([
+ {"url": "https://ex.com/long", "status": "200", "title_length": 65, "title": "T" * 65},
+ {"url": "https://ex.com/slow2", "status": "200", "response_time_ms": 5000, "title": "Slow"},
+ {"url": "https://ex.com/nolang2", "status": "200", "title": "X", "page_analysis": "{}"},
+ {"url": "https://ex.com/vp", "status": "200", "viewport_present": "true", "viewport_content": "initial-scale=1", "page_analysis": "{}"},
+ {"url": "https://ex.com/read", "status": "200", "reading_level": 14, "title": "Hard"},
+ ])
+ with patch.object(Ctx, "load_payload", return_value={"content_urls": {}}), patch.object(Ctx, "load_crawl_df", return_value=crawl_only):
+ assert issue_mod.list_pages_title_too_long(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_slow_response(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_missing_html_lang(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_invalid_viewport(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_pages_high_reading_level(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_payload", return_value={"hreflang_issue_urls": []}), patch.object(Ctx, "load_crawl_df", return_value=empty):
+ assert issue_mod.list_hreflang_issue_pages(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_payload", return_value=None), patch.object(Ctx, "load_crawl_df", return_value=empty):
+ assert issue_mod.list_pages_missing_og_tags(conn, ctx, {})["error"]
+ assert issue_mod.list_pages_missing_twitter_cards(conn, ctx, {})["error"]
+
+ no_dom = {"language_summary": {"counts": {}}}
+ with patch.object(Ctx, "load_payload", return_value=no_dom), patch.object(Ctx, "load_crawl_df", return_value=df):
+ mixed = issue_mod.list_pages_mixed_language(conn, ctx, {})
+ assert mixed["total"] == 0
+
+ lh_edge = {
+ "lighthouse_failure_urls": {},
+ "lighthouse_by_url": {
+ "https://ex.com/lcp2": {"lcp": 4.0, "cwv_failures": "lcp"},
+ "https://ex.com/inp2": {"inp": 500, "top_failures": [{"id": "inp-slow"}]},
+ "https://ex.com/cls2": {"cls": 0.25, "cwv_failures": "cls"},
+ "https://ex.com/audit": {"lcp": 1.0, "audits": {"lcp": {"score": 0.5, "title": "LCP audit"}}},
+ "bad": "skip",
+ },
+ }
+ with patch.object(Ctx, "load_payload", return_value=lh_edge):
+ assert issue_mod.list_lighthouse_failure_lcp(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_lighthouse_failure_inp(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_lighthouse_failure_cls(conn, ctx, {})["total"] >= 1
+ assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["total"] == 0
+ with patch.object(Ctx, "load_payload", return_value={"lighthouse_by_url": {"https://ex.com/badseo": {"seo": "bad"}}}):
+ assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["total"] == 0
+
+ # google_lists gaps
+ no_google = None
+ google_handlers = [
+ google_mod.list_gsc_pages_by_impressions,
+ google_mod.list_gsc_pages_by_clicks,
+ google_mod.list_gsc_queries_by_impressions,
+ google_mod.list_gsc_queries_by_clicks,
+ google_mod.list_gsc_ctr_underperformers,
+ google_mod.list_ga4_landing_pages,
+ google_mod.list_ga4_pages_by_bounce_rate,
+ google_mod.list_ga4_pages_by_engagement_rate,
+ google_mod.list_gsc_ga4_mismatch_pages,
+ google_mod.list_gsc_pages_by_position_band,
+ google_mod.get_gsc_site_benchmarks,
+ ]
+ with patch.object(Ctx, "load_google_full", return_value=no_google), patch.object(Ctx, "load_google", return_value=no_google):
+ for handler in google_handlers:
+ assert handler(conn, ctx, {}).get("missing") is True
+
+ assert google_mod._gsc_rows(None, "pages") == []
+ assert google_mod._gsc_rows({"gsc_full": {"pages": "bad"}}, "pages") == []
+
+ decay_curr = {"gsc_full": {"pages": [{"page": "https://ex.com/p", "clicks": 5, "impressions": 100, "position": 8}]}}
+ decay_prior = {"gsc_full": {"pages": [{"page": "https://ex.com/p", "clicks": 20, "impressions": 200, "position": 5}]}}
+ assert google_mod._gsc_deltas(
+ google_mod._gsc_rows(decay_curr, "pages"),
+ google_mod._gsc_rows(decay_prior, "pages"),
+ ("page",),
+ decay=True,
+ )
+ assert google_mod._gsc_deltas(
+ google_mod._gsc_rows(decay_curr, "pages"),
+ google_mod._gsc_rows(decay_prior, "pages"),
+ ("page",),
+ decay=False,
+ ) == []
+
+ with patch.object(google_mod, "_load_google_pair", return_value=(decay_curr, decay_prior)):
+ assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["total"] >= 1
+ assert google_mod.list_gsc_decaying_queries(conn, ctx, {"limit": 5})["total"] == 0
+
+ with patch.object(google_mod, "_load_google_pair", return_value=(None, decay_prior)):
+ assert google_mod.list_gsc_decaying_queries(conn, ctx, {})["missing"] is True
+ assert google_mod.list_gsc_new_queries(conn, ctx, {})["missing"] is True
+
+ ga4_by_path = {"ga4_full": {"by_path": {"/only": {"sessions": 5, "bounceRate": 0.5, "engagementRate": 0.4}}}}
+ with patch.object(Ctx, "load_google_full", return_value=ga4_by_path):
+ assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] == 1
+ assert google_mod.list_ga4_pages_by_bounce_rate(conn, ctx, {})["total"] == 1
+ assert google_mod.list_ga4_pages_by_engagement_rate(conn, ctx, {})["total"] == 1
+
+ daily_dims = {
+ "gsc": {"daily": [{"dimensions": {"page": "https://ex.com/dim"}, "clicks": 1}, "skip"]},
+ "ga4": {"daily": "bad"},
+ }
+ with patch.object(Ctx, "load_google_full", return_value=daily_dims):
+ assert google_mod._daily_series(daily_dims, "ga4", "path", "/") == []
+ assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com/dim"})["daily"]
+
+ high_ctr_only = {"gsc_full": {"pages": [{"page": "https://ex.com/g", "clicks": 50, "impressions": 100, "ctr": "50%", "position": 3}]}}
+ with patch.object(Ctx, "load_google_full", return_value=high_ctr_only):
+ assert google_mod.list_gsc_ctr_underperformers(conn, ctx, {})["total"] == 0
+
+ # keyword_lists gaps
+ kw = _keyword_data()
+ with patch.object(Ctx, "load_keywords", return_value=None), patch(
+ "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property",
+ return_value=[kw],
+ ):
+ cur, prior = kw_mod._load_keyword_pair(ctx, conn)
+ assert cur is kw and prior is None
+
+ entered_prior = {
+ "rows": [{"keyword": "entered", "gsc_position": 15, "gsc_impressions": 10}],
+ }
+ entered_curr = {
+ "rows": [{"keyword": "entered", "gsc_position": 8, "gsc_impressions": 20}],
+ }
+ assert kw_mod._top_ten_transitions(entered_curr, entered_prior, entered=True)
+ fell_curr = {"rows": [{"keyword": "fell", "gsc_position": 15, "gsc_impressions": 5}]}
+ fell_prior = {"rows": [{"keyword": "fell", "gsc_position": 5, "gsc_impressions": 50}]}
+ assert kw_mod._top_ten_transitions(fell_curr, fell_prior, entered=False)
+
+ no_prop = Ctx(property_id=None, report_id=1)
+ assert kw_mod.list_cannibalisation_urls(conn, no_prop, {})["missing"] is True
+ assert kw_mod.list_misaligned_queries(conn, no_prop, {})["missing"] is True
+ assert kw_mod.list_semantic_cluster_pages(conn, no_prop, {})["missing"] is True
+ assert kw_mod.get_keyword_opportunity_score(conn, no_prop, {"keyword": "x"})["missing"] is True
+ assert kw_mod.get_keyword_serp_snapshot(conn, no_prop, {"keyword": "x"})["missing"] is True
+
+ with patch.object(Ctx, "load_keywords", return_value=kw), patch.object(Ctx, "load_payload", return_value={}):
+ assert kw_mod._keyword_bucket(conn, ctx, {}, key="semantic_keyword_clusters", item_key="clusters")["total"] >= 1
+
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert kw_mod._filter_keywords(conn, ctx, {}, lambda r: True)["missing"] is True
+ assert kw_mod._pair_delta_tool(conn, ctx, {}, builder=lambda a, b: [], item_key="keywords")["missing"] is True
+
+ with patch.object(Ctx, "load_keywords", return_value={"rows": []}), patch(
+ "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property",
+ return_value=[],
+ ):
+ assert kw_mod._load_keyword_pair(ctx, conn) == ({"rows": []}, None)
+
+ with patch.object(Ctx, "load_keywords", return_value=kw), patch.object(Ctx, "load_payload", return_value={}):
+ assert kw_mod._semantic_clusters(ctx, conn)
+
+ assert kw_mod._keyword_rows(None) == []
+ assert kw_mod._position({"gsc_position": 0}) is None
+ assert kw_mod._position({"gsc_position": "bad"}) is None
+
+ with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "x"}]}):
+ assert kw_mod.list_keywords_by_competition_band(conn, ctx, {"min_competition": "bad"})["error"]
+
+ # backlink_lists error branches
+ no_prop = Ctx(property_id=None, report_id=1)
+ for handler, args in [
+ (bl_mod.list_backlinks_by_anchor_text, {}),
+ (bl_mod.list_backlinks_to_url, {"url": "https://ex.com"}),
+ (bl_mod.list_backlinks_from_domain, {"domain": "x.com"}),
+ ]:
+ assert handler(conn, no_prop, args)["error"]
+ with patch.object(Ctx, "load_gsc_links", return_value=None):
+ assert bl_mod.list_backlinks_by_anchor_text(conn, ctx, {})["missing"] is True
+ assert bl_mod.list_backlinks_to_url(conn, ctx, {"url": "https://ex.com"})["missing"] is True
+ assert bl_mod.list_backlinks_from_domain(conn, ctx, {"domain": "x.com"})["missing"] is True
+ assert bl_mod.get_anchor_text_distribution(conn, ctx, {})["missing"] is True
+ with patch("website_profiling.tools.audit_tools.backlink_lists.urlparse", side_effect=ValueError("bad")):
+ assert bl_mod._norm_domain("bad://") == "bad://"
+
+ # content_lists gaps
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["error"]
+ assert content_mod.list_schema_errors_by_type(conn, ctx, {})["error"]
+
+ cat_issues = {
+ "optional_audit_urls": {},
+ "categories": [
+ {
+ "issues": [
+ {"message": "spell issue on page"},
+ {"message": "html markup broken"},
+ {"message": "amp validation failed"},
+ {"message": "pagination rel=next broken"},
+ {"message": "needle custom audit"},
+ ],
+ },
+ ],
+ }
+ with patch.object(Ctx, "load_payload", return_value=cat_issues):
+ assert content_mod.list_spell_check_issues(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_html_validation_issues(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_amp_validation_issues(conn, ctx, {})["total"] >= 1
+ assert content_mod.list_pagination_issues(conn, ctx, {})["total"] >= 1
+
+ dup_bad = {"content_duplicates": ["skip", {"member_urls": "bad", "representative_url": "https://ex.com/a"}]}
+ with patch.object(Ctx, "load_payload", return_value=dup_bad):
+ assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["total"] == 0
+
+ schema_payload = {"rich_results_validation": ["skip", {"status": "fail", "type": "Product"}]}
+ with patch.object(Ctx, "load_payload", return_value=schema_payload):
+ assert content_mod.list_schema_errors_by_type(conn, ctx, {})["total"] == 1
+ assert content_mod.list_schema_errors_by_type(conn, ctx, {"schema_type": "product"})["total"] == 1
+
+ article_df = pd.DataFrame([
+ {
+ "url": "https://ex.com/blog/my-post",
+ "status": "200",
+ "title": "Post",
+ "content_excerpt": "Posted by author on Monday. " + ("word " * 210),
+ "page_analysis": json.dumps({"json_ld_types": ["WebPage"]}),
+ },
+ {
+ "url": "https://ex.com/with-schema",
+ "status": "200",
+ "title": "Article",
+ "content_excerpt": "short",
+ "page_analysis": json.dumps({"json_ld_types": ["NewsArticle"]}),
+ },
+ ])
+ with patch.object(Ctx, "load_crawl_df", return_value=article_df):
+ assert content_mod.list_pages_missing_article_schema(conn, ctx, {})["total"] >= 1
+
+ kw_payload = {
+ "text_content_analysis": {
+ "keyword_index": [{"word": "term", "top_pages": ["skip", ("https://ex.com/t", 1)]}],
+ },
+ }
+ with patch.object(Ctx, "load_payload", return_value=kw_payload), patch.object(Ctx, "load_crawl_df", return_value=empty):
+ assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "term"})["total"] == 1
+
+ # link_lists gaps
+ graph_only = {"graph_edges": ["skip", [1, 2]], "top_pages": ["skip", {"url": "https://ex.com/z", "pagerank": None}]}
+ with patch.object(Ctx, "load_payload", return_value=graph_only):
+ assert link_mod._load_link_edges(graph_only)
+ assert link_mod._pagerank_rows(graph_only) == []
+
+ # indexation_lists gaps
+ assert idx_mod._cap_indexation_urls("bad", {})["total"] == 0
+ with patch.object(Ctx, "load_payload", return_value=_payload()):
+ cov_err = idx_mod.list_indexation_indexed_not_submitted(conn, ctx, {})
+ assert cov_err["total"] >= 1
+
+ log_non_list = {
+ "analysis": {
+ "top_paths": "bad",
+ "paths_5xx": "bad",
+ "googlebot_paths": ["skip"],
+ "status_counts": {"500": 12, "503": 3},
+ },
+ }
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log_non_list), patch.object(
+ Ctx, "load_payload", return_value=_payload(),
+ ):
+ assert idx_mod.list_log_paths_by_hits(conn, ctx, {})["total"] == 0
+ assert idx_mod.list_log_5xx_paths(conn, ctx, {})["total"] >= 1
+ assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {})["total"] >= 0
+
+ orphan_log = {
+ **_payload(),
+ "orphan_urls": ["https://ex.com/orphan"],
+ "links": [{"url": "https://ex.com/orphan"}],
+ }
+ log_row = {
+ "analysis": {
+ "top_paths": [{"path": "/orphan", "hits": 50}, {"path": "/other", "hits": 3}],
+ "googlebot_paths": [{"path": "/orphan", "hits": 0}],
+ },
+ }
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log_row), patch.object(
+ Ctx, "load_payload", return_value=orphan_log,
+ ):
+ assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["total"] >= 1
+ assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": 10, "max_googlebot_hits": 0})["total"] >= 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame([{"url": "https://ex.com/r", "status": "301", "redirect_chain_length": "bad"}])):
+ assert idx_mod.list_redirect_chains_by_length(conn, ctx, {})["total"] == 0
+
+ href_df = pd.DataFrame([
+ {"url": "https://ex.com/a", "status": "200", "page_analysis": json.dumps({
+ "hreflang_alternates": [{"href": "https://ex.com/b", "hreflang": "en"}],
+ })},
+ {"url": "https://ex.com/b", "status": "200", "page_analysis": json.dumps({
+ "hreflang_alternates": [{"href": "https://ex.com/c", "hreflang": "de"}],
+ })},
+ ])
+ with patch.object(Ctx, "load_crawl_df", return_value=href_df):
+ gaps = idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})
+ assert gaps["total"] >= 1
+
+ with patch("website_profiling.tools.audit_tools.indexation_lists.url_to_path", side_effect=RuntimeError("bad")):
+ assert idx_mod._norm_path("https://ex.com/x") == "https://ex.com/x"
+
+ # compare_list_tools line 156 (skip non-losers)
+ winner_current = {
+ "report_generated_at": "2026-06-07",
+ "google": {"gsc_full": {"pages": [{"page": "https://ex.com/win", "clicks": 100, "impressions": 200}]}},
+ }
+ winner_baseline = {
+ "report_generated_at": "2026-05-01",
+ "google": {"gsc_full": {"pages": [{"page": "https://ex.com/win", "clicks": 10, "impressions": 50}]}},
+ }
+ with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(winner_current, winner_baseline, 2, 1, None)):
+ losers = cmp_mod.list_compare_traffic_losers(conn, ctx, {})
+ assert losers["total"] == 0
+
+ # geo_list_tools gaps
+ low_score_df = pd.DataFrame([{
+ "url": "https://ex.com/low",
+ "status": "200",
+ "title": "Low",
+ "content_excerpt": "tiny",
+ "word_count": 10,
+ "page_analysis": "{}",
+ }])
+ with patch.object(Ctx, "load_crawl_df", return_value=low_score_df):
+ assert geo_list_mod.list_pages_ai_citation_signals(conn, ctx, {"min_score": 50})["total"] == 0
+
+ assert geo_list_mod._parse_robots_txt("") == ""
+ ok_resp = MagicMock(status_code=200, text="User-agent: *\nAllow: /\n# comment\nUser-agent: ClaudeBot\nDisallow: /")
+ with patch("website_profiling.tools.audit_tools.geo_list_tools.requests.get", return_value=ok_resp):
+ robots = geo_list_mod._parse_robots_txt("ex.com")
+ assert "User-agent" in robots
+ assert geo_list_mod._agent_blocked(robots, "ClaudeBot") is True
+
+ # --- final line coverage targets ---
+ assert bl_mod._load_links(Ctx(property_id=None), conn) is None
+ assert bl_mod.get_anchor_text_distribution(conn, Ctx(property_id=None), {})["error"]
+
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert content_mod.list_spell_check_issues(conn, ctx, {})["error"]
+
+ optional_cat = {
+ "categories": [
+ "skip",
+ {"issues": ["skip", {"message": "needle custom audit on page"}]},
+ ],
+ }
+ with patch.object(Ctx, "load_payload", return_value=optional_cat):
+ assert content_mod.list_pagination_issues(conn, ctx, {})["total"] >= 0
+
+ with patch.object(Ctx, "load_payload", return_value={"content_duplicates": "bad"}):
+ assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["total"] == 0
+ with patch.object(Ctx, "load_payload", return_value={"rich_results_validation": "bad"}):
+ assert content_mod.list_schema_errors_by_type(conn, ctx, {})["total"] == 0
+
+ article_types = pd.DataFrame([{
+ "url": "https://ex.com/story-longform",
+ "status": "200",
+ "title": "Story",
+ "content_excerpt": "Posted by author on Monday. " + ("word " * 210),
+ "page_analysis": json.dumps({"json_ld_types": ["WebPage"]}),
+ }])
+ with patch.object(Ctx, "load_crawl_df", return_value=article_types):
+ content_mod.list_pages_missing_article_schema(conn, ctx, {})
+
+ kw_skip = {
+ "text_content_analysis": {
+ "keyword_index": ["skip", {"word": "needle", "top_pages": [{"url": "https://ex.com/n", "count": 1}]}],
+ },
+ }
+ with patch.object(Ctx, "load_payload", return_value=kw_skip), patch.object(Ctx, "load_crawl_df", return_value=empty):
+ assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "needle"})["total"] == 1
+
+ link_empty = {"link_edges": "bad", "links": ["skip", {"url": "https://ex.com/p", "pagerank": 0.5}]}
+ with patch.object(Ctx, "load_payload", return_value=link_empty):
+ assert link_mod._load_link_edges(link_empty) == []
+ assert link_mod._pagerank_rows(link_empty)[0]["url"] == "https://ex.com/p"
+
+ improving = {"gsc_full": {"pages": [{"page": "https://ex.com/up", "clicks": 30, "impressions": 300, "position": 3}]}}
+ declining = {"gsc_full": {"pages": [{"page": "https://ex.com/up", "clicks": 10, "impressions": 100, "position": 5}]}}
+ assert google_mod._gsc_deltas(
+ google_mod._gsc_rows(improving, "pages"),
+ google_mod._gsc_rows(declining, "pages"),
+ ("page",),
+ decay=True,
+ ) == []
+
+ with patch.object(google_mod, "_load_google_pair", return_value=(None, declining)):
+ assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["missing"] is True
+ assert google_mod.list_gsc_new_queries(conn, ctx, {"limit": 1})["missing"] is True
+
+ ga4_empty = {"ga4_full": {"top_pages": [], "by_path": {}}}
+ with patch.object(Ctx, "load_google_full", return_value=ga4_empty):
+ assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] == 0
+
+ mismatch_data = {
+ "gsc_full": {
+ "by_page": {"https://ex.com/ratio": {"clicks": 10, "impressions": 50}},
+ "pages": [],
+ },
+ "ga4_full": {"by_path": {"/ratio": {"sessions": 40}}},
+ }
+ with patch.object(Ctx, "load_google_full", return_value=mismatch_data):
+ assert google_mod.list_gsc_ga4_mismatch_pages(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_google_full", return_value={"gsc_full": {"pages": []}}):
+ assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com/missing"})["missing"] is True
+
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert google_mod.list_gsc_branded_queries(conn, ctx, {})["missing"] is True
+ assert google_mod.list_gsc_non_branded_queries(conn, ctx, {})["missing"] is True
+
+ with patch.object(google_mod, "_load_google_pair", return_value=(_google_mismatch(), None)):
+ assert google_mod.compare_gsc_periods(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_payload", return_value={"issues": {"seo": "not-list"}}):
+ assert issue_mod._issues_by_type(conn, ctx, {}, "x")["total"] == 0
+
+ bad_crawl = pd.DataFrame([
+ {"url": "https://ex.com/badrt", "status": "200", "response_time_ms": "x", "title": "X"},
+ {"url": "https://ex.com/badread", "status": "200", "reading_level": "x", "title": "X"},
+ ])
+ with patch.object(Ctx, "load_payload", return_value={"content_urls": {}}), patch.object(Ctx, "load_crawl_df", return_value=bad_crawl):
+ assert issue_mod.list_pages_slow_response(conn, ctx, {})["total"] == 0
+ assert issue_mod.list_pages_high_reading_level(conn, ctx, {})["total"] == 0
+
+ lh_skip = {
+ "lighthouse_by_url": {
+ "https://ex.com/seo-fail2": {"seo": 40},
+ "skip": "x",
+ },
+ }
+ with patch.object(Ctx, "load_payload", return_value=lh_skip):
+ assert issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "seo")["total"] >= 1
+ assert issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "lcp")["total"] == 0
+
+ with patch.object(Ctx, "load_payload", return_value={"indexation_coverage": {"lists": {}, "lists_total": {}}}):
+ idx_mod.list_indexation_submitted_not_indexed(conn, ctx, {})
+ idx_mod.list_crawl_urls_not_in_sitemap(conn, ctx, {})
+
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()), patch.object(
+ Ctx, "load_payload", return_value=_payload(),
+ ):
+ assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": "bad"})["paths"] is not None
+
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()), patch.object(
+ Ctx, "load_payload", return_value=None,
+ ):
+ assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["error"]
+
+ href_skip = pd.DataFrame([
+ {"url": "https://ex.com/a", "status": "200", "page_analysis": json.dumps({
+ "hreflang_alternates": ["skip", {"href": "https://ex.com/b", "hreflang": "en"}],
+ })},
+ {"url": "https://ex.com/b", "status": "200", "page_analysis": "{}"},
+ ])
+ with patch.object(Ctx, "load_crawl_df", return_value=href_skip):
+ assert idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame([{"url": "https://ex.com/r2", "status": "301", "redirect_chain_length": 2}])):
+ idx_mod.list_redirect_chains_by_length(conn, ctx, {"min_length": "bad"})
+
+ skip_kw = {"rows": [{"keyword": "skip-top10", "gsc_position": 8}]}
+ skip_prior = {"rows": [{"keyword": "skip-top10", "gsc_position": 5}]}
+ assert kw_mod._top_ten_transitions(skip_kw, skip_prior, entered=True) == []
+ skip_fell = {"rows": [{"keyword": "fell2", "gsc_position": 20}]}
+ skip_fell_prior = {"rows": [{"keyword": "fell2", "gsc_position": 6}]}
+ assert kw_mod._top_ten_transitions(skip_fell, skip_fell_prior, entered=False)
+
+ assert kw_mod._rank_delta_rows(
+ {"rows": [{"keyword": "x", "gsc_position": None}]},
+ {"rows": [{"keyword": "x", "gsc_position": 8}]},
+ improved=True,
+ ) == []
+ assert kw_mod._rank_delta_rows(
+ {"rows": [{"keyword": "y", "gsc_position": 10}]},
+ {"rows": [{"keyword": "y", "gsc_position": 8}]},
+ improved=True,
+ ) == []
+
+ with patch.object(Ctx, "load_keywords", return_value=None), patch.object(Ctx, "load_payload", return_value={"semantic_keyword_clusters": [{"keywords": ["a"]}]}):
+ assert kw_mod._semantic_clusters(ctx, conn)
+
+ with patch.object(Ctx, "load_payload", return_value={"semantic_keyword_clusters": []}), patch.object(Ctx, "load_keywords", return_value=None):
+ assert kw_mod.list_semantic_cluster_pages(conn, ctx, {})["missing"] is True
+
+ with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "widgets", "gsc_position": 5}]}):
+ assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "widgets"})["opportunity_score"] >= 0
+ assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {"keyword": "missing-kw"})["missing"] is True
+
+ # --- remaining uncovered line targets ---
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert issue_mod._issues_by_type(conn, ctx, {}, "missing_title")["error"]
+
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "lcp")["error"]
+
+ lh_bad_cwv = {
+ "lighthouse_by_url": {
+ "https://ex.com/bad-cwv": {"lcp": "n/a", "cwv_failures": "lcp", "top_failures": []},
+ "https://ex.com/bad-seo": {"seo": "bad"},
+ },
+ }
+ with patch.object(Ctx, "load_payload", return_value=lh_bad_cwv):
+ issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "lcp")
+ issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "seo")
+
+ under_ctr = {
+ "gsc_full": {
+ "pages": [
+ {"page": "https://ex.com/high", "clicks": 50, "impressions": 500, "ctr": 0.08, "position": 3},
+ {"page": "https://ex.com/low-ctr", "clicks": 1, "impressions": 500, "ctr": 0.01, "position": 5},
+ ],
+ },
+ }
+ with patch.object(Ctx, "load_google_full", return_value=under_ctr):
+ assert google_mod.list_gsc_ctr_underperformers(conn, ctx, {})["total"] >= 1
+
+ decay_curr = {"gsc_full": {"queries": [{"query": "decay-q", "clicks": 1, "impressions": 50, "position": 12}]}}
+ decay_prior = {"gsc_full": {"queries": [{"query": "decay-q", "clicks": 10, "impressions": 200, "position": 5}]}}
+ with patch.object(google_mod, "_load_google_pair", return_value=(decay_curr, None)):
+ assert google_mod.list_gsc_decaying_queries(conn, ctx, {})["missing"] is True
+ assert google_mod.list_gsc_new_queries(conn, ctx, {})["missing"] is True
+ with patch.object(google_mod, "_load_google_pair", return_value=(None, decay_prior)):
+ assert google_mod.compare_gsc_periods(conn, ctx, {})["missing"] is True
+
+ ga4_bad_pages = {"ga4_full": {"top_pages": "bad", "by_path": {"/x": {"sessions": 1}}}}
+ with patch.object(Ctx, "load_google_full", return_value=ga4_bad_pages):
+ assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] == 0
+
+ assert google_mod._daily_series(None, "gsc", "page", "/") == []
+
+ with patch.object(Ctx, "load_google_full", return_value={"gsc_full": {"queries": [{"query": "snap-q", "clicks": 2}]}}):
+ snap = google_mod.get_gsc_query_trend(conn, ctx, {"query": "snap-q"})
+ assert snap.get("missing") is True and snap.get("snapshot")
+ fallback = google_mod.get_gsc_query_trend(conn, ctx, {"query": "not-in-gsc"})
+ assert fallback.get("daily") == []
+
+ with patch.object(Ctx, "load_google_full", return_value=_google_mismatch()):
+ assert google_mod.list_gsc_pages_by_position_band(conn, ctx, {"max_position": object()})["total"] >= 0
+
+ mismatch_pages_only = {
+ "gsc_full": {
+ "pages": [{"page": "https://ex.com/p-only", "clicks": 15, "impressions": 80, "position": 4}],
+ },
+ "ga4_full": {"by_path": {"/p-only": {"sessions": 0}}},
+ }
+ with patch.object(Ctx, "load_google_full", return_value=mismatch_pages_only):
+ assert google_mod.list_gsc_ga4_mismatch_pages(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_google_full", return_value=_google_mismatch()):
+ assert google_mod.list_gsc_pages_by_position_band(conn, ctx, {"min_position": object()})["total"] >= 0
+
+ fell_still_top = {"rows": [{"keyword": "still-top", "gsc_position": 8}]}
+ fell_still_prior = {"rows": [{"keyword": "still-top", "gsc_position": 6}]}
+ assert kw_mod._top_ten_transitions(fell_still_top, fell_still_prior, entered=False) == []
+
+ assert kw_mod.list_keywords_near_page_one(conn, Ctx(property_id=None, report_id=1), {})["missing"] is True
+
+ cann_empty_url = {
+ "cannibalisation": [{"query": "q", "pages": [{"url": "", "position": 1}, {"url": "https://ex.com/c", "position": 2}]}],
+ }
+ with patch.object(Ctx, "load_keywords", return_value=cann_empty_url):
+ assert kw_mod.list_cannibalisation_urls(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert kw_mod.list_cannibalisation_urls(conn, ctx, {})["missing"] is True
+ assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "x"})["missing"] is True
+ assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {"keyword": "x"})["missing"] is True
+
+ custom_audit = {
+ "optional_audit_urls": {},
+ "categories": [{"issues": [{"message": "custom foo audit detected on page"}]}],
+ }
+ with patch.object(Ctx, "load_payload", return_value=custom_audit):
+ assert content_mod._optional_audit_urls(conn, ctx, {}, "foo")["total"] >= 1
+
+ kw_index = {
+ "text_content_analysis": {
+ "keyword_index": [
+ {"word": "unrelated", "top_pages": [{"url": "https://ex.com/x", "count": 1}]},
+ {"word": "needleword", "top_pages": [{"url": "https://ex.com/y", "count": 2}]},
+ ],
+ },
+ }
+ with patch.object(Ctx, "load_payload", return_value=kw_index), patch.object(Ctx, "load_crawl_df", return_value=empty):
+ assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "needle"})["total"] == 1
+
+ article_str_types = pd.DataFrame([{
+ "url": "https://ex.com/blog/posted-by-author",
+ "status": "200",
+ "title": "Story",
+ "content_excerpt": "Posted by author on Monday. " + ("word " * 210),
+ "page_analysis": json.dumps({"json_ld_types": "WebPage"}),
+ }])
+ with patch.object(Ctx, "load_crawl_df", return_value=article_str_types):
+ content_mod.list_pages_missing_article_schema(conn, ctx, {})
+
+ no_cov_payload = {k: v for k, v in _payload().items() if k != "indexation_coverage"}
+ with patch.object(Ctx, "load_payload", return_value=no_cov_payload):
+ assert idx_mod.list_indexation_indexed_not_submitted(conn, ctx, {})["error"]
+ assert idx_mod.list_crawl_urls_not_in_sitemap(conn, ctx, {})["error"]
+
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()):
+ assert idx_mod.list_log_5xx_paths(conn, Ctx(property_id=None, report_id=1), {})["error"]
+ assert idx_mod.list_log_googlebot_low_crawl(conn, Ctx(property_id=None, report_id=1), {})["error"]
+
+ bot_log = {
+ "analysis": {
+ "top_paths": ["skip", {"path": "/popular", "hits": 100}, {"path": "/crawled", "hits": 50}],
+ "googlebot_paths": [{"path": "/popular", "hits": 2}, {"path": "/crawled", "hits": 1}],
+ },
+ }
+ bot_payload = {
+ **_payload(),
+ "links": [{"url": "https://ex.com/crawled"}, {"url": "https://ex.com/popular"}],
+ }
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=bot_log), patch.object(
+ Ctx, "load_payload", return_value=bot_payload,
+ ):
+ assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": 20, "max_googlebot_hits": 5})["total"] == 0
+
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()):
+ assert idx_mod.list_log_orphan_high_traffic(conn, Ctx(property_id=None, report_id=1), {})["error"]
+
+ orphan_payload = {
+ **_payload(),
+ "orphan_urls": ["", "https://ex.com/orphan"],
+ }
+ orphan_log2 = {
+ "analysis": {
+ "top_paths": ["skip", {"path": "/orphan", "hits": 50}],
+ },
+ }
+ with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=orphan_log2), patch.object(
+ Ctx, "load_payload", return_value=orphan_payload,
+ ):
+ assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["total"] >= 1
+
+ with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()):
+ assert idx_mod.list_redirect_chains_by_length(conn, ctx, {})["pages"] == []
+
+ href_empty_url = pd.DataFrame([
+ {"url": "", "status": "200", "page_analysis": "{}"},
+ {"url": "https://ex.com/a", "status": "200", "page_analysis": json.dumps({
+ "hreflang_alternates": [{"href": "https://ex.com/b", "hreflang": "en"}],
+ })},
+ {"url": "https://ex.com/b", "status": "200", "page_analysis": "{}"},
+ ])
+ with patch.object(Ctx, "load_crawl_df", return_value=href_empty_url):
+ idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})
+
+ bad_graph = {"link_edges": [], "graph_edges": "bad", "top_pages": "bad", "links": [{"url": "https://ex.com/p", "pagerank": 0.3}]}
+ with patch.object(Ctx, "load_payload", return_value=bad_graph):
+ assert link_mod._load_link_edges(bad_graph) == []
+ assert link_mod._pagerank_rows(bad_graph) == []
+
diff --git a/tests/test_audit_tools_expanded.py b/tests/test_audit_tools_expanded.py
index a0ffdcc7..ead7e11f 100644
--- a/tests/test_audit_tools_expanded.py
+++ b/tests/test_audit_tools_expanded.py
@@ -178,7 +178,7 @@ def conn() -> MagicMock:
def test_handler_schema_parity() -> None:
names = {t["name"] for t in TOOL_DEFINITIONS}
assert names == tool_handler_names()
- assert len(TOOL_DEFINITIONS) == 221
+ assert len(TOOL_DEFINITIONS) == 340
def test_slice_helpers() -> None:
diff --git a/tests/test_categories_coverage.py b/tests/test_categories_coverage.py
index f857995c..0017d638 100644
--- a/tests/test_categories_coverage.py
+++ b/tests/test_categories_coverage.py
@@ -467,7 +467,7 @@ def test_category_html_accessibility_alt_thin_reading_level() -> None:
def test_category_html_accessibility_score_zero_floor() -> None:
df = pd.DataFrame([{"url": "https://example.com/", "status": "200", "h1_count": 1}])
with patch(
- "website_profiling.reporting.categories._score_deductions",
+ "website_profiling.reporting.categories.accessibility._score_deductions",
return_value=0,
):
cat = category_html_accessibility(df)
diff --git a/tests/test_cli_dispatch.py b/tests/test_cli_dispatch.py
index ebec6d01..bba1d27e 100644
--- a/tests/test_cli_dispatch.py
+++ b/tests/test_cli_dispatch.py
@@ -17,5 +17,5 @@ def test_cli_help_lists_commands():
timeout=15,
)
assert proc.returncode == 0
- for cmd in ("crawl", "report", "plot", "lighthouse", "keywords", "warnings", "enrich", "google", "gsc-links-import"):
+ for cmd in ("crawl", "content_analysis", "report", "plot", "lighthouse", "keywords", "warnings", "enrich", "google", "gsc-links-import"):
assert cmd in proc.stdout
diff --git a/tests/test_config_schema_keys.py b/tests/test_config_schema_keys.py
index 97c211a4..9395716c 100644
--- a/tests/test_config_schema_keys.py
+++ b/tests/test_config_schema_keys.py
@@ -18,6 +18,11 @@
"store_outlinks",
"store_content_excerpt",
"content_excerpt_max_chars",
+ "store_page_html",
+ "max_stored_html_bytes",
+ "run_content_analysis",
+ "content_analysis_strategy",
+ "content_analysis_workers",
"preserve_crawl_history",
"crawl_stream_to_db",
"crawl_exclude_urls",
diff --git a/tests/test_content_analysis.py b/tests/test_content_analysis.py
new file mode 100644
index 00000000..c1fee8a6
--- /dev/null
+++ b/tests/test_content_analysis.py
@@ -0,0 +1,59 @@
+"""Tests for content_analysis package."""
+from __future__ import annotations
+
+import json
+
+from website_profiling.content_analysis.dom_cleanup import cleanup_dom
+from website_profiling.content_analysis.html_loader import load_soup
+from website_profiling.content_analysis.main_content import find_main_content
+from website_profiling.content_analysis.page import analyze_page_html
+from website_profiling.content_analysis.tokenize import tokenize_words
+
+
+def test_analyze_page_html_counts_body_words() -> None:
+ html = "Hello world again.
Menu Home About "
+ out = analyze_page_html(html, strategy="main_only")
+ assert out["word_count"] > 0
+ assert out["reading_level"] >= 0
+ assert json.loads(out["top_keywords"])
+
+
+def test_main_only_excludes_sidebar_noise() -> None:
+ html = """
+
+
+ Article about widgets and reviews for buyers.
Visible
")
+ cleaned = cleanup_dom(soup)
+ assert "Visible" in cleaned.get_text()
+ assert cleaned.find("script") is None
+
+
+def test_excerpt_truncation() -> None:
+ words = " ".join(f"word{i}" for i in range(80))
+ html = f"{words}
noise
Primary copy here ")
+ root = find_main_content(cleanup_dom(soup), strategy="main_only")
+ text = root.get_text(strip=True)
+ assert "Primary" in text
+ assert "noise" not in text
+
+
+def test_tokenize_words_min_length() -> None:
+ tokens = tokenize_words("I am ok testing")
+ assert "I" not in tokens
+ assert "am" in tokens
diff --git a/tests/test_content_analysis_coverage.py b/tests/test_content_analysis_coverage.py
new file mode 100644
index 00000000..e61be94b
--- /dev/null
+++ b/tests/test_content_analysis_coverage.py
@@ -0,0 +1,163 @@
+"""Additional coverage for content_analysis batch, pipeline, and related paths."""
+from __future__ import annotations
+
+import argparse
+from unittest.mock import MagicMock
+
+import pandas as pd
+import pytest
+
+from website_profiling.content_analysis import batch as ca_batch
+from website_profiling.content_analysis.excerpt import build_excerpt
+from website_profiling.content_analysis.pipeline import run_content_analysis
+from website_profiling.content_analysis.dom_cleanup import cleanup_dom
+from website_profiling.content_analysis.html_loader import load_soup
+from website_profiling.content_analysis.main_content import find_main_content
+from website_profiling.crawl.html_capture import should_store_page_html
+from website_profiling.db import crawl_store as cs
+from website_profiling.parsing.content import _count_syllables, parse_content_text
+
+
+def test_build_excerpt_returns_full_when_under_limit() -> None:
+ assert build_excerpt("short readable text", 100) == "short readable text"
+
+
+def test_cleanup_dom_removes_aria_hidden() -> None:
+ soup = load_soup(
+ 'Keep
Hidden '
+ )
+ cleaned = cleanup_dom(soup)
+ text = cleaned.get_text()
+ assert "Keep" in text
+ assert "Hidden" not in text
+
+
+def test_find_main_content_falls_back_to_body() -> None:
+ soup = load_soup("Only body copy here today.
")
+ root = find_main_content(cleanup_dom(soup), strategy="main_only")
+ assert "Only body copy" in root.get_text()
+
+
+def test_should_store_page_html_rejects_invalid_status() -> None:
+ assert not should_store_page_html(
+ enabled=True,
+ status="not-a-number",
+ content_type="text/html",
+ html="",
+ max_bytes=1000,
+ )
+
+
+def test_parse_content_text_delegates_to_analyzer() -> None:
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup("Hello content world.
", "lxml")
+ out = parse_content_text(soup, "Hello content world.
")
+ assert out["word_count"] > 0
+
+
+def test_count_syllables_wrapper() -> None:
+ assert _count_syllables("hello") >= 1
+
+
+def test_analyze_row_skips_missing_html() -> None:
+ assert ca_batch._analyze_row({"url": "https://x.com", "html": ""}, excerpt_max_chars=0, strategy="main_only") is None
+
+
+def test_analyze_run_html_empty() -> None:
+ assert ca_batch.analyze_run_html(MagicMock(), 1) == []
+
+
+def test_analyze_run_html_single_worker(monkeypatch: pytest.MonkeyPatch) -> None:
+ rows = [
+ {
+ "url": "https://example.com",
+ "html": "hello world content here ",
+ }
+ ]
+ monkeypatch.setattr(ca_batch, "iter_html_pages", lambda *_a, **_k: iter(rows))
+ out = ca_batch.analyze_run_html(MagicMock(), 3, workers=1)
+ assert len(out) == 1
+ assert out[0]["url"] == "https://example.com"
+ assert out[0]["word_count"] > 0
+
+
+def test_analyze_run_html_parallel_workers(monkeypatch: pytest.MonkeyPatch) -> None:
+ rows = [
+ {"url": f"https://example.com/{i}", "html": f"page {i} content words "}
+ for i in range(2)
+ ]
+ monkeypatch.setattr(ca_batch, "iter_html_pages", lambda *_a, **_k: iter(rows))
+ out = ca_batch.analyze_run_html(MagicMock(), 3, workers=2)
+ assert len(out) == 2
+
+
+def test_iter_html_pages_paginates(monkeypatch: pytest.MonkeyPatch) -> None:
+ chunks = [
+ [{"url": "https://a.com", "html": "a"}] * 500,
+ [{"url": "https://b.com", "html": "b"}],
+ ]
+ calls: list[tuple] = []
+
+ def _fake_read(_conn, crawl_run_id, *, limit=5000, offset=0):
+ calls.append((crawl_run_id, limit, offset))
+ idx = offset // 500
+ return iter(chunks[idx] if idx < len(chunks) else [])
+
+ monkeypatch.setattr(ca_batch, "read_page_html_for_run", _fake_read)
+ got = list(ca_batch.iter_html_pages(MagicMock(), 9))
+ assert len(got) == 501
+ assert calls[0] == (9, 500, 0)
+ assert calls[1] == (9, 500, 500)
+
+
+def test_run_content_analysis_skips_when_no_crawl_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ class _Ctx:
+ def __enter__(self):
+ return object()
+
+ def __exit__(self, *_a):
+ return False
+
+ monkeypatch.setattr("website_profiling.content_analysis.pipeline.db_session", lambda: _Ctx())
+ monkeypatch.setattr(
+ "website_profiling.content_analysis.pipeline.get_latest_crawl_run_id",
+ lambda _c: None,
+ )
+ summary = run_content_analysis()
+ assert summary["pages_analyzed"] == 0
+ assert summary["crawl_run_id"] is None
+
+
+def test_merge_crawl_result_fields_batch_skips_invalid_rows(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(cs, "_executemany", lambda *_a, **_k: None)
+ conn = MagicMock()
+ assert cs.merge_crawl_result_fields_batch(conn, 1, []) == 0
+ assert cs.merge_crawl_result_fields_batch(conn, 1, [{"url": ""}]) == 0
+ assert cs.merge_crawl_result_fields_batch(conn, 1, [{"url": "https://x.com"}]) == 0
+
+
+def test_write_crawl_deletes_html_when_table_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+ executed: list[str] = []
+
+ class _Tx:
+ def __enter__(self):
+ return conn
+
+ def __exit__(self, *_a):
+ return False
+
+ class _Conn:
+ def transaction(self):
+ return _Tx()
+
+ def execute(self, sql, params=None):
+ executed.append(sql)
+ if "crawl_page_html" in sql:
+ raise RuntimeError("missing table")
+
+ conn = _Conn()
+ monkeypatch.setattr(cs, "_crawl_rows_from_df", lambda df, run_id: [])
+ monkeypatch.setattr(cs, "_write_crawl_rows", lambda *_a, **_k: None)
+ cs.write_crawl(conn, pd.DataFrame([{"url": "https://a.com"}]), crawl_run_id=5) # type: ignore[arg-type]
+ assert any("crawl_page_html" in sql for sql in executed)
diff --git a/tests/test_content_analysis_pipeline.py b/tests/test_content_analysis_pipeline.py
new file mode 100644
index 00000000..bb328494
--- /dev/null
+++ b/tests/test_content_analysis_pipeline.py
@@ -0,0 +1,104 @@
+"""Tests for merge_crawl_result_fields_batch and content analysis pipeline."""
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from website_profiling.db import crawl_store as cs
+
+
+def test_merge_crawl_result_fields_batch(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[tuple] = []
+
+ def _fake_executemany(conn, sql, params, *, page_size=500):
+ calls.append((sql, params))
+
+ monkeypatch.setattr(cs, "_executemany", _fake_executemany)
+ conn = MagicMock()
+ n = cs.merge_crawl_result_fields_batch(
+ conn,
+ 3,
+ [{"url": "https://example.com/a", "word_count": 120, "top_keywords": "[]"}],
+ commit=True,
+ )
+ assert n == 1
+ conn.commit.assert_called_once()
+ assert len(calls) == 1
+ sql, params = calls[0]
+ assert "crawl_results" in sql
+ assert params[0][1] == 3
+ assert params[0][2] == "https://example.com/a"
+
+
+def test_run_content_analysis_skips_without_database(capsys) -> None:
+ from website_profiling.commands import pipeline_cmd
+
+ pipeline_cmd._run_content_analysis({"store_page_html": True}, False)
+ assert "database required" in capsys.readouterr().out.lower()
+
+
+def test_run_content_analysis_skips_without_html_storage(capsys) -> None:
+ from website_profiling.commands import pipeline_cmd
+
+ pipeline_cmd._run_content_analysis({"store_page_html": False}, True)
+ assert "store_page_html" in capsys.readouterr().out
+
+
+def test_run_content_analysis_runs_when_html_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ from website_profiling.commands import pipeline_cmd
+
+ calls: list[dict] = []
+
+ monkeypatch.setattr(
+ "website_profiling.content_analysis.run_content_analysis",
+ lambda **kwargs: calls.append(kwargs) or {"pages_analyzed": 2},
+ )
+
+ pipeline_cmd._run_content_analysis(
+ {
+ "store_page_html": True,
+ "store_content_excerpt": True,
+ "content_excerpt_max_chars": "512",
+ "content_analysis_strategy": "full_body",
+ "content_analysis_workers": "2",
+ },
+ True,
+ )
+ assert calls[0]["strategy"] == "full_body"
+ assert calls[0]["excerpt_max_chars"] == 512
+ assert calls[0]["workers"] == 2
+
+
+
+def test_run_content_analysis_invokes_batch(monkeypatch: pytest.MonkeyPatch) -> None:
+ from website_profiling.content_analysis import pipeline as ca_pipeline
+
+ calls: list[dict] = []
+
+ class _Ctx:
+ def __enter__(self):
+ return object()
+
+ def __exit__(self, *_a):
+ return False
+
+ monkeypatch.setattr(ca_pipeline, "db_session", lambda: _Ctx())
+ monkeypatch.setattr(ca_pipeline, "get_latest_crawl_run_id", lambda _c: 9)
+ monkeypatch.setattr(
+ ca_pipeline,
+ "analyze_run_html",
+ lambda *_a, **_k: [{"url": "https://ex.com", "word_count": 10, "top_keywords": "[]"}],
+ )
+ monkeypatch.setattr(
+ ca_pipeline,
+ "merge_crawl_result_fields_batch",
+ lambda _c, run_id, updates, commit=True: calls.append({"run_id": run_id, "n": len(updates)}),
+ )
+ monkeypatch.setattr(ca_pipeline, "emit_phase_start", MagicMock())
+ monkeypatch.setattr(ca_pipeline, "emit_phase_done", MagicMock())
+ monkeypatch.setattr(ca_pipeline, "emit_progress", MagicMock())
+
+ summary = ca_pipeline.run_content_analysis(excerpt_max_chars=0, strategy="main_only", workers=1)
+ assert summary["pages_analyzed"] == 1
+ assert calls[0]["run_id"] == 9
diff --git a/tests/test_crawl_db_writer_imports.py b/tests/test_crawl_db_writer_imports.py
index c5b845a8..17c34596 100644
--- a/tests/test_crawl_db_writer_imports.py
+++ b/tests/test_crawl_db_writer_imports.py
@@ -5,6 +5,7 @@ def test_crawl_db_writer_enqueue_and_batch_flush(monkeypatch: pytest.MonkeyPatch
from website_profiling.crawl.crawler import _CrawlDbWriter
written: list[tuple[int, int, bool]] = []
+ html_written: list[tuple[int, int, bool]] = []
class _FakeConn:
pass
@@ -25,15 +26,58 @@ def __exit__(self, _t, _v, _tb):
"website_profiling.db.crawl_store.write_crawl_batch",
lambda _conn, rows, run_id, commit=True: written.append((len(rows), run_id, commit)),
)
+ monkeypatch.setattr(
+ "website_profiling.db.html_store.write_page_html_batch",
+ lambda _conn, rows, run_id, commit=True: html_written.append((len(rows), run_id, commit)),
+ )
- writer = _CrawlDbWriter(crawl_run_id=5, batch_size=50)
+ writer = _CrawlDbWriter(crawl_run_id=5, batch_size=50, store_page_html=True)
for i in range(51):
writer.enqueue({"url": f"https://a.com/{i}"})
+ writer.enqueue_html({"url": "https://a.com/html", "html": "", "status": "200"})
writer.finish()
writer.run()
writer.raise_if_failed()
assert written == [(50, 5, True), (1, 5, True)]
+ assert html_written == [(1, 5, True)]
+
+
+def test_crawl_db_writer_skips_html_when_disabled() -> None:
+ from website_profiling.crawl.crawler import _CrawlDbWriter
+
+ writer = _CrawlDbWriter(crawl_run_id=1, batch_size=50, store_page_html=False)
+ writer.enqueue_html({"url": "https://a.com", "html": ""})
+ writer.finish()
+ writer.run()
+ writer.raise_if_failed()
+
+
+def test_crawl_db_writer_flushes_html_mid_batch(monkeypatch: pytest.MonkeyPatch) -> None:
+ from website_profiling.crawl.crawler import _CrawlDbWriter
+
+ html_written: list[tuple[int, int, bool]] = []
+
+ class _FakeCtx:
+ def __enter__(self):
+ return object()
+
+ def __exit__(self, _t, _v, _tb):
+ return False
+
+ monkeypatch.setattr("website_profiling.db.db_session", lambda: _FakeCtx())
+ monkeypatch.setattr(
+ "website_profiling.db.html_store.write_page_html_batch",
+ lambda _conn, rows, run_id, commit=True: html_written.append((len(rows), run_id, commit)),
+ )
+
+ writer = _CrawlDbWriter(crawl_run_id=2, batch_size=50, store_page_html=True)
+ for i in range(51):
+ writer.enqueue_html({"url": f"https://a.com/{i}", "html": f"{i}"})
+ writer.finish()
+ writer.run()
+ writer.raise_if_failed()
+ assert html_written == [(50, 2, True), (1, 2, True)]
def test_crawl_db_writer_records_run_errors(monkeypatch: pytest.MonkeyPatch) -> None:
diff --git a/tests/test_crawl_frontier.py b/tests/test_crawl_frontier.py
new file mode 100644
index 00000000..4e8b40e9
--- /dev/null
+++ b/tests/test_crawl_frontier.py
@@ -0,0 +1,70 @@
+"""Unit tests for crawl frontier (no network)."""
+
+from website_profiling.crawl.config import CrawlConfig
+from website_profiling.crawl.frontier import CrawlFrontier, url_matches_exclude
+
+
+def test_url_matches_exclude_prefix() -> None:
+ assert url_matches_exclude("https://a.com/blog/post", ["https://a.com/blog"])
+ assert not url_matches_exclude("https://a.com/about", ["https://a.com/blog"])
+ assert url_matches_exclude("https://a.com/blog", ["https://a.com/blog"])
+
+
+def test_enqueue_seed_respects_exclude_and_domain() -> None:
+ frontier = CrawlFrontier(
+ "https://example.com",
+ exclude_urls=["https://example.com/private"],
+ allow_external=False,
+ )
+ frontier.enqueue_seed("https://example.com/page", 0)
+ frontier.enqueue_seed("https://other.com/page", 0)
+ frontier.enqueue_seed("https://example.com/private", 0)
+ assert frontier.queue.qsize() == 1
+ assert frontier.depths["https://example.com/page"] == 0
+
+
+def test_try_enqueue_link_depth_limit() -> None:
+ frontier = CrawlFrontier("https://example.com", max_depth=0, follow_links=True)
+ frontier.depths["https://example.com"] = 0
+ assert frontier.try_enqueue_link("https://example.com/child", "https://example.com") is False
+
+ frontier.max_depth = 1
+ assert frontier.try_enqueue_link("https://example.com/child", "https://example.com") is True
+ assert frontier.depths["https://example.com/child"] == 1
+
+
+def test_mark_visited_dedupes() -> None:
+ frontier = CrawlFrontier("https://example.com")
+ assert frontier.mark_visited("https://example.com/a") is True
+ assert frontier.mark_visited("https://example.com/a") is False
+
+
+def test_try_enqueue_link_skips_already_visited() -> None:
+ frontier = CrawlFrontier("https://example.com", follow_links=True)
+ frontier.depths["https://example.com"] = 0
+ frontier.visited.add("https://example.com/child")
+ assert frontier.try_enqueue_link("https://example.com/child", "https://example.com") is False
+
+
+def test_queue_contains_returns_false_on_queue_error() -> None:
+ class BadDeque:
+ def __iter__(self):
+ raise RuntimeError("broken")
+
+ class BadQueueWrapper:
+ queue = BadDeque()
+
+ frontier = CrawlFrontier("https://example.com")
+ frontier.queue = BadQueueWrapper()
+ assert frontier.queue_contains("https://example.com/x") is False
+
+
+def test_crawl_config_javascript_render_properties() -> None:
+ cfg = CrawlConfig(
+ start_url="https://example.com",
+ render_mode="javascript",
+ js_concurrency=5,
+ concurrency=10,
+ )
+ assert cfg.effective_concurrency == 5
+ assert cfg.fetcher_render_mode == "javascript"
diff --git a/tests/test_crawl_gap_coverage.py b/tests/test_crawl_gap_coverage.py
index aa14403a..2468ae01 100644
--- a/tests/test_crawl_gap_coverage.py
+++ b/tests/test_crawl_gap_coverage.py
@@ -251,7 +251,7 @@ def test_crawler_robots_txt_override_parses(monkeypatch) -> None:
lambda *_a, **_k: [],
)
monkeypatch.setattr(
- "website_profiling.crawl.crawler.load_robots",
+ "website_profiling.crawl.frontier.load_robots",
lambda _u: (_ for _ in ()).throw(AssertionError("load_robots should not run")),
)
c = Crawler(
@@ -269,7 +269,7 @@ def test_crawler_loads_robots_when_no_override(monkeypatch) -> None:
"website_profiling.crawl.sitemap.discover_sitemap_urls",
lambda *_a, **_k: [],
)
- monkeypatch.setattr("website_profiling.crawl.crawler.load_robots", lambda _u: object())
+ monkeypatch.setattr("website_profiling.crawl.frontier.load_robots", lambda _u: object())
c = Crawler(start_url="https://site.com", ignore_robots=False)
assert c.rp is not None
diff --git a/tests/test_crawl_html_storage.py b/tests/test_crawl_html_storage.py
new file mode 100644
index 00000000..00404e57
--- /dev/null
+++ b/tests/test_crawl_html_storage.py
@@ -0,0 +1,126 @@
+"""Tests for HTML capture during crawl."""
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from website_profiling.crawl.html_capture import build_page_html_record, should_store_page_html
+
+
+def test_should_store_page_html_accepts_200_html() -> None:
+ assert should_store_page_html(
+ enabled=True,
+ status=200,
+ content_type="text/html; charset=utf-8",
+ html="ok",
+ max_bytes=10_000,
+ )
+
+
+def test_should_store_page_html_rejects_when_disabled() -> None:
+ assert not should_store_page_html(
+ enabled=False,
+ status=200,
+ content_type="text/html",
+ html="",
+ max_bytes=10_000,
+ )
+
+
+def test_should_store_page_html_rejects_404() -> None:
+ assert not should_store_page_html(
+ enabled=True,
+ status=404,
+ content_type="text/html",
+ html="",
+ max_bytes=10_000,
+ )
+
+
+def test_should_store_page_html_rejects_non_html_mime() -> None:
+ assert not should_store_page_html(
+ enabled=True,
+ status=200,
+ content_type="application/json",
+ html='{"a":1}',
+ max_bytes=10_000,
+ )
+
+
+def test_should_store_page_html_rejects_oversized() -> None:
+ html = "x" * 100
+ assert not should_store_page_html(
+ enabled=True,
+ status=200,
+ content_type="text/html",
+ html=html,
+ max_bytes=50,
+ )
+
+
+def test_build_page_html_record_normalizes_url() -> None:
+ rec = build_page_html_record(
+ url="https://example.com/page/",
+ html="hi",
+ status=200,
+ content_type="text/html",
+ fetch_method="rendered",
+ max_bytes=5000,
+ )
+ assert rec is not None
+ assert rec["url"] == "https://example.com/page"
+ assert rec["fetch_method"] == "rendered"
+ assert rec["byte_length"] == len("hi".encode("utf-8"))
+
+
+def test_crawler_capture_uses_buffer_when_no_db_writer(monkeypatch: pytest.MonkeyPatch) -> None:
+ from website_profiling.crawl.crawler import Crawler
+
+ monkeypatch.setattr(
+ "website_profiling.crawl.crawler.build_fetcher",
+ lambda **kwargs: MagicMock(fetch=MagicMock()),
+ )
+ monkeypatch.setattr(
+ "website_profiling.crawl.frontier.CrawlFrontier.seed_initial_urls",
+ lambda *args, **kwargs: None,
+ )
+
+ crawler = Crawler(
+ start_url="https://example.com",
+ max_pages=1,
+ store_page_html=True,
+ max_stored_html_bytes=50_000,
+ )
+ crawler._capture_page_html(
+ "https://example.com/a",
+ "text",
+ 200,
+ "text/html",
+ "static",
+ )
+ assert len(crawler._html_buffer) == 1
+ assert crawler._html_buffer[0]["url"] == "https://example.com/a"
+
+
+def test_crawler_skips_html_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ from website_profiling.crawl.crawler import Crawler
+
+ monkeypatch.setattr(
+ "website_profiling.crawl.crawler.build_fetcher",
+ lambda **kwargs: MagicMock(fetch=MagicMock()),
+ )
+ monkeypatch.setattr(
+ "website_profiling.crawl.frontier.CrawlFrontier.seed_initial_urls",
+ lambda *args, **kwargs: None,
+ )
+
+ crawler = Crawler(start_url="https://example.com", max_pages=1, store_page_html=False)
+ crawler._capture_page_html(
+ "https://example.com/a",
+ "text",
+ 200,
+ "text/html",
+ "static",
+ )
+ assert crawler._html_buffer == []
diff --git a/tests/test_crawler_deep.py b/tests/test_crawler_deep.py
index fe3e2d99..1f8c58e1 100644
--- a/tests/test_crawler_deep.py
+++ b/tests/test_crawler_deep.py
@@ -8,6 +8,7 @@
def test_worker_success_path_populates_many_fields(monkeypatch):
import website_profiling.crawl.crawler as mod
+ import website_profiling.crawl.page_record as pr_mod
from website_profiling.crawl.fetchers.base import FetchResult
monkeypatch.setattr(
@@ -34,7 +35,7 @@ def test_worker_success_path_populates_many_fields(monkeypatch):
fetch_method="static",
)
monkeypatch.setattr(
- mod,
+ pr_mod,
"parse_link_edges",
lambda _u, _t: (
"T",
@@ -44,9 +45,9 @@ def test_worker_success_path_populates_many_fields(monkeypatch):
],
),
)
- monkeypatch.setattr(mod, "parse_seo", lambda *_a, **_k: ("desc", 4, "h1", 1, "https://site.com/canon"))
+ monkeypatch.setattr(pr_mod, "parse_seo", lambda *_a, **_k: ("desc", 4, "h1", 1, "https://site.com/canon"))
monkeypatch.setattr(
- mod,
+ pr_mod,
"parse_seo_extended",
lambda *_a, **_k: {
"viewport_present": True,
@@ -62,9 +63,9 @@ def test_worker_success_path_populates_many_fields(monkeypatch):
"mixed_content_count": 0,
},
)
- monkeypatch.setattr(mod, "parse_resources", lambda *_a, **_k: {"script_count": 1, "link_stylesheet_count": 1})
+ monkeypatch.setattr(pr_mod, "parse_resources", lambda *_a, **_k: {"script_count": 1, "link_stylesheet_count": 1})
monkeypatch.setattr(
- mod,
+ pr_mod,
"parse_content_text",
lambda *_a, **_k: {
"word_count": 10,
@@ -75,7 +76,7 @@ def test_worker_success_path_populates_many_fields(monkeypatch):
},
)
monkeypatch.setattr(
- mod,
+ pr_mod,
"parse_social_meta",
lambda *_a, **_k: {
"og_title": "og",
@@ -87,8 +88,8 @@ def test_worker_success_path_populates_many_fields(monkeypatch):
"twitter_image": "",
},
)
- monkeypatch.setattr(mod, "parse_tech_stack", lambda *_a, **_k: "[]")
- monkeypatch.setattr(mod, "analyze_html", lambda *_a, **_k: {"ok": True})
+ monkeypatch.setattr(pr_mod, "parse_tech_stack", lambda *_a, **_k: "[]")
+ monkeypatch.setattr(pr_mod, "analyze_html", lambda *_a, **_k: {"ok": True})
out = c.worker("https://site.com")
assert out["status"] == 200
@@ -358,9 +359,10 @@ def test_crawl_streams_rows_to_db_writer(monkeypatch):
class FakeDbWriter:
instances: list["FakeDbWriter"] = []
- def __init__(self, crawl_run_id: int, batch_size: int) -> None:
+ def __init__(self, crawl_run_id: int, batch_size: int, *, store_page_html: bool = False) -> None:
self.crawl_run_id = crawl_run_id
self.batch_size = batch_size
+ self.store_page_html = store_page_html
self.enqueued: list[dict] = []
self.started = False
self.finished = False
@@ -539,3 +541,82 @@ def __exit__(self, _t, _v, _tb):
df = mod.run_crawler("https://a.com", output_db=True, crawl_stream_to_db=True, show_progress=False)
assert not df.empty
+
+def test_run_crawler_flushes_buffered_html(monkeypatch):
+ import website_profiling.crawl.crawler as mod
+
+ class FakeCrawler:
+ def __init__(self, **_kwargs):
+ self.link_edges_accum = []
+ self.store_page_html = True
+ self._html_buffer = [
+ {"url": "https://a.com", "html": "", "status": "200"},
+ ]
+
+ def crawl(self, **_kwargs):
+ return pd.DataFrame([{"url": "https://a.com", "status": 200}])
+
+ class _Ctx:
+ def __enter__(self):
+ return object()
+
+ def __exit__(self, _t, _v, _tb):
+ return False
+
+ html_writes: list[tuple] = []
+
+ fake_db = types.SimpleNamespace(
+ backup_db_if_exists=lambda: None,
+ create_crawl_run=lambda *_a, **_k: 7,
+ db_session=lambda: _Ctx(),
+ read_historical_data=lambda: {},
+ restore_historical_data=lambda *_a, **_k: None,
+ write_crawl=lambda *_a, **_k: None,
+ )
+ fake_storage = types.SimpleNamespace(ensure_crawl_tables_cleared=lambda *_a, **_k: None)
+ monkeypatch.setattr(mod, "Crawler", FakeCrawler)
+ monkeypatch.setitem(__import__("sys").modules, "website_profiling.db", fake_db)
+ monkeypatch.setitem(__import__("sys").modules, "website_profiling.db.storage", fake_storage)
+ monkeypatch.setattr(
+ "website_profiling.db.html_store.write_page_html_batch",
+ lambda _conn, rows, run_id, commit=True: html_writes.append((len(rows), run_id)),
+ )
+
+ mod.run_crawler(
+ "https://a.com",
+ output_db=True,
+ preserve_crawl_history=False,
+ show_progress=False,
+ store_page_html=True,
+ )
+ assert html_writes == [(1, 7)]
+
+
+def test_capture_page_html_enqueues_to_stream_writer():
+ import website_profiling.crawl.crawler as mod
+
+ class _Writer:
+ def __init__(self):
+ self.records: list[dict] = []
+
+ def enqueue_html(self, record: dict) -> None:
+ self.records.append(record)
+
+ crawler = mod.Crawler(
+ start_url="https://site.com",
+ ignore_robots=True,
+ store_page_html=True,
+ max_pages=1,
+ )
+ writer = _Writer()
+ crawler._db_writer = writer
+ crawler._capture_page_html(
+ "https://site.com",
+ "Hello",
+ 200,
+ "text/html",
+ "static",
+ )
+ assert len(writer.records) == 1
+ assert writer.records[0]["url"] == "https://site.com"
+
diff --git a/tests/test_crawler_unit.py b/tests/test_crawler_unit.py
index 339e1350..06717b3e 100644
--- a/tests/test_crawler_unit.py
+++ b/tests/test_crawler_unit.py
@@ -8,7 +8,7 @@ def _mock_sitemap_unless_seeding_test(monkeypatch, request):
if request.node.name == "test_crawler_seeds_sitemap_urls":
return
monkeypatch.setattr(
- "website_profiling.crawl.crawler.discover_sitemap_urls",
+ "website_profiling.crawl.sitemap.discover_sitemap_urls",
lambda *_a, **_k: [],
)
@@ -45,7 +45,7 @@ def test_crawler_sitemap_seed_exception_is_ignored(monkeypatch) -> None:
def _boom(*_a, **_k):
raise RuntimeError("sitemap unavailable")
- monkeypatch.setattr("website_profiling.crawl.crawler.discover_sitemap_urls", _boom)
+ monkeypatch.setattr("website_profiling.crawl.sitemap.discover_sitemap_urls", _boom)
c = Crawler(start_url="https://site.com", ignore_robots=True, use_wappalyzer=False)
assert c.queue.qsize() == 1
@@ -54,7 +54,7 @@ def test_crawler_sitemap_seed_filters_exclude_external_and_duplicates(monkeypatc
from website_profiling.crawl.crawler import Crawler
monkeypatch.setattr(
- "website_profiling.crawl.crawler.discover_sitemap_urls",
+ "website_profiling.crawl.sitemap.discover_sitemap_urls",
lambda *_a, **_k: [
"https://site.com/skip-me",
"https://external.com/page",
@@ -81,7 +81,7 @@ def test_crawler_seeds_sitemap_urls(monkeypatch) -> None:
from website_profiling.crawl.crawler import Crawler
monkeypatch.setattr(
- "website_profiling.crawl.crawler.discover_sitemap_urls",
+ "website_profiling.crawl.sitemap.discover_sitemap_urls",
lambda *_a, **_k: ["https://site.com/from-sitemap"],
)
c = Crawler(
@@ -300,7 +300,7 @@ def test_worker_uses_wappalyzer_when_enabled(monkeypatch) -> None:
html = "T ok"
monkeypatch.setattr(
- "website_profiling.crawl.crawler.detect_tech_wappalyzer",
+ "website_profiling.crawl.page_record.detect_tech_wappalyzer",
lambda *_a, **_k: '["React"]',
)
c = Crawler(
diff --git a/tests/test_google_store_full.py b/tests/test_google_store_full.py
new file mode 100644
index 00000000..6c6c802a
--- /dev/null
+++ b/tests/test_google_store_full.py
@@ -0,0 +1,22 @@
+"""Tests for full Google data blob reader."""
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock
+
+from website_profiling.integrations.google.store import read_google_data_full
+
+
+def test_read_google_data_full_returns_blob() -> None:
+ blob = {"gsc_full": {"by_page": []}, "ga4_full": {"by_path": []}}
+ conn = MagicMock()
+ conn.execute.return_value.fetchone.return_value = {"data": blob}
+ result = read_google_data_full(conn, property_id=1)
+ assert result == blob
+
+
+def test_read_google_data_full_none_when_missing() -> None:
+ conn = MagicMock()
+ conn.execute.return_value.fetchone.return_value = None
+ result = read_google_data_full(conn, property_id=1)
+ assert result is None
diff --git a/tests/test_historical_keywords_crawl_store_unit.py b/tests/test_historical_keywords_crawl_store_unit.py
index 48b7fb44..c9e86eb3 100644
--- a/tests/test_historical_keywords_crawl_store_unit.py
+++ b/tests/test_historical_keywords_crawl_store_unit.py
@@ -191,7 +191,7 @@ def test_ensure_crawl_tables_cleared_commits():
conn = _Conn()
ensure_crawl_tables_cleared(conn) # type: ignore[arg-type]
assert conn.commits == 1
- assert any("TRUNCATE crawl_results, edges, nodes" in s for s, _ in conn.executed)
+ assert any("TRUNCATE crawl_results, crawl_page_html, edges, nodes" in s for s, _ in conn.executed)
def test_read_lh_runs_by_url_and_page_summaries():
diff --git a/tests/test_html_store.py b/tests/test_html_store.py
new file mode 100644
index 00000000..5772abdc
--- /dev/null
+++ b/tests/test_html_store.py
@@ -0,0 +1,151 @@
+"""Tests for crawl_page_html storage."""
+from __future__ import annotations
+
+import pytest
+
+from website_profiling.db import html_store as hs
+
+
+class _Cursor:
+ def __init__(self, rows=None):
+ self._rows = rows or []
+ self.executemany_calls: list[tuple] = []
+
+ def executemany(self, sql, params):
+ self.executemany_calls.append((sql, params))
+
+ def fetchall(self):
+ return list(self._rows)
+
+ def fetchone(self):
+ return self._rows[0] if self._rows else None
+
+
+class _Conn:
+ def __init__(self, select_rows=None):
+ self.select_rows = select_rows or []
+ self.executed: list[tuple] = []
+ self.commits = 0
+ self.executemany_calls: list[tuple] = []
+ self._cursor = _Cursor()
+
+ def execute(self, sql, params=None):
+ self.executed.append((sql, params))
+ if "SELECT" in sql.upper():
+ self._cursor = _Cursor(self.select_rows)
+ return self._cursor
+
+ def commit(self):
+ self.commits += 1
+
+ def cursor(self):
+ cur = _Cursor()
+ cur.executemany_calls = self.executemany_calls
+
+ class _CM:
+ def __enter__(self_inner):
+ return cur
+
+ def __exit__(self_inner, _t, _v, _tb):
+ return False
+
+ return _CM()
+
+
+def test_write_page_html_batch_builds_rows_and_commits() -> None:
+ conn = _Conn()
+ records = [
+ {
+ "url": "https://example.com/page",
+ "html": "Hello",
+ "status": "200",
+ "content_type": "text/html",
+ "fetch_method": "rendered",
+ "byte_length": 32,
+ }
+ ]
+ hs.write_page_html_batch(conn, records, crawl_run_id=7, commit=True)
+ assert conn.commits == 1
+ assert len(conn.executemany_calls) == 1
+ _sql, params = conn.executemany_calls[0]
+ assert "crawl_page_html" in _sql
+ assert params[0][0] == 7
+ assert params[0][1] == "https://example.com/page"
+
+
+def test_write_page_html_batch_skips_empty_html() -> None:
+ conn = _Conn()
+ hs.write_page_html_batch(conn, [{"url": "https://x.com", "html": ""}], crawl_run_id=1)
+ assert conn.commits == 0
+ assert not conn.executed
+
+
+def test_read_page_html_returns_row() -> None:
+ row = {
+ "url": "https://example.com",
+ "html": "",
+ "status": "200",
+ "content_type": "text/html",
+ "fetch_method": "static",
+ "byte_length": 13,
+ "captured_at": "2026-01-01",
+ }
+ conn = _Conn(select_rows=[row])
+ out = hs.read_page_html(conn, 3, "https://example.com/")
+ assert out is not None
+ assert out["html"] == ""
+
+
+def test_read_page_html_for_run_yields_rows() -> None:
+ rows = [
+ {"url": "https://a.com", "html": "a", "status": "200",
+ "content_type": "text/html", "fetch_method": "static", "byte_length": 1, "captured_at": "t"},
+ ]
+ conn = _Conn(select_rows=rows)
+ got = list(hs.read_page_html_for_run(conn, 1, limit=10))
+ assert len(got) == 1
+ assert got[0]["url"] == "https://a.com"
+
+
+def test_delete_page_html_for_run() -> None:
+ conn = _Conn()
+ hs.delete_page_html_for_run(conn, 9, commit=True)
+ assert conn.commits == 1
+ assert any("DELETE FROM crawl_page_html" in sql for sql, _ in conn.executed)
+
+
+def test_read_page_html_empty_url_returns_none() -> None:
+ conn = _Conn()
+ assert hs.read_page_html(conn, 1, "") is None
+
+
+def test_read_page_html_returns_none_when_missing() -> None:
+ conn = _Conn(select_rows=[])
+ assert hs.read_page_html(conn, 3, "https://missing.example") is None
+
+
+def test_read_page_html_handles_db_error() -> None:
+ class _BadConn:
+ def execute(self, *_a, **_k):
+ raise RuntimeError("db down")
+
+ assert hs.read_page_html(_BadConn(), 1, "https://example.com") is None # type: ignore[arg-type]
+
+
+def test_read_page_html_for_run_handles_db_error() -> None:
+ class _BadConn:
+ def execute(self, *_a, **_k):
+ raise RuntimeError("db down")
+
+ assert list(hs.read_page_html_for_run(_BadConn(), 1)) == [] # type: ignore[arg-type]
+
+
+def test_delete_page_html_for_run_handles_db_error() -> None:
+ class _BadConn:
+ def execute(self, *_a, **_k):
+ raise RuntimeError("db down")
+
+ def commit(self):
+ raise RuntimeError("should not commit")
+
+ hs.delete_page_html_for_run(_BadConn(), 1, commit=True) # type: ignore[arg-type]
diff --git a/tests/test_log_parser.py b/tests/test_log_parser.py
index 8d32165d..c2add630 100644
--- a/tests/test_log_parser.py
+++ b/tests/test_log_parser.py
@@ -10,6 +10,14 @@ def test_parse_access_log_lines_counts_googlebot() -> None:
assert out["top_paths"][0]["path"] == "/page"
+def test_parse_access_log_lines_tracks_5xx_paths() -> None:
+ lines = [
+ '127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /broken HTTP/1.1" 503 0 "-" "curl/8.0"',
+ ]
+ out = parse_access_log_lines(lines)
+ assert out["paths_5xx"] == [{"path": "/broken", "hits": 1}]
+
+
def test_parse_access_log_lines_skips_blank_and_comments() -> None:
lines = ["", "# comment", "not-a-log-line"]
out = parse_access_log_lines(lines)
diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py
index 67d8fded..c693d500 100644
--- a/tests/test_mcp_registry.py
+++ b/tests/test_mcp_registry.py
@@ -3,11 +3,17 @@
from unittest.mock import MagicMock, patch
-from website_profiling.tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool
+from website_profiling.tools.audit_tools.registry import (
+ TOOL_DEFINITIONS,
+ dispatch_tool,
+ search_tools,
+ validate_tool_registry,
+)
+from website_profiling.tools.audit_tools.tool_domains import TIER_0_TOOLS
def test_tool_definitions_schema() -> None:
- assert len(TOOL_DEFINITIONS) == 221
+ assert len(TOOL_DEFINITIONS) == 340
for tool in TOOL_DEFINITIONS:
assert tool.get("name")
assert tool.get("description")
@@ -16,6 +22,22 @@ def test_tool_definitions_schema() -> None:
assert schema.get("type") == "object"
+def test_validate_tool_registry() -> None:
+ assert validate_tool_registry() == []
+
+
+def test_tier0_tools_have_handlers() -> None:
+ from website_profiling.tools.audit_tools.registry import tool_handler_names
+
+ assert TIER_0_TOOLS <= tool_handler_names()
+
+
+def test_search_tools_finds_broken_links() -> None:
+ matches = search_tools("broken links", limit=5)
+ names = {m["name"] for m in matches}
+ assert "list_broken_links" in names or "list_broken_link_sources" in names
+
+
def test_dispatch_list_properties_roundtrip() -> None:
conn = MagicMock()
props = [{"id": 1, "name": "ex.com", "canonical_domain": "ex.com"}]
diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py
index d092518b..ca2c1428 100644
--- a/tests/test_mcp_server_helpers.py
+++ b/tests/test_mcp_server_helpers.py
@@ -60,23 +60,26 @@ def test_read_glossary_excerpt_missing(monkeypatch) -> None:
def test_tools_catalog_json_includes_security_tools() -> None:
- catalog = json.loads(mcp_server._tools_catalog_json())
- assert catalog["tool_count"] >= 221
+ with patch.dict(os.environ, {"WP_MCP_DOMAIN": "full"}):
+ catalog = json.loads(mcp_server._tools_catalog_json())
+ assert catalog["tool_count"] >= 340
assert "get_security_findings" in catalog["domains"]["security"]
assert "get_geo_readiness_score" in catalog["domains"]["geo"]
assert "get_gsc_url_inspection" in catalog["domains"]["integrations"]
+ assert catalog["mcp_domain"] == "full"
def test_tools_catalog_json_backlinks_domain() -> None:
- fake_tools = [
- {
- "name": "get_bing_overview",
- "description": "Bing overview without link in name.",
- "inputSchema": {"type": "object", "properties": {}},
- },
- ]
- with patch("website_profiling.mcp.server.TOOL_DEFINITIONS", fake_tools):
- catalog = json.loads(mcp_server._tools_catalog_json())
+ with patch.dict(os.environ, {"WP_MCP_DOMAIN": "full"}):
+ with patch(
+ "website_profiling.mcp.server.mcp_tool_names",
+ return_value={"get_bing_overview"},
+ ):
+ with patch(
+ "website_profiling.mcp.server.tools_catalog_by_domain",
+ return_value={"backlinks": ["get_bing_overview"]},
+ ):
+ catalog = json.loads(mcp_server._tools_catalog_json())
assert catalog["domains"]["backlinks"] == ["get_bing_overview"]
@@ -164,21 +167,104 @@ async def __aexit__(self, *_args):
monkeypatch.setitem(sys.modules, "mcp.server.stdio", fake_stdio_mod)
monkeypatch.setitem(sys.modules, "mcp.types", fake_types_mod)
- with patch.dict(os.environ, {"WP_PROPERTY_ID": "7"}, clear=False):
+ with patch.dict(os.environ, {"WP_PROPERTY_ID": "7", "WP_MCP_DOMAIN": "full"}, clear=False):
mcp_server.main()
- assert captured["name"] == "site-audit"
+ assert captured["name"] == "site-audit-full"
assert captured["ran"] is True
tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type]
- assert len(tools) >= 221
+ assert len(tools) >= 340
resources = asyncio.run(captured["list_resources"]()) # type: ignore[arg-type]
assert any(r["uri"] == "audit://property/7" for r in resources)
+ assert any(r["uri"] == "audit://domains" for r in resources)
with patch("website_profiling.mcp.server.dispatch_tool", return_value={"ok": True}):
content = asyncio.run(captured["call_tool"]("list_properties", {"property_id": 1})) # type: ignore[arg-type]
assert content[0]["text"] == json.dumps({"ok": True}, indent=2, default=str)
read_text = asyncio.run(captured["read_resource"]("audit://tools")) # type: ignore[arg-type]
assert read_text.startswith("{")
+ domains_text = asyncio.run(captured["read_resource"]("audit://domains")) # type: ignore[arg-type]
+ assert "current_mcp_domain" in domains_text
+
+
+def test_mcp_call_tool_rejects_tools_outside_domain(monkeypatch) -> None:
+ captured: dict[str, object] = {}
+
+ class FakeServer:
+ def __init__(self, name: str) -> None:
+ captured["name"] = name
+
+ def list_tools(self):
+ def decorator(fn):
+ captured["list_tools"] = fn
+ return fn
+ return decorator
+
+ def call_tool(self):
+ def decorator(fn):
+ captured["call_tool"] = fn
+ return fn
+ return decorator
+
+ def list_resources(self):
+ def decorator(fn):
+ return fn
+ return decorator
+
+ def read_resource(self):
+ def decorator(fn):
+ return fn
+ return decorator
+
+ def create_initialization_options(self):
+ return {}
+
+ async def run(self, *_args, **_kwargs) -> None:
+ return None
+
+ class FakeStdioCM:
+ async def __aenter__(self):
+ return (MagicMock(), MagicMock())
+
+ async def __aexit__(self, *_args):
+ return False
+
+ fake_server_mod = MagicMock()
+ fake_server_mod.Server = FakeServer
+ fake_stdio_mod = MagicMock()
+ fake_stdio_mod.stdio_server = MagicMock(return_value=FakeStdioCM())
+ fake_types_mod = MagicMock()
+ fake_types_mod.TextContent = lambda **kwargs: kwargs
+ fake_types_mod.Resource = lambda **kwargs: kwargs
+ fake_types_mod.Tool = lambda **kwargs: kwargs
+
+ monkeypatch.setitem(sys.modules, "mcp", MagicMock())
+ monkeypatch.setitem(sys.modules, "mcp.server", fake_server_mod)
+ monkeypatch.setitem(sys.modules, "mcp.server.stdio", fake_stdio_mod)
+ monkeypatch.setitem(sys.modules, "mcp.types", fake_types_mod)
+
+ with patch.dict(os.environ, {"WP_MCP_DOMAIN": "core"}, clear=False):
+ mcp_server.main()
+
+ tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type]
+ assert len(tools) < 340
+ blocked = asyncio.run(captured["call_tool"]("export_audit_report", {"format": "pdf"})) # type: ignore[arg-type]
+ assert "not exposed" in blocked[0]["text"]
+
+
+def test_mcp_core_server_main() -> None:
+ with patch("website_profiling.mcp.core_server.run_domain_server") as mock_run:
+ from website_profiling.mcp import core_server
+ core_server.main()
+ mock_run.assert_called_once_with("core")
+
+
+def test_mcp_domain_server_sets_env() -> None:
+ with patch("website_profiling.mcp.domain_server.main") as mock_main:
+ from website_profiling.mcp.domain_server import run_domain_server
+ run_domain_server("google")
+ assert os.environ.get("WP_MCP_DOMAIN") == "google"
+ mock_main.assert_called_once()
def test_mcp_package_main(monkeypatch) -> None:
diff --git a/tests/test_page_record.py b/tests/test_page_record.py
new file mode 100644
index 00000000..cbd8dbab
--- /dev/null
+++ b/tests/test_page_record.py
@@ -0,0 +1,50 @@
+"""Unit tests for crawl page record builder (no network)."""
+
+from website_profiling.crawl.page_record import PageRecordBuilder
+from website_profiling.crawl.schema import CRAWL_ROW_COLUMNS, empty_crawl_row, empty_crawl_row_ext
+
+
+def test_empty_crawl_row_has_core_fields() -> None:
+ row = empty_crawl_row(url="https://a.com", status=200)
+ assert row["url"] == "https://a.com"
+ assert row["status"] == 200
+ assert row["fetch_method"] == "static"
+ assert row["page_analysis"] == "{}"
+
+
+def test_empty_crawl_row_ext_headers() -> None:
+ ext = empty_crawl_row_ext(
+ "https://a.com",
+ headers_dict={"Cache-Control": "no-cache", "X-Robots-Tag": "noindex"},
+ )
+ assert ext["cache_control"] == "no-cache"
+ assert ext["x_robots_tag"] == "noindex"
+
+
+def test_crawl_row_columns_match_dataframe_schema() -> None:
+ assert "url" in CRAWL_ROW_COLUMNS
+ assert "fetch_method" in CRAWL_ROW_COLUMNS
+ assert "page_analysis" in CRAWL_ROW_COLUMNS
+
+
+def test_build_robots_blocked_row() -> None:
+ row = PageRecordBuilder.build_robots_blocked_row(
+ "https://a.com", store_outlinks=True
+ )
+ assert row["status"] == "blocked_by_robots"
+ assert row["outlink_targets"] == "[]"
+
+
+def test_parse_page_content_minimal_html() -> None:
+ builder = PageRecordBuilder(use_wappalyzer=False)
+ html = "Hi Hello "
+ parsed = builder.parse_page_content(
+ "https://a.com",
+ html,
+ "https://a.com",
+ {},
+ 0,
+ )
+ assert parsed["title"] == "Hi"
+ assert parsed["h1_text"] == "Hello"
+ assert parsed["h1_count"] == 1
diff --git a/tests/test_pipeline_cmd_run_unit.py b/tests/test_pipeline_cmd_run_unit.py
index 872dae05..08b57a75 100644
--- a/tests/test_pipeline_cmd_run_unit.py
+++ b/tests/test_pipeline_cmd_run_unit.py
@@ -352,3 +352,49 @@ def test_lighthouse_main_prints_unicode_summary_on_cp1252(monkeypatch, tmp_path)
output = buffer.getvalue().decode("utf-8", errors="replace")
assert "2500" in output
+
+def test_pipeline_run_content_analysis_command(monkeypatch) -> None:
+ from website_profiling.commands import pipeline_cmd
+
+ called = {"content": 0}
+
+ monkeypatch.setattr(pipeline_cmd, "_run_crawl", lambda *_a, **_k: None)
+ monkeypatch.setattr(pipeline_cmd, "_run_report", lambda *_a, **_k: None)
+ monkeypatch.setattr(pipeline_cmd, "_run_plot", lambda *_a, **_k: None)
+ monkeypatch.setattr(
+ pipeline_cmd,
+ "_run_content_analysis",
+ lambda *_a, **_k: called.__setitem__("content", called["content"] + 1),
+ )
+
+ cfg = {
+ "start_url": "https://site.com",
+ "run_crawl": "false",
+ "run_content_analysis": "true",
+ "store_page_html": "true",
+ "run_report": "false",
+ "run_plot": "false",
+ }
+ pipeline_cmd.run(cfg, argparse.Namespace(command="content_analysis"))
+ assert called["content"] == 1
+
+
+def test_pipeline_lists_content_analysis_in_steps(monkeypatch, capsys) -> None:
+ from website_profiling.commands import pipeline_cmd
+
+ monkeypatch.setattr(pipeline_cmd, "_run_crawl", lambda *_a, **_k: None)
+ monkeypatch.setattr(pipeline_cmd, "_run_content_analysis", lambda *_a, **_k: None)
+ monkeypatch.setattr(pipeline_cmd, "_run_report", lambda *_a, **_k: None)
+ monkeypatch.setattr(pipeline_cmd, "_run_plot", lambda *_a, **_k: None)
+
+ cfg = {
+ "start_url": "https://site.com",
+ "run_crawl": "false",
+ "run_content_analysis": "true",
+ "store_page_html": "true",
+ "run_report": "false",
+ "run_plot": "false",
+ }
+ pipeline_cmd.run(cfg, argparse.Namespace(command=None))
+ assert "content-analysis" in capsys.readouterr().out
+
diff --git a/tests/test_property_google.py b/tests/test_property_google.py
index 4e7df8f5..72843dea 100644
--- a/tests/test_property_google.py
+++ b/tests/test_property_google.py
@@ -1,6 +1,8 @@
"""Per-property Google resolution and scoped google_data."""
from __future__ import annotations
+from unittest.mock import patch
+
from website_profiling.db.property_store import canonical_domain_from_start_url
@@ -26,3 +28,44 @@ def test_build_credentials_accepts_property_id():
sig = inspect.signature(build_credentials)
assert "property_id" in sig.parameters
+
+
+def test_build_credentials_property_service_account_uses_app_sa():
+ from website_profiling.integrations.google.auth import build_credentials
+
+ fake_sa = {"type": "service_account", "project_id": "p"}
+ with (
+ patch(
+ "website_profiling.integrations.google.auth._property_google_auth",
+ return_value=("", "service_account", "example.com"),
+ ),
+ patch(
+ "website_profiling.db.google_app_store.build_service_account_credentials",
+ return_value={"ok": True},
+ ) as mock_sa,
+ ):
+ creds = build_credentials(property_id=42)
+ assert creds == {"ok": True}
+ mock_sa.assert_called_once()
+
+
+def test_build_credentials_property_no_oauth_falls_back_to_app_sa():
+ from website_profiling.integrations.google.auth import build_credentials
+
+ with (
+ patch(
+ "website_profiling.integrations.google.auth._property_google_auth",
+ return_value=("", "oauth", "example.com"),
+ ),
+ patch(
+ "website_profiling.db.google_app_store.has_service_account",
+ return_value=True,
+ ),
+ patch(
+ "website_profiling.db.google_app_store.build_service_account_credentials",
+ return_value={"sa": True},
+ ) as mock_sa,
+ ):
+ creds = build_credentials(property_id=7)
+ assert creds == {"sa": True}
+ mock_sa.assert_called_once()
diff --git a/tests/test_property_profile.py b/tests/test_property_profile.py
index 684d9e41..b7929765 100644
--- a/tests/test_property_profile.py
+++ b/tests/test_property_profile.py
@@ -553,7 +553,7 @@ def fake_get(url, timeout=8):
session = MagicMock()
session.get.side_effect = fake_get
- monkeypatch.setattr("website_profiling.reporting.builder.requests.Session", lambda: session)
+ monkeypatch.setattr("website_profiling.reporting.site_level.requests.Session", lambda: session)
out = _fetch_site_level("https://example.com/", timeout=1)
assert out["ads_txt_present"] is True
assert out["security_txt_present"] is True
diff --git a/tests/test_reporting_builder_modules.py b/tests/test_reporting_builder_modules.py
new file mode 100644
index 00000000..97206895
--- /dev/null
+++ b/tests/test_reporting_builder_modules.py
@@ -0,0 +1,849 @@
+"""Coverage for reporting modules split from builder.py (reporting gate)."""
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from website_profiling.reporting import content_analytics, edges_report, lighthouse_report, report_metadata, seo_summary, site_level
+
+
+def _crawl_df() -> pd.DataFrame:
+ return pd.DataFrame(
+ [
+ {
+ "url": "https://example.com/",
+ "status": "200",
+ "title": "A" * 45,
+ "meta_description_len": 100,
+ "meta_description": "desc",
+ "h1_count": 1,
+ "h1": "Heading",
+ "content_length": 500,
+ "word_count": 250,
+ "reading_level": 10,
+ "content_html_ratio": 25,
+ "outlinks": 3,
+ "depth": 0,
+ "response_time_ms": 2500,
+ "content_type": "text/html",
+ "og_title": "OG",
+ "twitter_card": "summary",
+ "og_image": "https://cdn.example.com/img.png",
+ "tech_stack": json.dumps(["WordPress", "nginx"]),
+ "top_keywords": json.dumps([{"word": "seo audit", "count": 4}]),
+ "heading_sequence": "h1,h2",
+ "script_count": 2,
+ "link_stylesheet_count": 1,
+ "page_analysis": json.dumps(
+ {
+ "html_lang": "en",
+ "hreflang_alternates": [{"hreflang": "en", "href": "https://example.com/"}],
+ "external_links": ["https://partner.com/page"],
+ }
+ ),
+ "outlink_targets": "https://example.com/about,https://partner.com/x",
+ "fetch_method": "static",
+ },
+ {
+ "url": "https://example.com/about",
+ "status": "200",
+ "title": "Short",
+ "meta_description_len": 50,
+ "h1_count": 0,
+ "content_length": 150,
+ "word_count": 80,
+ "depth": 1,
+ "response_time_ms": 150,
+ "content_type": "text/html",
+ "og_title": "",
+ "twitter_card": "",
+ "og_image": "",
+ "tech_stack": "[]",
+ "top_keywords": "not-json",
+ "script_count": 0,
+ "link_stylesheet_count": 0,
+ "fetch_method": "rendered",
+ },
+ {
+ "url": "https://example.com/missing",
+ "status": "404",
+ "title": "",
+ "meta_description": "",
+ "h1": "",
+ "heading_sequence": "",
+ "word_count": 0,
+ "content_length": 0,
+ "h1_count": 0,
+ "script_count": 0,
+ "link_stylesheet_count": 0,
+ },
+ {
+ "url": "https://example.com/redirect",
+ "status": "301",
+ "final_url": "https://example.com/dest",
+ "title": "T" * 70,
+ "meta_description": "",
+ "h1": "",
+ "heading_sequence": "",
+ "word_count": 0,
+ "meta_description_len": 200,
+ "h1_count": 3,
+ "content_length": 50,
+ "script_count": 0,
+ "link_stylesheet_count": 0,
+ },
+ {
+ "url": "https://example.com/error",
+ "status": "error",
+ "title": "",
+ "meta_description": "",
+ "h1": "",
+ "heading_sequence": "",
+ "word_count": 0,
+ "content_length": 0,
+ "h1_count": 0,
+ "script_count": 0,
+ "link_stylesheet_count": 0,
+ },
+ ]
+ )
+
+
+def test_compute_summary_seo_issues() -> None:
+ out = seo_summary._compute_summary_seo_issues(_crawl_df())
+ assert out["summary"]["total_urls"] == 5
+ assert out["issues"]["broken"]
+ assert out["issues"]["redirects"]
+ assert out["issues"]["seo"]
+ assert out["recommendations"]
+
+
+def test_content_analytics_helpers() -> None:
+ df = _crawl_df()
+ content = content_analytics._build_content_analytics(df)
+ assert content["word_count_stats"]["mean"] > 0
+ assert content["thin_pages"]
+ assert content_analytics._parse_top_keywords_items(None) == []
+ assert content_analytics._parse_top_keywords_items('["list"]') == []
+ assert content_analytics._parse_top_keywords_items(json.dumps([{"word": "x", "count": 2}]))[0]["word"] == "x"
+
+ social = content_analytics._build_social_coverage(df)
+ assert social["og_coverage_pct"] > 0
+ assert social["missing_og"]
+
+ tech = content_analytics._build_tech_stack_summary(df)
+ assert tech["technologies"]
+
+ rt = content_analytics._build_response_time_stats(df)
+ assert rt["slow_pages"]
+ assert rt["p50"] > 0
+
+ depth = content_analytics._build_depth_distribution(df)
+ assert depth["max_depth"] == 1
+
+ kw = content_analytics._build_keyword_opportunities(df, {"include_keyword_opportunities": "true"})
+ assert "quick_wins" in kw
+ assert content_analytics._build_keyword_opportunities(df, {"include_keyword_opportunities": "false"}) == {}
+
+
+def test_build_image_inventory_empty_urls(capsys) -> None:
+ with patch("website_profiling.analysis.image_probe.probe_image_urls") as probe:
+ inventory, summary = content_analytics._build_image_inventory(
+ [{"url": "https://ex.com/", "page_analysis": {}}],
+ {"probe_image_inventory": "true"},
+ )
+ probe.assert_not_called()
+ assert inventory == []
+ assert summary["inventory_available"] is False
+
+
+def test_build_image_inventory_counts_failures(capsys) -> None:
+ links = [{"url": "https://ex.com/p", "page_analysis": {"image_urls": ["https://cdn.ex.com/a.png"]}}]
+ probed = [{"url": "https://cdn.ex.com/a.png", "status": None, "content_type": None, "size_bytes": None, "error": "timeout"}]
+ with patch("website_profiling.analysis.image_probe.probe_image_urls", return_value=probed):
+ inventory, summary = content_analytics._build_image_inventory(
+ links,
+ {"probe_image_inventory": "true", "max_image_probe_urls": "10"},
+ )
+ assert summary["failed"] == 1
+ assert summary["inventory_available"] is True
+ assert inventory[0]["error"] == "timeout"
+
+
+def test_report_metadata_helpers(capsys) -> None:
+ df = _crawl_df()
+ assert report_metadata._parse_page_analysis_cell(float("nan")) == {}
+ assert report_metadata._parse_page_analysis_cell("{bad") == {}
+
+ outbound = report_metadata._build_outbound_link_domains(df, "https://example.com/", 10)
+ assert outbound[0]["host"] == "partner.com"
+
+ fps = report_metadata._build_url_fingerprints(df)
+ assert fps[0]["content_fingerprint"]
+
+ hreflang = report_metadata._build_hreflang_summary(df)
+ assert hreflang["pages_with_hreflang_links"] == 1
+
+ payload = {"links": [{}], "summary": {"total_urls": 2}, "report_meta": {"crawl_scope": {"pages_crawled": 3}}}
+ report_metadata._validate_report_url_counts(payload, 1)
+ assert payload["ml_errors"]
+
+ meta = report_metadata._build_report_metadata(
+ df,
+ {"max_pages": "2", "export_logo_url": "https://logo.example/logo.png", "crawl_render_mode": "static"},
+ {"url": "https://example.com/"},
+ {
+ "fetched_at": "2026-01-01",
+ "date_range_days": 28,
+ "gsc": {"row_count": 10},
+ "ga4": {"sessions": 1},
+ },
+ {"rows": [{"word": "test", "source": "crawl"}], "enriched_at": "2026-01-02"},
+ {"llm_meta": {"model": "gpt-test"}},
+ 42,
+ "2026-01-01T00:00:00Z",
+ {
+ "imported_at": "2026-01-03",
+ "top_linking_sites": ["a.com"],
+ "sample_links": [{"url": "x"}],
+ "latest_links": [],
+ },
+ )
+ assert "lighthouse" in meta["data_sources"]
+ assert "search_console" in meta["data_sources"]
+ assert "analytics" in meta["data_sources"]
+ assert "ai" in meta["data_sources"]
+ assert "estimated" in meta["data_sources"]
+ assert meta["export_logo_url"] == "https://logo.example/logo.png"
+ assert meta["crawl_run_id"] == 42
+
+
+def test_site_level_branches(monkeypatch) -> None:
+ assert site_level._fetch_site_level("not-a-url")["robots_present"] is False
+
+ session = MagicMock()
+ session.get.side_effect = RuntimeError("network")
+ monkeypatch.setattr("website_profiling.reporting.site_level.requests.Session", lambda: session)
+ out = site_level._fetch_site_level("https://example.com/", timeout=1)
+ assert out["robots_present"] is False
+ assert out["sitemap_present"] is False
+
+
+def test_lighthouse_report_helpers() -> None:
+ assert lighthouse_report._strip_www("WWW.Example.COM") == "example.com"
+ assert lighthouse_report._url_hostname("") == ""
+ assert lighthouse_report._url_hostname("not a url") == ""
+ assert lighthouse_report._hosts_match("www.example.com", "example.com")
+ assert not lighthouse_report._hosts_match("", "example.com")
+
+ by_url = {"https://example.com/": {"score": 90}, "https://other.com/": {"score": 50}}
+ filtered = lighthouse_report.filter_lighthouse_by_host(by_url, "example.com")
+ assert len(filtered) == 1
+
+ assert lighthouse_report.lighthouse_for_url(by_url, "https://example.com")["score"] == 90
+ assert lighthouse_report.lighthouse_for_url(by_url, "https://missing.com") is None
+
+ host = lighthouse_report._derive_expected_host("", pd.DataFrame({"url": ["https://derived.com/x"]}))
+ assert host == "derived.com"
+
+ picked = lighthouse_report._pick_lighthouse_summary(
+ by_url,
+ "https://example.com/",
+ {"url": "https://example.com/", "performance": 88},
+ "example.com",
+ )
+ assert picked is not None
+
+ with patch("website_profiling.reporting.lighthouse_report.ssl.create_default_context") as ctx_mock:
+ cert_sock = MagicMock()
+ cert_sock.getpeercert.return_value = {"notAfter": "Jan 1 00:00:00 2030 GMT"}
+ ctx_mock.return_value.wrap_socket.return_value.__enter__.return_value = cert_sock
+ with patch("website_profiling.reporting.lighthouse_report.socket.create_connection"):
+ iso = lighthouse_report.fetch_site_ssl_expires_iso("example.com")
+ assert iso is not None
+ assert lighthouse_report.fetch_site_ssl_expires_iso("") is None
+ with patch("website_profiling.reporting.lighthouse_report.socket.create_connection", side_effect=OSError("fail")):
+ assert lighthouse_report.fetch_site_ssl_expires_iso("example.com") is None
+
+
+def test_build_lighthouse_by_url_for_report(monkeypatch) -> None:
+ conn = MagicMock()
+ raw = {
+ "lighthouseResult": {
+ "finalUrl": "https://example.com/",
+ "audits": {
+ "first-contentful-paint": {
+ "score": 0.5,
+ "title": "FCP",
+ "helpText": "Improve FCP",
+ }
+ },
+ }
+ }
+
+ monkeypatch.setattr(
+ "website_profiling.db.read_lighthouse_page_summaries",
+ lambda _c: {"https://example.com": {"url": "https://example.com/", "median_metrics": {}}},
+ )
+ monkeypatch.setattr(
+ "website_profiling.db.read_lh_runs_by_url",
+ lambda _c: {"https://example.com": [99]},
+ )
+ monkeypatch.setattr("website_profiling.db.read_lighthouse_run_json", lambda _c, _id: raw)
+ monkeypatch.setattr(
+ "website_profiling.db.read_lh_audits_with_items",
+ lambda _c, _id: [{"id": "fcp"}],
+ )
+ monkeypatch.setattr(
+ "website_profiling.lighthouse.runner.extract_from_lighthouse_json",
+ lambda _raw: {"performance_score": 50, "category_scores": {"performance": 50}},
+ )
+ monkeypatch.setattr(
+ "website_profiling.tools.warnings.parse_lighthouse_to_diagnostics",
+ lambda _raw, max_nodes_in_refs=None: [{"id": "diag"}],
+ )
+ monkeypatch.setattr(
+ "website_profiling.tools.warnings.resolve_impact",
+ lambda *_a, **_k: "high",
+ )
+ monkeypatch.setattr(
+ "website_profiling.lighthouse.runner._evidence_from_audit",
+ lambda _a: "evidence",
+ )
+
+ out = lighthouse_report.build_lighthouse_by_url_for_report(conn)
+ assert "https://example.com" in out
+ assert out["https://example.com"]["top_failures"]
+
+
+def test_build_edges_from_df_paths(monkeypatch, tmp_path) -> None:
+ edges_csv = str(tmp_path / "edges.csv")
+ with patch("website_profiling.reporting.edges_report.load_edges", return_value=[("https://a.com", "https://a.com/b")]):
+ loaded = edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://a.com"]}),
+ edges_csv,
+ True,
+ 10,
+ 2,
+ 5,
+ 0.0,
+ )
+ assert loaded == [("https://a.com", "https://a.com/b")]
+
+ df = pd.DataFrame(
+ {
+ "url": ["https://example.com/", "https://other.com/"],
+ "outlink_targets": ["https://example.com/about", "https://other.com/x"],
+ }
+ )
+ from_column = edges_report.build_edges_from_df(df, "", False, 10, 2, 5, 0.0)
+ assert ("https://example.com/", "https://example.com/about") in from_column
+
+ html = 'About Ext '
+
+ class FakeResp:
+ status_code = 200
+ text = html
+
+ @property
+ def headers(self):
+ return {"Content-Type": "text/html"}
+
+ session = MagicMock()
+ session.get.return_value = FakeResp()
+ monkeypatch.setattr("website_profiling.reporting.edges_report.requests.Session", lambda: session)
+
+ fetched = edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://example.com/"]}),
+ "",
+ True,
+ 10,
+ 1,
+ 5,
+ 0.0,
+ )
+ assert any(t == "https://example.com/about" for _s, t in fetched)
+
+ fetcher = MagicMock()
+ fetcher.fetch.return_value = SimpleNamespace(status=200, text=html)
+ fetcher.close = MagicMock()
+ monkeypatch.setattr("website_profiling.crawl.fetchers.build_fetcher", lambda **_k: fetcher)
+ js_edges = edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://example.com/"]}),
+ "",
+ True,
+ 10,
+ 1,
+ 5,
+ 0.0,
+ render_mode="javascript",
+ )
+ fetcher.close.assert_called_once()
+ assert js_edges
+
+ fetcher.fetch.return_value = SimpleNamespace(status=404, text="")
+ assert edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://example.com/"]}),
+ "",
+ True,
+ 10,
+ 1,
+ 5,
+ 0.0,
+ render_mode="auto",
+ ) == []
+
+ session.get.side_effect = RuntimeError("boom")
+ assert edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://example.com/"]}),
+ "",
+ True,
+ 10,
+ 1,
+ 5,
+ 0.0,
+ ) == []
+
+
+def test_content_analytics_edge_branches() -> None:
+ assert content_analytics._build_content_analytics(pd.DataFrame())["word_count_stats"]["mean"] == 0
+ assert content_analytics._build_content_analytics(pd.DataFrame({"url": ["x"], "status": ["404"]}))["word_count_stats"]["mean"] == 0
+
+ df = pd.DataFrame(
+ [
+ {"url": float("nan"), "status": "200", "word_count": 100},
+ {"url": "https://example.com/thin", "status": "200", "word_count": 50},
+ ]
+ )
+ assert content_analytics._build_content_analytics(df)["thin_pages"] == [{"url": "https://example.com/thin", "word_count": 50}]
+
+ assert content_analytics._parse_top_keywords_items(json.dumps(["bad"])) == []
+ assert content_analytics._parse_top_keywords_items(json.dumps([{"word": "", "count": 1}])) == []
+
+ hist_df = pd.DataFrame(
+ [
+ {
+ "url": f"https://example.com/{i}",
+ "status": "200",
+ "top_keywords": json.dumps([{"word": "shared-term", "count": 1}]),
+ }
+ for i in range(25)
+ ]
+ )
+ hist = content_analytics._build_text_content_analysis(hist_df)["keyword_frequency_histogram"]
+ assert hist["21+"] == 1
+
+ assert content_analytics._build_content_analytics(pd.DataFrame({"url": ["x"], "status": ["404"], "word_count": [0]}))["word_count_stats"]["mean"] == 0
+
+ assert content_analytics._build_social_coverage(pd.DataFrame())["og_coverage_pct"] == 0
+ non_html = pd.DataFrame([{"url": "https://example.com/x", "status": "200", "content_type": "application/json", "og_title": "x"}])
+ assert content_analytics._build_social_coverage(non_html)["og_coverage_pct"] == 0
+
+ social_df = pd.DataFrame(
+ [
+ {"url": float("nan"), "status": "200", "content_type": "text/html", "og_title": "x", "twitter_card": "x", "og_image": "x"},
+ {"url": "https://example.com/a", "status": "200", "content_type": "text/html", "og_title": "", "twitter_card": "", "og_image": ""},
+ ]
+ )
+ social = content_analytics._build_social_coverage(social_df)
+ assert social["missing_og"] == ["https://example.com/a"]
+
+ assert content_analytics._build_tech_stack_summary(pd.DataFrame({"url": ["x"], "status": ["200"]}))["technologies"] == []
+ tech_df = pd.DataFrame(
+ [
+ {"url": "https://example.com/a", "status": "200", "content_type": "application/json", "tech_stack": '["React"]'},
+ {"url": "https://example.com/b", "status": "200", "content_type": "text/html", "tech_stack": "bad-json"},
+ ]
+ )
+ assert content_analytics._build_tech_stack_summary(tech_df)["technologies"] == []
+
+ assert content_analytics._build_response_time_stats(pd.DataFrame({"url": ["x"]}))["p50"] == 0
+ assert content_analytics._build_response_time_stats(pd.DataFrame({"url": ["x"], "response_time_ms": [None]}))["p50"] == 0
+ assert content_analytics._build_depth_distribution(pd.DataFrame({"url": ["x"]}))["max_depth"] == 0
+ assert content_analytics._build_depth_distribution(pd.DataFrame({"url": ["x"], "depth": [None]}))["max_depth"] == 0
+
+ empty_kw = content_analytics._build_keyword_opportunities(
+ pd.DataFrame({"url": ["https://example.com"], "status": ["404"]}),
+ {"include_keyword_opportunities": "true"},
+ )
+ assert empty_kw["quick_wins"] == []
+
+
+def test_report_metadata_extra_branches() -> None:
+ assert report_metadata._parse_page_analysis_cell("{}") == {}
+ df = pd.DataFrame(
+ [
+ {
+ "url": "",
+ "status": "200",
+ "page_analysis": json.dumps({"external_links": ["https://partner.com/x", 123]}),
+ "outlink_targets": "https://example.com/,https://partner.com/y",
+ },
+ {
+ "url": "https://example.com/ok",
+ "status": "200",
+ "page_analysis": json.dumps({"external_links": ["https://partner.com/z"]}),
+ },
+ ]
+ )
+ hosts = report_metadata._build_outbound_link_domains(df, "https://example.com/", 10)
+ assert any(h["host"] == "partner.com" for h in hosts)
+
+ hreflang = report_metadata._build_hreflang_summary(
+ pd.DataFrame(
+ [
+ {"url": "https://example.com/a", "status": "200", "page_analysis": json.dumps({"html_lang": "", "hreflang_alternates": []})},
+ {"url": "https://example.com/b", "status": "404", "page_analysis": "{}"},
+ ]
+ )
+ )
+ assert hreflang["pages_missing_html_lang"] == 1
+
+ meta = report_metadata._build_report_metadata(
+ pd.DataFrame([{"url": "https://example.com", "status": "200"}]),
+ {},
+ None,
+ {"gsc": {"row_count": 1}},
+ None,
+ {},
+ None,
+ None,
+ {"imported_at": "2026-01-01", "top_linking_sites": [], "sample_links": [], "latest_links": []},
+ )
+ assert meta["gsc_links_imported_at"] == "2026-01-01"
+
+
+def test_seo_summary_skips_blank_urls() -> None:
+ df = pd.DataFrame(
+ [
+ {"url": float("nan"), "status": "200", "title": "Missing URL row"},
+ {"url": "https://example.com/a", "status": "200", "title": "", "meta_description_len": 10, "h1_count": 0, "content_length": 100},
+ ]
+ )
+ out = seo_summary._compute_summary_seo_issues(df)
+ assert out["issues"]["seo"]
+
+
+def test_site_level_reads_robots_sitemap_hint(monkeypatch) -> None:
+ class FakeResp:
+ def __init__(self, code, text):
+ self.status_code = code
+ self.text = text
+
+ session = MagicMock()
+ session.get.side_effect = lambda url, timeout=8: {
+ "https://example.com/robots.txt": FakeResp(200, "User-agent: *\nSitemap: https://example.com/sitemap.xml"),
+ "https://example.com/sitemap.xml": FakeResp(200, "Link '
+
+ @property
+ def headers(self):
+ return {"Content-Type": "text/html"}
+
+ session = MagicMock()
+ session.get.return_value = FakeResp()
+ monkeypatch.setattr("website_profiling.reporting.edges_report.requests.Session", lambda: session)
+ with patch("website_profiling.reporting.edges_report.time.sleep") as sleep:
+ edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://example.com/"]}),
+ "",
+ True,
+ 10,
+ 1,
+ 5,
+ 0.5,
+ )
+ sleep.assert_called()
+
+ def raise_result(_self):
+ raise RuntimeError("worker failed")
+
+ with patch("website_profiling.reporting.edges_report.as_completed") as completed:
+ future = MagicMock()
+ future.result = raise_result
+ completed.return_value = [future]
+ monkeypatch.setattr(
+ "website_profiling.reporting.edges_report.ThreadPoolExecutor",
+ lambda *a, **k: MagicMock(__enter__=lambda s: s, __exit__=lambda *a: None, submit=lambda fn, u: future),
+ )
+ session.get.return_value = FakeResp()
+ result = edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://example.com/"]}),
+ "",
+ True,
+ 10,
+ 1,
+ 5,
+ 0.0,
+ )
+ assert result == []
+
+
+def test_content_analytics_remaining_lines() -> None:
+ assert content_analytics._parse_top_keywords_items(object()) == [] # type: ignore[arg-type]
+ assert content_analytics._parse_top_keywords_items("{bad-json") == []
+
+ na_kw = pd.DataFrame([{"url": float("nan"), "status": "200", "top_keywords": json.dumps([{"word": "x", "count": 1}])}])
+ assert content_analytics._build_text_content_analysis(na_kw)["vocabulary_stats"]["unique_terms"] == 0
+
+ assert content_analytics._build_keyword_opportunities(pd.DataFrame({"url": ["x"]}), {"include_keyword_opportunities": "true"})["quick_wins"] == []
+
+ only_404 = pd.DataFrame([{"url": "https://example.com/x", "status": "404", "top_keywords": "[]"}])
+ assert content_analytics._build_text_content_analysis(only_404)["keyword_index"] == []
+
+ spread_df = pd.DataFrame(
+ [
+ {
+ "url": f"https://example.com/{i}",
+ "status": "200",
+ "top_keywords": json.dumps([{"word": "shared-six", "count": 1}]),
+ }
+ for i in range(6)
+ ]
+ )
+ assert content_analytics._build_text_content_analysis(spread_df)["keyword_frequency_histogram"]["6-20"] == 1
+
+ assert content_analytics._build_tech_stack_summary(
+ pd.DataFrame([{"url": "https://example.com", "status": "200", "content_type": "application/pdf", "tech_stack": '["X"]'}])
+ )["total_pages_analyzed"] == 0
+
+ kw_df = pd.DataFrame([{"url": "https://example.com", "status": "200", "content_text": "uniquewords " * 20}])
+ with patch("website_profiling.reporting.content_analytics.extract_candidates_from_df", return_value=[]):
+ assert content_analytics._build_keyword_opportunities(kw_df, {"include_keyword_opportunities": "true"})["quick_wins"] == []
+ with patch("website_profiling.reporting.content_analytics.extract_candidates_from_df", return_value=[{"word": "a", "volume": 0.1}]), patch(
+ "website_profiling.reporting.content_analytics.score_keywords",
+ return_value=[{"word": "a", "volume": 0.1, "difficulty": 80}],
+ ), patch("website_profiling.reporting.content_analytics.cluster_keywords", return_value=[]):
+ out = content_analytics._build_keyword_opportunities(kw_df, {"include_keyword_opportunities": "true"})
+ assert out["high_value"]
+
+
+def test_report_metadata_remaining_lines() -> None:
+ df = pd.DataFrame(
+ [
+ {
+ "url": "https://example.com/",
+ "status": "200",
+ "page_analysis": json.dumps({"external_links": ["https://example.com/internal", 99]}),
+ "word_count": 0,
+ "content_length": 0,
+ "h1_count": 0,
+ "script_count": 0,
+ "link_stylesheet_count": 0,
+ },
+ ]
+ )
+ assert report_metadata._build_outbound_link_domains(df, "https://example.com/", 5) == []
+ assert report_metadata._build_url_fingerprints(pd.DataFrame([{"url": " ", "status": "200"}])) == []
+
+ meta = report_metadata._build_report_metadata(
+ pd.DataFrame([{"url": "https://example.com", "status": "200"}]),
+ {},
+ None,
+ None,
+ None,
+ {},
+ None,
+ None,
+ {"imported_at": "2026-01-01", "top_linking_sites": [], "sample_links": [], "latest_links": []},
+ )
+ assert "search_console" in meta["data_sources"]
+
+
+def test_seo_summary_remaining_lines() -> None:
+ df = pd.DataFrame(
+ [
+ {
+ "url": "https://example.com/a",
+ "status": "200",
+ "title": "ok",
+ "meta_description_len": 0,
+ "h1_count": 2,
+ "content_length": 100,
+ },
+ {
+ "url": float("nan"),
+ "status": "200",
+ "title": "x",
+ "meta_description_len": 80,
+ "h1_count": 0,
+ "content_length": 100,
+ },
+ ]
+ )
+ out = seo_summary._compute_summary_seo_issues(df)
+ assert any(i["type"] == "h1_multi" for i in out["issues"]["seo"])
+
+
+def test_lighthouse_report_remaining_lines(monkeypatch) -> None:
+ cert_sock = MagicMock()
+ cert_sock.getpeercert.return_value = {"notAfter": None}
+ ctx = MagicMock()
+ ctx.wrap_socket.return_value.__enter__.return_value = cert_sock
+ with patch("website_profiling.reporting.lighthouse_report.ssl.create_default_context", return_value=ctx), patch(
+ "website_profiling.reporting.lighthouse_report.socket.create_connection"
+ ):
+ assert lighthouse_report.fetch_site_ssl_expires_iso("example.com") is None
+
+ by_url = {"https://www.example.com/": {"score": 1}}
+ assert lighthouse_report._pick_lighthouse_summary(by_url, "https://www.example.com/", None, "example.com") is not None
+ assert lighthouse_report._pick_lighthouse_summary(by_url, "", None, "example.com")["score"] == 1
+ assert lighthouse_report._derive_expected_host("", pd.DataFrame({"url": [float("nan")]})) == ""
+
+ conn = MagicMock()
+ monkeypatch.setattr("website_profiling.db.read_lighthouse_page_summaries", lambda _c: {"https://empty.com": {}})
+ monkeypatch.setattr("website_profiling.db.read_lh_runs_by_url", lambda _c: {"https://empty.com": []})
+ assert lighthouse_report.build_lighthouse_by_url_for_report(conn) == {}
+
+ assert lighthouse_report._derive_expected_host("https://host-only.com/", pd.DataFrame()) == "host-only.com"
+
+ mismatch = lighthouse_report._pick_lighthouse_summary(
+ {},
+ "https://example.com/",
+ {"url": "https://other-host.com/", "performance": 1},
+ "example.com",
+ )
+ assert mismatch is None
+
+ matched = lighthouse_report._pick_lighthouse_summary(
+ {},
+ "",
+ {"url": "https://www.example.com/", "performance": 99},
+ "example.com",
+ )
+ assert matched["performance"] == 99
+
+ conn2 = MagicMock()
+ monkeypatch.setattr(
+ "website_profiling.db.read_lighthouse_page_summaries",
+ lambda _c: {"https://needs-url.com": {"median_metrics": {"performance_score": 1}}},
+ )
+ monkeypatch.setattr("website_profiling.db.read_lh_runs_by_url", lambda _c: {})
+ out2 = lighthouse_report.build_lighthouse_by_url_for_report(conn2)
+ assert out2["https://needs-url.com"]["url"] == "https://needs-url.com"
+
+ assert lighthouse_report.lighthouse_for_url({"https://example.com/": {"score": 9}}, "https://example.com")["score"] == 9
+ assert lighthouse_report.lighthouse_for_url({"https://example.com": {"score": 7}}, "https://example.com/")["score"] == 7
+
+
+def test_build_edges_remaining_lines(monkeypatch) -> None:
+ df = pd.DataFrame({"url": ["https://example.com/"], "links": [None]})
+ assert edges_report.build_edges_from_df(df, "", True, 10, 1, 5, 0.0) == []
+
+ df_blank = pd.DataFrame({"url": ["https://example.com/"], "outlink_targets": [[" ", "https://example.com/about"]]})
+ edges_blank = edges_report.build_edges_from_df(df_blank, "", False, 10, 1, 5, 0.0)
+ assert ("https://example.com/", "https://example.com/about") in edges_blank
+
+ df_external = pd.DataFrame({"url": ["https://example.com/"], "outlink_targets": ["https://external.com/page"]})
+ assert edges_report.build_edges_from_df(df_external, "", True, 10, 1, 5, 0.0) == []
+
+ class FakeResp:
+ status_code = 404
+ text = ""
+
+ @property
+ def headers(self):
+ return {"Content-Type": "text/html"}
+
+ session = MagicMock()
+ session.get.return_value = FakeResp()
+ monkeypatch.setattr("website_profiling.reporting.edges_report.requests.Session", lambda: session)
+ assert edges_report.build_edges_from_df(
+ pd.DataFrame({"url": ["https://example.com/"]}),
+ "",
+ True,
+ 10,
+ 1,
+ 5,
+ 0.0,
+ ) == []
+
+
+def test_builder_exposes_llm_keyword_cluster_imports() -> None:
+ """Regression: LLM keyword cluster branch must not NameError after builder split."""
+ import website_profiling.reporting.builder as builder_mod
+ from website_profiling.analysis.text_hygiene import is_junk_semantic_term
+ from website_profiling.llm.enrich import cluster_keywords_llm
+
+ assert builder_mod.is_junk_semantic_term is is_junk_semantic_term
+ assert builder_mod.cluster_keywords_llm is cluster_keywords_llm
diff --git a/tests/test_router_tools.py b/tests/test_router_tools.py
new file mode 100644
index 00000000..61c4187f
--- /dev/null
+++ b/tests/test_router_tools.py
@@ -0,0 +1,31 @@
+"""Tests for run_domain_agent fallback behavior."""
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from website_profiling.tools.audit_tools import AuditToolContext
+from website_profiling.tools.audit_tools.router_tools import run_domain_agent
+
+
+def test_run_domain_agent_falls_back_to_global_search() -> None:
+ conn = MagicMock()
+ ctx = AuditToolContext(property_id=1)
+ fake_matches = [
+ {"name": "list_broken_links", "description": "", "domain": "links", "tier": 1},
+ {"name": "get_schema_coverage", "description": "", "domain": "schema", "tier": 1},
+ ]
+ with patch(
+ "website_profiling.tools.audit_tools.registry.search_tools",
+ return_value=fake_matches,
+ ):
+ with patch(
+ "website_profiling.tools.audit_tools.registry.tool_names_for_domain",
+ return_value=["get_unrelated_tool"],
+ ):
+ result = run_domain_agent(conn, ctx, {
+ "task": "broken links audit",
+ "domain": "schema",
+ "max_steps": 2,
+ })
+
+ assert result["tools_used"] == ["list_broken_links", "get_schema_coverage"]
diff --git a/tests/test_tool_selector.py b/tests/test_tool_selector.py
new file mode 100644
index 00000000..8378d77e
--- /dev/null
+++ b/tests/test_tool_selector.py
@@ -0,0 +1,101 @@
+"""Tests for dynamic chat tool selection."""
+from __future__ import annotations
+
+import os
+from unittest.mock import patch
+
+from website_profiling.tools.audit_tools.registry import mcp_tool_names, tier0_tool_names, tool_handler_names
+from website_profiling.tools.audit_tools.tool_selector import chat_tool_max, select_tools_for_turn
+
+
+def test_select_tools_always_includes_tier0() -> None:
+ names = select_tools_for_turn("hello")
+ assert tier0_tool_names() <= names
+ assert len(names) <= chat_tool_max()
+
+
+def test_select_tools_google_domain_boost() -> None:
+ names = select_tools_for_turn("Show me GSC clicks and GA4 landing pages")
+ assert "get_google_summary" in names or "get_gsc_top_queries" in names
+
+
+def test_select_tools_full_mode() -> None:
+ with patch.dict(os.environ, {"CHAT_TOOL_MODE": "full"}):
+ names = select_tools_for_turn("anything")
+ assert names == tool_handler_names()
+
+
+def test_select_tools_broken_links_in_subset() -> None:
+ names = select_tools_for_turn("list broken internal links")
+ assert "list_broken_links" in names
+
+
+def test_broken_link_tools_classified_as_links() -> None:
+ from website_profiling.tools.audit_tools.tool_domains import classify_tool_domain
+
+ assert classify_tool_domain("list_broken_links") == "links"
+ assert classify_tool_domain("list_broken_link_sources") == "links"
+
+
+def test_mcp_core_includes_tier0_tools() -> None:
+ core = mcp_tool_names("core")
+ assert tier0_tool_names() <= core
+
+
+def test_compare_tools_classified_as_drift() -> None:
+ from website_profiling.tools.audit_tools.tool_domains import classify_tool_domain
+
+ assert classify_tool_domain("compare_issue_deltas") == "drift"
+ assert classify_tool_domain("compare_reports") == "drift"
+
+
+def test_audit_report_overview_prefers_portfolio_not_export() -> None:
+ names = select_tools_for_turn("show me the audit report overview")
+ assert "get_report_summary" in names
+ assert "export_audit_report" not in names
+
+
+def test_compare_prompt_loads_compare_issue_deltas() -> None:
+ names = select_tools_for_turn("compare issue deltas since last crawl")
+ assert "compare_issue_deltas" in names
+
+
+def test_playbook_anchors_load_specialized_tools() -> None:
+ assert "get_image_audit_summary" in select_tools_for_turn("image alt audit")
+ assert "export_audit_report" in select_tools_for_turn("export as pdf")
+ assert "get_category_scores" in select_tools_for_turn("show category scores")
+ assert "get_critical_issues" in select_tools_for_turn("top critical issues")
+ assert "get_lighthouse_summary" in select_tools_for_turn("lighthouse scores")
+
+
+def test_catalog_does_not_match_ops_domain() -> None:
+ from website_profiling.tools.audit_tools.tool_selector import _score_domains
+
+ assert _score_domains("show catalog") == []
+
+
+def test_search_expansion_applies_soft_cap() -> None:
+ from website_profiling.llm.agent import _expand_active_tools_from_result
+ from website_profiling.tools.audit_tools.tool_selector import chat_tool_search_cap, tier0_tool_names
+
+ active = tier0_tool_names()
+ many = [f"tool_{i}" for i in range(100)]
+ expanded = _expand_active_tools_from_result(
+ "search_audit_tools",
+ {"tool_names": many},
+ active,
+ )
+ assert len(expanded) <= chat_tool_search_cap()
+
+
+def test_search_audit_tools_expansion_names() -> None:
+ from website_profiling.llm.agent import _expand_active_tools_from_result
+
+ active = tier0_tool_names()
+ expanded = _expand_active_tools_from_result(
+ "search_audit_tools",
+ {"tool_names": ["list_broken_links", "get_schema_coverage"]},
+ active,
+ )
+ assert "list_broken_links" in expanded
+ assert "get_schema_coverage" in expanded
diff --git a/tests/test_tools_gate100_coverage.py b/tests/test_tools_gate100_coverage.py
new file mode 100644
index 00000000..c547f88d
--- /dev/null
+++ b/tests/test_tools_gate100_coverage.py
@@ -0,0 +1,556 @@
+"""Coverage for batch-100 foundation modules (data_coverage, insight, router, registry)."""
+from __future__ import annotations
+
+import os
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from website_profiling.tools.audit_tools import insight_helpers as ih
+from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx
+from website_profiling.tools.audit_tools import (
+ crawl as crawl_mod,
+ data_coverage as dc_mod,
+ google as google_mod,
+ insight_tools as insight_mod,
+ keywords as kw_mod,
+ registry,
+ router_tools as router_mod,
+)
+from website_profiling.tools.audit_tools.tool_domains import (
+ CANONICAL_DOMAINS,
+ TIER_0_TOOLS,
+ classify_tool_domain,
+ domains_catalog,
+ tool_names_for_mcp_bundle,
+ tool_names_for_tier,
+ tools_by_domain,
+)
+from website_profiling.tools.audit_tools.tool_selector import (
+ apply_tool_cap,
+ chat_tool_max,
+ compact_tool_list,
+ select_tools_for_turn,
+)
+
+
+@pytest.fixture
+def conn() -> MagicMock:
+ return MagicMock()
+
+
+@pytest.fixture
+def ctx() -> Ctx:
+ return Ctx(property_id=1, report_id=1)
+
+
+def test_context_load_google_full_and_pair_fallbacks(conn: MagicMock, ctx: Ctx) -> None:
+ embedded = {"gsc": {"summary": {"clicks": 1}}, "fetched_at": "2026-01-01"}
+ prior = {"gsc_full": {"summary": {"clicks": 0}}}
+
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_google_data_full",
+ return_value=None,
+ ), patch(
+ "website_profiling.tools.audit_tools.context.read_report_payload",
+ return_value={"google": embedded},
+ ):
+ assert ctx.load_google_full(conn) == embedded
+
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_google_data_full",
+ return_value=None,
+ ), patch(
+ "website_profiling.tools.audit_tools.context.read_prior_google_snapshot",
+ return_value=prior,
+ ), patch(
+ "website_profiling.tools.audit_tools.context.read_report_payload",
+ return_value={"google": embedded},
+ ):
+ current, prior_out = ctx.load_google_pair(conn)
+ assert current == embedded
+ assert prior_out == prior
+
+ with patch(
+ "website_profiling.tools.audit_tools.context.read_google_data_full",
+ return_value={"gsc_full": {"summary": {}}},
+ ):
+ assert ctx.load_google_full(conn) == {"gsc_full": {"summary": {}}}
+
+
+def test_data_coverage_report_all_branches(conn: MagicMock, ctx: Ctx) -> None:
+ assert dc_mod.get_data_coverage_report(conn, Ctx(property_id=None), {})["error"]
+
+ with patch("website_profiling.tools.audit_tools.data_coverage.get_property_by_id", return_value=None):
+ assert dc_mod.get_data_coverage_report(conn, ctx, {})["error"] == "property not found"
+
+ prop = {"google_refresh_token": "tok"}
+ payload = {
+ "image_inventory": [{"url": "x"}],
+ "axe_audit_summary": {"violations": 1},
+ "rich_results_validation": {"ok": True},
+ "text_content_analysis": {"words": 100},
+ "semantic_keyword_clusters": [{"k": "a"}],
+ "log_analysis": {"paths": []},
+ }
+ google = {
+ "gsc": {"summary": {"clicks": 10}},
+ "ga4": {"summary": {"sessions": 20}},
+ }
+ keywords = {"rows": [{"keyword": "a"}], "fetched_at": "2026-01-01"}
+ gsc_links = {
+ "sample_links": [{"url": "x"}],
+ "third_party_overlays": [{"source": "moz"}],
+ }
+ google_full = {"gsc_full": {"summary": {}}, "ga4_full": {"summary": {}}}
+
+ with patch("website_profiling.tools.audit_tools.data_coverage.get_property_by_id", return_value=prop), patch.object(
+ Ctx, "load_payload", return_value=payload,
+ ), patch.object(Ctx, "load_google", return_value=google), patch.object(
+ Ctx, "load_keywords", return_value=keywords,
+ ), patch.object(Ctx, "load_gsc_links", return_value=gsc_links), patch.object(
+ Ctx, "load_google_full", return_value=google_full,
+ ), patch(
+ "website_profiling.integrations.google.store.read_prior_google_snapshot",
+ return_value={"gsc": {}},
+ ):
+ result = dc_mod.get_data_coverage_report(conn, ctx, {})
+ assert result["missing_count"] == 0
+ assert len(result["checks"]) >= 10
+
+ sparse_prop = {"id": 1}
+ with patch("website_profiling.tools.audit_tools.data_coverage.get_property_by_id", return_value=sparse_prop), patch.object(
+ Ctx, "load_payload", return_value={},
+ ), patch.object(Ctx, "load_google", return_value=None), patch.object(
+ Ctx, "load_keywords", return_value=None,
+ ), patch.object(Ctx, "load_gsc_links", return_value=None), patch.object(
+ Ctx, "load_google_full", return_value=None,
+ ), patch(
+ "website_profiling.integrations.google.store.read_prior_google_snapshot",
+ return_value=None,
+ ):
+ sparse = dc_mod.get_data_coverage_report(conn, ctx, {})
+ assert sparse["missing_count"] > 0
+ assert sparse["checks"][0]["config_hint"]
+
+
+def test_google_series_and_page_queries(conn: MagicMock, ctx: Ctx) -> None:
+ assert google_mod.get_gsc_daily_trend(conn, ctx, {})["missing"]
+ assert google_mod.get_ga4_daily_trend(conn, ctx, {})["missing"]
+ assert google_mod.get_ga4_by_device(conn, ctx, {})["missing"]
+ assert google_mod.get_ga4_by_channel(conn, ctx, {})["missing"]
+ assert google_mod.get_gsc_page_queries(conn, ctx, {})["error"] == "url is required"
+ with patch.object(Ctx, "load_google_full", return_value=None), patch.object(Ctx, "load_google", return_value=None):
+ assert google_mod.get_gsc_page_queries(conn, ctx, {"url": "https://ex.com/"})["missing"]
+
+ raw = {
+ "gsc_full": {
+ "by_page": {
+ "https://ex.com/": {
+ "queries": [{"query": "q1", "clicks": 1}],
+ },
+ },
+ },
+ "fetched_at": "2026-01-01",
+ }
+ with patch.object(Ctx, "load_google_full", return_value=raw):
+ ok = google_mod.get_gsc_page_queries(conn, ctx, {"url": "https://ex.com/"})
+ assert ok["total"] == 1
+
+ google_data = {
+ "gsc": {"daily": [{"date": "2026-01-01", "clicks": 1}]},
+ "ga4": {"daily": [], "by_device": [{"device": "mobile"}], "by_channel": [{"channel": "organic"}]},
+ "fetched_at": "2026-01-01",
+ "date_range": "28d",
+ }
+ with patch.object(Ctx, "load_google", return_value=google_data):
+ assert google_mod.get_gsc_daily_trend(conn, ctx, {})["provenance"] == "Search Console"
+ assert google_mod.get_ga4_daily_trend(conn, ctx, {})["provenance"] == "Google Analytics 4"
+ assert google_mod.get_ga4_by_device(conn, ctx, {})["by_device"]
+ assert google_mod.get_ga4_by_channel(conn, ctx, {})["by_channel"]
+
+
+def test_insight_helpers_all_branches() -> None:
+ assert ih._num("bad", 5.0) == 5.0
+ assert ih.classify_opportunity_quadrant(
+ {"position": 10, "impressions": 200}, {"sessions": 20, "engagementRate": 0.6},
+ ) == "high_impact"
+ assert ih.classify_opportunity_quadrant(
+ {"position": 10, "impressions": 200}, {"sessions": 0, "engagementRate": 0.1},
+ ) == "worth_optimizing"
+ assert ih.classify_opportunity_quadrant(
+ {"position": 1, "impressions": 10}, {"sessions": 100, "engagementRate": 0.9},
+ ) == "good_but_capped"
+ assert ih.classify_opportunity_quadrant({"position": 1, "impressions": 1}, None) == "low_priority"
+
+ assert ih.traffic_health_ratio({}, {})["diagnosis"] == "no_data"
+ low = ih.traffic_health_ratio({"clicks": 100}, {"sessions": 10})
+ assert low["diagnosis"] == "tracking_gap"
+ high = ih.traffic_health_ratio({"clicks": 10}, {"sessions": 100})
+ assert high["diagnosis"] == "filter_issue"
+
+ gsc_pages = {
+ "https://ex.com/a": {"impressions": 500, "clicks": 10, "position": 8, "ctr": 0.02},
+ "https://ex.com/skip": "not-a-dict",
+ "https://ex.com/low": {"impressions": 5, "clicks": 0, "position": 50, "ctr": 0.001},
+ }
+ ga4_paths = {
+ "/a": {"full_url": "https://ex.com/a", "sessions": 15, "engagementRate": 0.55},
+ "/by-path": {"sessions": 8, "engagementRate": 0.4},
+ "bad": "skip",
+ }
+ rows = ih.blend_landing_pages(gsc_pages, ga4_paths, limit=5, min_impressions=100)
+ assert rows[0]["quadrant"] == "high_impact"
+ assert rows[0]["ga4_sessions"] == 15
+ path_match = ih.blend_landing_pages(
+ {"https://ex.com/by-path": {"impressions": 200, "clicks": 1, "position": 15, "ctr": 0.01}},
+ ga4_paths,
+ limit=5,
+ min_impressions=0,
+ )
+ assert path_match[0]["ga4_sessions"] == 8
+
+ payload = {
+ "categories": [
+ "bad",
+ {
+ "id": "onpage",
+ "issues": [
+ {"url": "https://ex.com/page", "priority": "High", "message": "missing title"},
+ {"url": "https://other.com", "priority": "Low", "message": "skip"},
+ "bad-issue",
+ {"priority": "Medium", "message": "site-wide"},
+ ],
+ },
+ ],
+ }
+ flags = ih.page_issue_flags("https://ex.com/page", payload)
+ assert flags[0]["message"] == "missing title"
+ assert any(f.get("message") == "site-wide" for f in flags)
+
+ green = ih.composite_page_score(
+ {"position": 5},
+ {"engagementRate": 0.8},
+ {"position": 10},
+ {"engagementRate": 0.5},
+ [],
+ {"performance": 90, "seo": 95},
+ )
+ assert green["band"] == "green"
+
+ amber = ih.composite_page_score(
+ {"position": 20},
+ {"engagementRate": 0.5},
+ {"position": 10},
+ {"engagementRate": 0.5},
+ [{"priority": "High"}],
+ None,
+ )
+ assert amber["band"] == "amber"
+
+ red = ih.composite_page_score(
+ {"position": 30},
+ {"engagementRate": 0.1},
+ {"position": 10},
+ {"engagementRate": 0.5},
+ [{"priority": "Critical"}, {"priority": "Critical"}],
+ {"performance": 30, "seo": 50},
+ )
+ assert red["band"] == "red"
+
+
+def test_insight_tools_dispatch(conn: MagicMock, ctx: Ctx) -> None:
+ assert insight_mod.get_landing_page_blended_table(conn, ctx, {})["missing"]
+
+ google_full = {
+ "gsc_full": {
+ "by_page": {"https://ex.com/": {"impressions": 500, "clicks": 5, "position": 12, "ctr": 0.01}},
+ "top_pages": [{"page": "https://ex.com/", "impressions": 500, "clicks": 5, "position": 12}],
+ },
+ "ga4_full": {"by_path": {"/": {"sessions": 20, "engagementRate": 0.6, "full_url": "https://ex.com/"}}},
+ "fetched_at": "2026-01-01",
+ }
+ payload = {
+ "categories": [{"id": "c", "issues": [{"url": "https://ex.com/", "priority": "High", "message": "m"}]}],
+ "lighthouse_by_url": {
+ "https://ex.com/": {"performance": 80, "seo": 90},
+ "https://ex.com": {"performance": 70, "seo": 80},
+ },
+ "top_pages": [{"url": "https://ex.com/", "title": "Home"}],
+ }
+
+ with patch.object(Ctx, "load_google_full", return_value={
+ "gsc_full": {"top_pages": [{"page": "https://ex.com/", "impressions": 100, "clicks": 1, "position": 10}]},
+ "ga4_full": {"by_path": {}},
+ "fetched_at": "2026-01-01",
+ }):
+ top_pages_only = insight_mod.get_landing_page_blended_table(conn, ctx, {})
+ assert top_pages_only["total"] >= 0
+
+ with patch.object(Ctx, "load_google_full", return_value=google_full):
+ blended = insight_mod.get_landing_page_blended_table(conn, ctx, {})
+ assert blended["total"] >= 0
+ matrix = insight_mod.get_opportunity_matrix(conn, ctx, {})
+ assert "counts" in matrix
+
+ with patch.object(Ctx, "load_google_full", return_value=None), patch.object(
+ Ctx, "load_google", return_value={"gsc": {"summary": {"clicks": 1}}, "ga4": {"summary": {"sessions": 2}}},
+ ):
+ health = insight_mod.get_traffic_health_check(conn, ctx, {})
+ assert "diagnosis" in health
+
+ assert insight_mod.get_landing_page_full_diagnosis(conn, ctx, {})["error"] == "url is required"
+ with patch.object(Ctx, "load_payload", return_value=None):
+ assert insight_mod.get_landing_page_full_diagnosis(conn, ctx, {"url": "https://ex.com/"})["missing"]
+
+ with patch.object(Ctx, "load_payload", return_value=payload), patch.object(
+ Ctx, "load_google_full", return_value=google_full,
+ ):
+ diag = insight_mod.get_landing_page_full_diagnosis(conn, ctx, {"url": "https://ex.com/"})
+ assert diag["url"] == "https://ex.com/"
+ assert "diagnosis" in diag
+
+ with patch.object(Ctx, "load_google_full", return_value=None), patch.object(Ctx, "load_google", return_value=None):
+ assert insight_mod.get_traffic_health_check(conn, ctx, {})["missing"]
+
+ with patch.object(insight_mod, "get_landing_page_blended_table", return_value={"error": "no google data", "missing": True}):
+ assert insight_mod.get_opportunity_matrix(conn, ctx, {})["missing"]
+
+ with patch.object(Ctx, "load_payload", return_value={
+ "lighthouse_by_url": {"https://ex.com": {"performance": 70, "seo": 80}},
+ "top_pages": [],
+ }), patch.object(Ctx, "load_google_full", return_value=google_full):
+ slash_diag = insight_mod.get_landing_page_full_diagnosis(conn, ctx, {"url": "https://ex.com/"})
+ assert slash_diag["lighthouse"]["performance"] == 70
+
+ with patch("website_profiling.tools.audit_tools.insight_tools.list_issues", return_value={"error": "boom"}):
+ assert insight_mod.get_issue_to_traffic_map(conn, ctx, {})["error"] == "boom"
+
+ with patch("website_profiling.tools.audit_tools.insight_tools.list_issues", return_value={
+ "issues": ["bad", {
+ "url": "https://ex.com/x",
+ "priority": "High",
+ "category": "onpage",
+ "message": "issue",
+ "impact_score": 10,
+ "gsc_clicks": 5,
+ "ga4_sessions": 3,
+ }],
+ "total": 1,
+ "truncated": False,
+ }):
+ mapped2 = insight_mod.get_issue_to_traffic_map(conn, ctx, {})
+ assert mapped2["total"] == 1
+
+ issues_payload = {
+ "categories": [{
+ "id": "c",
+ "issues": [{
+ "url": "https://ex.com/x",
+ "priority": "High",
+ "category": "onpage",
+ "message": "issue",
+ "impact_score": 10,
+ "gsc_clicks": 5,
+ "ga4_sessions": 3,
+ }, "bad"],
+ }],
+ }
+ with patch.object(Ctx, "load_payload", return_value=issues_payload):
+ mapped = insight_mod.get_issue_to_traffic_map(conn, ctx, {})
+ assert mapped["total"] == 1
+
+
+def test_router_tools_workflows(conn: MagicMock, ctx: Ctx) -> None:
+ assert router_mod.search_audit_tools(conn, ctx, {})["error"] == "query is required"
+ found = router_mod.search_audit_tools(conn, ctx, {"query": "broken links", "limit": 3})
+ assert found["total"] >= 1
+ assert found["tool_names"]
+
+ domains = router_mod.list_tool_domains(conn, ctx, {})
+ assert domains["domains"]
+ assert domains["domain_tool_counts"]
+
+ with patch.object(router_mod, "_dispatch", return_value={"ok": True}) as dispatch:
+ traffic = router_mod.run_insight_workflow(conn, ctx, {"type": "traffic"})
+ assert traffic["steps"][0]["tool"] == "get_traffic_health_check"
+ landing = router_mod.run_insight_workflow(conn, ctx, {"type": "landing_pages"})
+ assert len(landing["steps"]) == 2
+ default = router_mod.run_insight_workflow(conn, ctx, {"type": "priorities"})
+ assert len(default["steps"]) == 2
+
+ tech = router_mod.run_technical_workflow(conn, ctx, {"baseline_report_id": 99})
+ assert any(s["tool"] == "compare_issue_deltas" for s in tech["steps"])
+
+ assert router_mod.run_keyword_workflow(conn, Ctx(property_id=None), {})["error"]
+ kw = router_mod.run_keyword_workflow(conn, ctx, {})
+ assert kw["workflow"] == "keyword"
+ assert len(kw["steps"]) == 3
+ assert dispatch.call_count >= 6
+
+ assert router_mod.run_domain_agent(conn, ctx, {})["error"] == "task is required"
+
+ with patch(
+ "website_profiling.tools.audit_tools.registry.search_tools",
+ return_value=[],
+ ), patch(
+ "website_profiling.tools.audit_tools.registry.tool_names_for_domain",
+ return_value=["get_schema_coverage", "list_broken_links"],
+ ), patch.object(router_mod, "_dispatch", return_value={"ok": True}):
+ domain_only = router_mod.run_domain_agent(conn, ctx, {"task": "schema", "domain": "schema"})
+ assert domain_only["tools_used"]
+
+ with patch(
+ "website_profiling.tools.audit_tools.registry.search_tools",
+ return_value=[{"name": "get_schema_coverage"}],
+ ), patch(
+ "website_profiling.tools.audit_tools.registry.tool_names_for_domain",
+ return_value=["get_schema_coverage"],
+ ), patch.object(router_mod, "_dispatch", return_value={"ok": True}) as dispatch2:
+ in_pool = router_mod.run_domain_agent(conn, ctx, {"task": "schema audit", "domain": "schema", "max_steps": 2})
+ assert in_pool["tools_used"] == ["get_schema_coverage"]
+ assert dispatch2.call_count == 1
+
+ with patch(
+ "website_profiling.tools.audit_tools.registry.search_tools",
+ return_value=[{"name": "get_report_summary"}],
+ ), patch(
+ "website_profiling.tools.audit_tools.registry.tool_names_for_domain",
+ return_value=[],
+ ), patch.object(router_mod, "_dispatch", return_value={"ok": True}):
+ global_pick = router_mod.run_domain_agent(conn, ctx, {"task": "report overview", "domain": "schema", "max_steps": 1})
+ assert global_pick["tools_used"] == ["get_report_summary"]
+
+ with patch(
+ "website_profiling.tools.audit_tools.registry.search_tools",
+ return_value=[],
+ ), patch(
+ "website_profiling.tools.audit_tools.registry.tool_names_for_domain",
+ return_value=["get_schema_coverage"],
+ ), patch.object(router_mod, "_dispatch", return_value={"ok": True}):
+ domain_fallback = router_mod.run_domain_agent(conn, ctx, {"task": "schema", "domain": "schema"})
+ assert domain_fallback["tools_used"] == ["get_schema_coverage"]
+
+ with patch(
+ "website_profiling.tools.audit_tools.registry.search_tools",
+ return_value=[
+ {"name": "list_broken_links"},
+ {"name": "get_schema_coverage"},
+ ],
+ ), patch.object(router_mod, "_dispatch", return_value={"ok": True}):
+ no_domain = router_mod.run_domain_agent(conn, ctx, {"task": "broken links audit", "max_steps": 1})
+ assert no_domain["tools_used"] == ["list_broken_links"]
+
+
+def test_registry_helpers_and_validation_errors() -> None:
+ assert registry.tool_definition("get_report_summary") is not None
+ assert registry.tool_definition("__missing__") is None
+ assert registry.tool_names_for_tier(0)
+ assert registry.list_domains_catalog()
+ assert registry.search_tools("") == []
+ assert registry.search_tools("get_report_summary", limit=5)[0]["name"] == "get_report_summary"
+
+ filtered = registry.openai_tools_schema({"get_report_summary"})
+ assert len(filtered) == 1
+ assert filtered[0]["function"]["name"] == "get_report_summary"
+
+ with patch.object(registry, "_TOOL_HANDLERS", {"a": lambda *a, **k: {}}):
+ with patch.object(registry, "TOOL_DEFINITIONS", [{"name": "b", "description": "", "inputSchema": {}}]):
+ errors = registry.validate_tool_registry()
+ assert any("handler/catalog mismatch" in e for e in errors)
+
+ with patch.object(registry, "_TOOL_HANDLERS", {"a": lambda *a, **k: {}}):
+ with patch.object(registry, "TOOL_DEFINITIONS", [{"name": "a", "description": "", "inputSchema": {}}]):
+ with patch.object(registry, "_TOOL_META", {"b": {"domain": "core", "tier": 1}}):
+ errors = registry.validate_tool_registry()
+ assert any("handler/meta mismatch" in e for e in errors)
+
+ all_handlers = registry.tool_handler_names()
+ missing_t0 = next(iter(registry.tier0_tool_names()))
+ with patch.object(registry, "tool_handler_names", return_value=all_handlers - {missing_t0}):
+ errors = registry.validate_tool_registry()
+ assert any("tier0 tools missing" in e for e in errors)
+
+
+def test_tool_domains_classify_and_catalog() -> None:
+ with patch(
+ "website_profiling.tools.audit_tools.tool_domains.TIER_0_TOOLS",
+ frozenset({"synthetic_tier0_tool"}),
+ ):
+ assert classify_tool_domain("synthetic_tier0_tool") == "core"
+
+ assert classify_tool_domain("get_landing_page_custom") == "insight"
+ assert classify_tool_domain("get_page_ctr_detail") == "ctr"
+ assert classify_tool_domain("list_competitor_domains") == "backlinks"
+ assert classify_tool_domain("list_compare_new_issues") == "drift"
+
+ meta = registry.tool_meta()
+ meta_with_unknown = {**meta, "orphan_tool": {"domain": "noncanonical_domain", "tier": 1}}
+ by_domain = tools_by_domain(meta_with_unknown)
+ assert "noncanonical_domain" in by_domain
+
+ tier1 = tool_names_for_tier(meta, 1)
+ assert tier1
+ full_bundle = tool_names_for_mcp_bundle(meta, "full")
+ assert len(full_bundle) == len(meta)
+ core_bundle = tool_names_for_mcp_bundle(meta, "core")
+ assert TIER_0_TOOLS <= core_bundle
+
+ catalog = domains_catalog(meta)
+ assert catalog
+ assert all(row["domain"] in CANONICAL_DOMAINS for row in catalog)
+ assert domains_catalog({}) == []
+
+
+def test_tool_selector_edge_cases() -> None:
+ with patch.dict(os.environ, {"CHAT_TOOL_MAX": "not-a-number"}):
+ assert chat_tool_max() >= len(TIER_0_TOOLS) + 1
+
+ history = [
+ {"role": "assistant", "content": "hi"},
+ {"role": "user", "content": "show gsc clicks and ga4 landing pages"},
+ ]
+ names = select_tools_for_turn("show gsc clicks", history=history)
+ assert "get_google_summary" in names or "get_gsc_top_queries" in names
+
+ with_extra = select_tools_for_turn("hello", extra_names={"search_audit_tools"})
+ assert "search_audit_tools" in with_extra
+
+ capped = apply_tool_cap(set(f"tool_{i}" for i in range(200)) | set(TIER_0_TOOLS), 50)
+ assert len(capped) <= 50
+ assert TIER_0_TOOLS <= capped
+
+ text = compact_tool_list({"b_tool", "a_tool"})
+ assert text.startswith("- a_tool")
+
+
+def test_keywords_brand_and_intent(conn: MagicMock, ctx: Ctx) -> None:
+ assert kw_mod.get_brand_keyword_split(conn, Ctx(property_id=None), {})["error"]
+ with patch.object(Ctx, "load_keywords", return_value=None):
+ assert kw_mod.get_brand_keyword_split(conn, ctx, {})["missing"]
+ with patch.object(Ctx, "load_keywords", return_value={
+ "brand_name": "Acme",
+ "rows": [
+ {"keyword": "acme shoes", "is_branded": True},
+ {"keyword": "buy shoes", "is_branded": False},
+ ],
+ }):
+ split = kw_mod.get_brand_keyword_split(conn, ctx, {})
+ assert split["branded_count"] == 1
+ assert split["non_branded_count"] == 1
+
+ assert kw_mod.list_keywords_by_intent(conn, ctx, {})["error"] == "intent is required"
+ with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "a", "intent": "informational"}]}):
+ assert kw_mod.list_keywords_by_intent(conn, ctx, {"intent": "informational"})["total"] == 1
+
+
+def test_crawl_js_delta_skips_empty_url(conn: MagicMock, ctx: Ctx) -> None:
+ df = pd.DataFrame([
+ {"url": "", "fetch_method": "static", "word_count": 100, "title": "A", "h1": "H"},
+ {"url": "https://ex.com/", "fetch_method": "static", "word_count": 100, "title": "A", "h1": "H"},
+ {"url": "https://ex.com/", "fetch_method": "rendered", "word_count": 200, "title": "B", "h1": "H2"},
+ ])
+ with patch.object(Ctx, "load_crawl_df", return_value=df):
+ result = crawl_mod.list_pages_js_rendering_delta(conn, ctx, {})
+ assert result["total"] == 1
diff --git a/web/app/api/crawl/page-html/route.ts b/web/app/api/crawl/page-html/route.ts
new file mode 100644
index 00000000..71f34003
--- /dev/null
+++ b/web/app/api/crawl/page-html/route.ts
@@ -0,0 +1,75 @@
+import { NextResponse, type NextRequest } from 'next/server';
+import { forbiddenIfNotLocal } from '@/server/localOnly';
+import { withReportDb } from '@/server/reportDb';
+import { deletePageHtmlForRun, listCrawlPageHtmlRuns } from '@/lib/loadReportDb';
+import type { ApiRouteHandler } from '@/types/api';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+type DeleteBody = {
+ crawlRunId?: number | null;
+};
+
+/**
+ * GET /api/crawl/page-html?limit=30
+ * Lists recent crawl runs with stored HTML stats.
+ */
+export const GET: ApiRouteHandler = async (request: NextRequest): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+
+ const limitRaw = Number(request.nextUrl.searchParams.get('limit') || '30');
+ const limit = Number.isFinite(limitRaw) ? Math.min(100, Math.max(1, limitRaw)) : 30;
+
+ try {
+ const runs = await withReportDb((client) => listCrawlPageHtmlRuns(client, { limit }));
+ return NextResponse.json({ runs });
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ return NextResponse.json({ error: msg, runs: [] }, { status: 500 });
+ }
+};
+
+/**
+ * DELETE /api/crawl/page-html
+ * Body: { crawlRunId: number }
+ * Removes raw HTML for one crawl run; crawl results and reports are kept.
+ */
+export const DELETE: ApiRouteHandler = async (request: NextRequest): Promise => {
+ const denied = forbiddenIfNotLocal(request);
+ if (denied) return denied;
+
+ let body: DeleteBody = {};
+ try {
+ body = (await request.json()) as DeleteBody;
+ } catch {
+ const crawlRunIdRaw = request.nextUrl.searchParams.get('crawlRunId');
+ if (crawlRunIdRaw) body.crawlRunId = Number(crawlRunIdRaw);
+ }
+
+ const crawlRunId =
+ body.crawlRunId != null && Number.isFinite(Number(body.crawlRunId))
+ ? Number(body.crawlRunId)
+ : null;
+
+ if (crawlRunId == null) {
+ return NextResponse.json({ error: 'crawlRunId is required' }, { status: 400 });
+ }
+
+ try {
+ const deletedPages = await withReportDb((client) => deletePageHtmlForRun(client, crawlRunId));
+ if (deletedPages === 0) {
+ return NextResponse.json({
+ ok: true,
+ crawlRunId,
+ deletedPages: 0,
+ message: 'No stored HTML found for this crawl run.',
+ });
+ }
+ return NextResponse.json({ ok: true, crawlRunId, deletedPages });
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ return NextResponse.json({ error: msg }, { status: 500 });
+ }
+};
diff --git a/web/app/api/jobs/[id]/route.ts b/web/app/api/jobs/[id]/route.ts
index e0849ade..54b94077 100644
--- a/web/app/api/jobs/[id]/route.ts
+++ b/web/app/api/jobs/[id]/route.ts
@@ -21,5 +21,6 @@ export const GET: ApiRouteHandlerWithParams<{ id: string }> = async (
exitCode: job.exitCode,
log: job.log,
error: job.error ?? null,
+ logTruncated: job.logTruncated ?? false,
});
};
diff --git a/web/app/api/jobs/route.ts b/web/app/api/jobs/route.ts
index 378047fa..8f9e4bd0 100644
--- a/web/app/api/jobs/route.ts
+++ b/web/app/api/jobs/route.ts
@@ -1,10 +1,6 @@
import { NextResponse, type NextRequest } from 'next/server';
import { forbiddenIfNotLocal } from '@/server/localOnly';
-import {
- getActiveRunningJob,
- listRecentPipelineJobs,
- reconcileStaleRunningJobs,
-} from '@/server/pipelineJobsDb';
+import { listPipelineJobsForApi } from '@/server/pipelineJobs';
import type { ApiRouteHandler } from '@/types/api';
export const runtime = 'nodejs';
@@ -24,11 +20,7 @@ export const GET: ApiRouteHandler = async (request: NextRequest): Promise {});
+ }).catch((err) => logPipelineDbError('writeAuditLog', err));
return NextResponse.json({ jobId: id });
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
- return NextResponse.json({ error: msg }, { status: 400 });
+ const isSlotTaken = msg === 'An audit job is already running';
+ return NextResponse.json({ error: msg }, { status: isSlotTaken ? 400 : 500 });
}
};
diff --git a/web/src/components/GoogleIntegrationsPanel.tsx b/web/src/components/GoogleIntegrationsPanel.tsx
index 84602e6b..2c181f7d 100644
--- a/web/src/components/GoogleIntegrationsPanel.tsx
+++ b/web/src/components/GoogleIntegrationsPanel.tsx
@@ -796,7 +796,7 @@ export default function GoogleIntegrationsPanel({
) : null;
const infoBannerText =
- 'Google Client ID/Secret and service account keys are stored in the database. Each site keeps its own OAuth connection and Search Console / Analytics property IDs.';
+ 'Google Client ID/Secret and service account JSON are stored app-wide in the database. Upload a service account for API access without per-site OAuth, or connect each site with OAuth for user-delegated access. Search Console and Analytics property IDs remain per site.';
const infoBanner = (
diff --git a/web/src/components/integrations/PropertyOpsSection.tsx b/web/src/components/integrations/PropertyOpsSection.tsx
index ba0c1fa6..388ddad0 100644
--- a/web/src/components/integrations/PropertyOpsSection.tsx
+++ b/web/src/components/integrations/PropertyOpsSection.tsx
@@ -27,14 +27,20 @@ export default function PropertyOpsSection({ propertyId }: PropertyOpsSectionPro
let cancelled = false;
setLoading(true);
void fetch(apiUrl(`/properties/${propertyId}/ops`))
- .then((res) => (res.ok ? res.json() : null))
- .then((data) => {
- if (cancelled || !data) return;
+ .then(async (res) => {
+ if (cancelled) return;
+ if (!res.ok) {
+ setMessage(s.loadFailed);
+ return;
+ }
+ const data = await res.json();
setScheduleCron(String(data.schedule_cron || ''));
setAlertWebhookUrl(String(data.alert_webhook_url || ''));
setAlertEmail(String(data.alert_email || ''));
})
- .catch(() => {})
+ .catch(() => {
+ if (!cancelled) setMessage(s.loadFailed);
+ })
.finally(() => {
if (!cancelled) setLoading(false);
});
@@ -47,9 +53,11 @@ export default function PropertyOpsSection({ propertyId }: PropertyOpsSectionPro
if (propertyId == null) return undefined;
let cancelled = false;
void fetch(apiUrl(`/properties/${propertyId}/google/links/status`))
- .then((res) => (res.ok ? res.json() : null))
- .then((status) => {
- if (cancelled || !status) return;
+ .then(async (res) => {
+ if (cancelled) return;
+ if (!res.ok) return;
+ const status = await res.json();
+ if (!status) return;
if (!status.hasData) {
setGscLinksStale(s.gscLinksMissing);
return;
@@ -63,11 +71,13 @@ export default function PropertyOpsSection({ propertyId }: PropertyOpsSectionPro
setGscLinksStale(null);
}
})
- .catch(() => {});
+ .catch(() => {
+ if (!cancelled) setGscLinksStale(s.loadFailed);
+ });
return () => {
cancelled = true;
};
- }, [propertyId, s.gscLinksMissing, s.gscLinksStale]);
+ }, [propertyId, s.gscLinksMissing, s.gscLinksStale, s.loadFailed]);
const handleSave = useCallback(async () => {
if (propertyId == null || readOnly) return;
diff --git a/web/src/components/overview/OverviewSummaryTab.tsx b/web/src/components/overview/OverviewSummaryTab.tsx
index 2484d69b..80864535 100644
--- a/web/src/components/overview/OverviewSummaryTab.tsx
+++ b/web/src/components/overview/OverviewSummaryTab.tsx
@@ -44,6 +44,7 @@ export function OverviewSummaryTab({ data, exportHref, compareHref, reportCount
const sj = strings.common;
const searchParams = useSearchParams();
const [healthDelta, setHealthDelta] = useState(null);
+ const [historyError, setHistoryError] = useState(null);
const keywordsHref = useMemo(
() => buildKeywordsHref(searchParams.toString()),
[searchParams],
@@ -62,16 +63,21 @@ export function OverviewSummaryTab({ data, exportHref, compareHref, reportCount
useEffect(() => {
const domain = data.site_name || '';
if (!domain) return;
+ setHistoryError(null);
void fetch(`/api/report/history?domain=${encodeURIComponent(domain)}&limit=2`)
- .then((r) => r.json())
- .then((payload: { history?: Array<{ healthScore?: number | null }> }) => {
+ .then(async (r) => {
+ if (!r.ok) {
+ setHistoryError(vo.historyTrendUnavailable ?? 'Could not load health trend.');
+ return;
+ }
+ const payload = (await r.json()) as { history?: Array<{ healthScore?: number | null }> };
const hist = payload.history || [];
if (hist.length >= 2 && currentHealth != null && hist[1]?.healthScore != null) {
setHealthDelta(currentHealth - Number(hist[1].healthScore));
}
})
- .catch(() => {});
- }, [data.site_name, currentHealth]);
+ .catch(() => setHistoryError(vo.historyTrendUnavailable ?? 'Could not load health trend.'));
+ }, [data.site_name, currentHealth, vo.historyTrendUnavailable]);
const execTopIssues = (data.executive_summary?.top_issues || []).slice(0, 5);
const execPriorities = (data.executive_summary?.priorities || []).filter(Boolean);
@@ -126,6 +132,9 @@ export function OverviewSummaryTab({ data, exportHref, compareHref, reportCount
{healthDelta} vs prior run)
)}
+ {historyError ? (
+ {historyError}
+ ) : null}
)}
{execSummary ? (
diff --git a/web/src/components/pipeline/CrawlPageHtmlManager.tsx b/web/src/components/pipeline/CrawlPageHtmlManager.tsx
new file mode 100644
index 00000000..a89f74f4
--- /dev/null
+++ b/web/src/components/pipeline/CrawlPageHtmlManager.tsx
@@ -0,0 +1,207 @@
+'use client';
+
+import { useCallback, useEffect, useState } from 'react';
+import { Loader2, Trash2 } from 'lucide-react';
+import { apiUrl } from '@/lib/publicBase';
+import { strings, format } from '@/lib/strings';
+import { formatReportGeneratedAt } from '@/lib/reportTimestamps';
+import type { CrawlPageHtmlRunRow } from '@/types/report';
+
+const sh = strings.pipelineRunner.storedHtml;
+
+function formatBytes(bytes: number): string {
+ if (!Number.isFinite(bytes) || bytes <= 0) return '0 B';
+ const units = ['B', 'KB', 'MB', 'GB'] as const;
+ let n = bytes;
+ let i = 0;
+ while (n >= 1024 && i < units.length - 1) {
+ n /= 1024;
+ i += 1;
+ }
+ const digits = i === 0 ? 0 : n >= 100 ? 0 : 1;
+ return `${n.toFixed(digits)} ${units[i]}`;
+}
+
+export interface CrawlPageHtmlManagerProps {
+ disabled?: boolean;
+}
+
+export default function CrawlPageHtmlManager({ disabled = false }: CrawlPageHtmlManagerProps) {
+ const [runs, setRuns] = useState([]);
+ const [loading, setLoading] = useState(true);
+ const [error, setError] = useState(null);
+ const [confirmRunId, setConfirmRunId] = useState(null);
+ const [deletingRunId, setDeletingRunId] = useState(null);
+
+ const loadRuns = useCallback(async () => {
+ setLoading(true);
+ setError(null);
+ try {
+ const res = await fetch(apiUrl('/crawl/page-html?limit=30'));
+ const body = (await res.json()) as { runs?: CrawlPageHtmlRunRow[]; error?: string };
+ if (!res.ok) {
+ setError(body.error || sh.loadFailed);
+ setRuns([]);
+ return;
+ }
+ setRuns(Array.isArray(body.runs) ? body.runs : []);
+ } catch {
+ setError(sh.loadFailed);
+ setRuns([]);
+ } finally {
+ setLoading(false);
+ }
+ }, []);
+
+ useEffect(() => {
+ void loadRuns();
+ }, [loadRuns]);
+
+ const handleDelete = async (crawlRunId: number) => {
+ setDeletingRunId(crawlRunId);
+ setError(null);
+ try {
+ const res = await fetch(apiUrl('/crawl/page-html'), {
+ method: 'DELETE',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ crawlRunId }),
+ });
+ const body = (await res.json().catch(() => ({}))) as { error?: string };
+ if (!res.ok) {
+ setError(body.error || sh.deleteFailed);
+ return;
+ }
+ setConfirmRunId(null);
+ setRuns((prev) =>
+ prev.map((row) =>
+ row.crawl_run_id === crawlRunId ? { ...row, page_count: 0, total_bytes: 0 } : row,
+ ),
+ );
+ } catch {
+ setError(sh.deleteFailed);
+ } finally {
+ setDeletingRunId(null);
+ }
+ };
+
+ const storedRuns = runs.filter((r) => r.page_count > 0);
+
+ return (
+
+
+
{sh.title}
+
{sh.hint}
+
+
+ {error ? (
+
+ {error}
+
+ ) : null}
+
+ {loading ? (
+
+
+ ) : storedRuns.length === 0 ? (
+
{sh.empty}
+ ) : (
+
+
+
+
+ {sh.colRun}
+ {sh.colSite}
+ {sh.colPages}
+ {sh.colSize}
+ {sh.colAction}
+
+
+
+ {storedRuns.map((row) => {
+ const isConfirm = confirmRunId === row.crawl_run_id;
+ const isDeleting = deletingRunId === row.crawl_run_id;
+ const createdLabel = row.created_at
+ ? formatReportGeneratedAt(row.created_at)
+ : format(sh.runIdLabel, { id: row.crawl_run_id });
+ return (
+
+
+ #{row.crawl_run_id}
+ {createdLabel}
+ {row.render_mode ? (
+ {row.render_mode}
+ ) : null}
+
+
+
+ {row.start_url}
+
+
+
+ {row.page_count.toLocaleString()}
+
+
+ {formatBytes(row.total_bytes)}
+
+
+ {isConfirm ? (
+
+
+ {format(sh.deleteConfirm, { id: row.crawl_run_id })}
+
+
+ setConfirmRunId(null)}
+ >
+ {sh.cancel}
+
+ void handleDelete(row.crawl_run_id)}
+ >
+ {isDeleting ? sh.deleting : sh.confirmDelete}
+
+
+
setConfirmRunId(row.crawl_run_id)}
+ className="inline-flex items-center gap-1 rounded-md px-2 py-1 text-muted-foreground hover:text-red-600 hover:bg-red-500/10 dark:hover:text-red-400 disabled:opacity-50"
+ >
+ {sh.delete}
+
+ )}
+
+
+ );
+ })}
+
+
+
+ )}
+
+ {!loading && storedRuns.length > 0 ? (
+
void loadRuns()}
+ disabled={disabled}
+ >
+ {sh.refresh}
+
+ ) : null}
+
(null);
@@ -395,6 +397,13 @@ export default function PipelineLogViewer({
) : null}
+ {logTruncated ? (
+
+
+
{strings.pipelineRunner.logTruncatedBanner}
+
diff --git a/web/src/components/pipeline/PipelineSettingsPanel.tsx b/web/src/components/pipeline/PipelineSettingsPanel.tsx
index d126fdc4..7676d9a1 100644
--- a/web/src/components/pipeline/PipelineSettingsPanel.tsx
+++ b/web/src/components/pipeline/PipelineSettingsPanel.tsx
@@ -13,6 +13,7 @@ import { useReadOnlySession } from '@/hooks/useReadOnlySession';
import Button from '@/components/Button';
import GoogleIntegrationsPanel from '@/components/GoogleIntegrationsPanel';
import ConfigField from './ConfigField';
+import CrawlPageHtmlManager from './CrawlPageHtmlManager';
import PipelineSettingsSectionTabs from './PipelineSettingsSectionTabs';
import {
PIPELINE_SETTINGS_GROUPS,
@@ -334,12 +335,15 @@ export default function PipelineSettingsPanel({
const pipelineSection = PIPELINE_CONFIG_SECTIONS.find((sec) => sec.id === sectionId);
if (pipelineSection) {
return (
- handlePipelineFieldChange(sectionId, key, value)}
- />
+ <>
+ handlePipelineFieldChange(sectionId, key, value)}
+ />
+ {sectionId === 'crawl' ? ('');
const [backgroundMode, setBackgroundMode] = useState(false);
const [configLoaded, setConfigLoaded] = useState(false);
@@ -191,6 +193,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) {
setBusy(true);
setStopping(false);
setLog('');
+ setLogTruncated(false);
setStatus('running');
setBackgroundMode(false);
if (jobCommand) {
@@ -210,6 +213,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) {
pollStopRef.current = pollPipelineJob(jobId, (job) => {
const displayLog = formatPipelineJobLog(job.log, job.error);
setLog(displayLog);
+ setLogTruncated(Boolean(job.logTruncated));
setStatus(job.status);
if (job.status === 'success' || job.status === 'error') {
stopPoll();
@@ -310,13 +314,23 @@ export function PipelineProvider({ children }: { children: ReactNode }) {
try {
const res = await fetch(apiUrl('/jobs?limit=1'));
const data = await res.json().catch(() => ({}));
- if (cancelled || !res.ok) return;
- const active = data.active as { id?: string; jobType?: string } | null;
- if (active?.id) {
+ if (cancelled) return;
+ if (!res.ok) {
+ const errMsg = String(data.error || res.statusText || s.resumeJobFailed);
+ setLoadError(errMsg);
+ logPipelineFailure('Resume active job failed', { status: res.status, error: errMsg });
+ return;
+ }
+ const active = data.active as { id?: string; jobType?: string; status?: string } | null;
+ if (active?.id && active.status === 'running') {
watchJob(active.id, { navigate: false, jobCommand: active.jobType || '' });
}
- } catch {
- /* non-fatal */
+ } catch (e) {
+ if (!cancelled) {
+ const errMsg = e instanceof Error ? e.message : s.resumeJobFailed;
+ setLoadError(errMsg);
+ logPipelineFailure('Resume active job failed', { error: e, message: errMsg });
+ }
}
})();
return () => {
@@ -398,7 +412,14 @@ export function PipelineProvider({ children }: { children: ReactNode }) {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ preset }),
- }).catch(() => {});
+ })
+ .then(async (res) => {
+ if (!res.ok) {
+ const data = await res.json().catch(() => ({}));
+ setSaveMsg(String(data.error || s.presetSaveFailed));
+ }
+ })
+ .catch(() => setSaveMsg(s.presetSaveFailed));
}
}, [configState.active_property_id]);
@@ -492,6 +513,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) {
stopPoll();
setBusy(true);
setLog('');
+ setLogTruncated(false);
setStatus('starting');
setBackgroundMode(false);
try {
@@ -597,6 +619,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) {
stopping,
activeJobId: activeJobIdRef.current,
log,
+ logTruncated,
status,
backgroundMode,
startUrl: String(configState.start_url ?? ''),
@@ -640,6 +663,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) {
busy,
stopping,
log,
+ logTruncated,
status,
backgroundMode,
browserCrawlStatus,
diff --git a/web/src/lib/formatPipelineLog.ts b/web/src/lib/formatPipelineLog.ts
index bb2a1c83..72c24964 100644
--- a/web/src/lib/formatPipelineLog.ts
+++ b/web/src/lib/formatPipelineLog.ts
@@ -13,6 +13,7 @@ export type PipelineLogLineKind =
export type PipelinePhase =
| 'config'
| 'crawl'
+ | 'content_analysis'
| 'lighthouse'
| 'report'
| 'keywords'
@@ -101,6 +102,7 @@ const ANSI_RE = /\x1b\[[0-9;]*m/g;
export const PHASE_LABELS: Record = {
config: 'Settings',
crawl: 'Crawl',
+ content_analysis: 'Content analysis',
lighthouse: 'Lighthouse',
report: 'Site audit',
keywords: 'Keywords',
@@ -133,6 +135,7 @@ const STEP_LABELS: Record = {
export const PIPELINE_STEPPER_PHASES: ProgressPhase[] = [
'config',
'crawl',
+ 'content_analysis',
'lighthouse',
'report',
'keywords',
@@ -148,6 +151,7 @@ export function phaseFromSectionTag(tag: string): PipelinePhase {
const t = tag.toLowerCase();
if (t.includes('config')) return 'config';
if (t.includes('crawl')) return 'crawl';
+ if (t.includes('content')) return 'content_analysis';
if (t.includes('lighthouse')) return 'lighthouse';
if (t.includes('report')) return 'report';
if (t.includes('plot')) return 'plot';
@@ -300,7 +304,7 @@ export function resolveActiveProgress(
function phaseFromProgressEvent(phase: string): PipelinePhase {
const p = phase.toLowerCase();
- if (p === 'config' || p === 'crawl' || p === 'lighthouse' || p === 'report' || p === 'keywords' || p === 'plot') {
+ if (p === 'config' || p === 'crawl' || p === 'content_analysis' || p === 'lighthouse' || p === 'report' || p === 'keywords' || p === 'plot') {
return p;
}
if (p === 'optional') return 'optional';
@@ -561,6 +565,7 @@ export const PIPELINE_LOG_LINE_CLASS: Record = {
export const PHASE_CHIP_CLASS: Record = {
config: 'border-violet-500/40 bg-violet-500/10 text-violet-200',
crawl: 'border-blue-500/40 bg-blue-500/10 text-blue-200',
+ content_analysis: 'border-teal-500/40 bg-teal-500/10 text-teal-200',
lighthouse: 'border-amber-500/40 bg-amber-500/10 text-amber-200',
report: 'border-green-500/40 bg-green-500/10 text-green-200',
keywords: 'border-pink-500/40 bg-pink-500/10 text-pink-200',
diff --git a/web/src/lib/loadReportDb.ts b/web/src/lib/loadReportDb.ts
index 83dda6a9..0a67ec98 100644
--- a/web/src/lib/loadReportDb.ts
+++ b/web/src/lib/loadReportDb.ts
@@ -1,5 +1,6 @@
import type { PoolClient } from 'pg';
import type {
+ CrawlPageHtmlRunRow,
CrawlRunRow,
CrawlRunSummary,
ReportListRow,
@@ -39,6 +40,69 @@ export async function getCrawlRunsRows(client: PoolClient): Promise> {
+ const stats = new Map();
+ if (!crawlRunIds.length) return stats;
+ try {
+ const { rows } = await client.query<{
+ crawl_run_id: string | number;
+ page_count: string | number;
+ total_bytes: string | number;
+ }>(
+ `SELECT crawl_run_id,
+ COUNT(*)::int AS page_count,
+ COALESCE(SUM(byte_length), 0)::bigint AS total_bytes
+ FROM crawl_page_html
+ WHERE crawl_run_id = ANY($1::bigint[])
+ GROUP BY crawl_run_id`,
+ [crawlRunIds],
+ );
+ for (const row of rows) {
+ stats.set(Number(row.crawl_run_id), {
+ page_count: Number(row.page_count) || 0,
+ total_bytes: Number(row.total_bytes) || 0,
+ });
+ }
+ } catch {
+ /* crawl_page_html may be missing */
+ }
+ return stats;
+}
+
+export async function listCrawlPageHtmlRuns(
+ client: PoolClient,
+ options: { limit?: number } = {},
+): Promise {
+ const limit = options.limit ?? 30;
+ const runs = await getCrawlRunsRows(client);
+ const slice = runs.slice(0, Math.max(1, limit));
+ const ids = slice.map((r) => r.id);
+ const stats = await getPageHtmlStatsByRunIds(client, ids);
+ return slice.map((run) => {
+ const s = stats.get(run.id);
+ return {
+ crawl_run_id: run.id,
+ start_url: run.start_url,
+ created_at: run.created_at,
+ render_mode: run.render_mode,
+ page_count: s?.page_count ?? 0,
+ total_bytes: s?.total_bytes ?? 0,
+ };
+ });
+}
+
+export async function deletePageHtmlForRun(client: PoolClient, crawlRunId: number): Promise {
+ try {
+ const res = await client.query('DELETE FROM crawl_page_html WHERE crawl_run_id = $1', [crawlRunId]);
+ return res.rowCount ?? 0;
+ } catch {
+ return 0;
+ }
+}
+
export async function getCrawlRunSummaries(client: PoolClient): Promise {
try {
const { rows } = await client.query(
diff --git a/web/src/lib/pipelineConfigSchema.ts b/web/src/lib/pipelineConfigSchema.ts
index ca1f0144..1613012f 100644
--- a/web/src/lib/pipelineConfigSchema.ts
+++ b/web/src/lib/pipelineConfigSchema.ts
@@ -147,6 +147,45 @@ export const PIPELINE_CONFIG_SECTIONS: PipelineConfigSection[] = [
{ key: 'store_outlinks', label: 'Store external links', type: 'bool', defaultValue: true },
{ key: 'store_content_excerpt', label: 'Store page text excerpt', type: 'bool', defaultValue: true },
{ key: 'content_excerpt_max_chars', label: 'Excerpt max chars', type: 'number', defaultValue: '4096' },
+ {
+ key: 'store_page_html',
+ label: 'Store raw page HTML',
+ type: 'bool',
+ defaultValue: false,
+ help: 'Persist fetched HTML per URL in the database for later content analysis. Increases DB size; does not affect CSV exports.',
+ },
+ {
+ key: 'max_stored_html_bytes',
+ label: 'Max stored HTML per page (bytes)',
+ type: 'number',
+ defaultValue: '2097152',
+ help: 'Skip pages whose HTML exceeds this size (default 2 MB). Only applies when store_page_html is enabled.',
+ },
+ {
+ key: 'run_content_analysis',
+ label: 'Run content analysis step',
+ type: 'bool',
+ defaultValue: false,
+ help: 'After crawl, analyze stored HTML and update word counts/keywords in crawl results. Requires store_page_html.',
+ },
+ {
+ key: 'content_analysis_strategy',
+ label: 'Content analysis strategy',
+ type: 'select',
+ defaultValue: 'main_only',
+ options: [
+ { value: 'main_only', label: 'Main content only (main/article)' },
+ { value: 'full_body', label: 'Full body text' },
+ ],
+ help: 'How to select page text when analyzing stored HTML.',
+ },
+ {
+ key: 'content_analysis_workers',
+ label: 'Content analysis workers',
+ type: 'number',
+ defaultValue: '4',
+ help: 'Parallel workers for the post-crawl content analysis step.',
+ },
{
key: 'custom_extraction_regex',
label: 'Custom extraction regex',
diff --git a/web/src/lib/pipelineDebug.ts b/web/src/lib/pipelineDebug.ts
index 79607aca..42646bf3 100644
--- a/web/src/lib/pipelineDebug.ts
+++ b/web/src/lib/pipelineDebug.ts
@@ -1,3 +1,9 @@
+/** Server-side pipeline DB write failures (visible in Next.js terminal). */
+export function logPipelineDbError(action: string, err: unknown): void {
+ const msg = err instanceof Error ? err.message : String(err);
+ console.error(`[pipeline-db] ${action}: ${msg}`);
+}
+
/** Structured pipeline errors in the browser console (devTools). */
export function logPipelineFailure(
context: string,
diff --git a/web/src/lib/pipelineJobEvents.ts b/web/src/lib/pipelineJobEvents.ts
index f9fb09db..d52a73fa 100644
--- a/web/src/lib/pipelineJobEvents.ts
+++ b/web/src/lib/pipelineJobEvents.ts
@@ -39,6 +39,7 @@ type JobPollUpdate = {
status: PipelineJob['status'];
log: string;
error?: string | null;
+ logTruncated?: boolean;
};
/**
@@ -64,8 +65,12 @@ export function pollPipelineJob(
if (cancelled) return;
try {
const res = await fetch(jobPath);
- const data: { status?: string; log?: string; error?: string | null } =
- await res.json().catch(() => ({}));
+ const data: {
+ status?: string;
+ log?: string;
+ error?: string | null;
+ logTruncated?: boolean;
+ } = await res.json().catch(() => ({}));
if (cancelled) return;
if (!res.ok) {
const errMsg = data.error || res.statusText;
@@ -81,7 +86,7 @@ export function pollPipelineJob(
const status = (data.status as PipelineJob['status']) || 'error';
const log = data.log || '';
const error = data.error ?? null;
- onUpdate({ status, log, error });
+ onUpdate({ status, log, error, logTruncated: Boolean(data.logTruncated) });
if (status === 'success' || status === 'error') {
finish();
}
diff --git a/web/src/lib/pipelineLiveEstimate.ts b/web/src/lib/pipelineLiveEstimate.ts
index 1a2dc2fe..cfb0ad82 100644
--- a/web/src/lib/pipelineLiveEstimate.ts
+++ b/web/src/lib/pipelineLiveEstimate.ts
@@ -24,6 +24,7 @@ export interface LivePipelineEstimate {
const PIPELINE_PHASE_ORDER: ProgressPhase[] = [
'config',
'crawl',
+ 'content_analysis',
'lighthouse',
'report',
'keywords',
diff --git a/web/src/server/crawlPageHtmlRoute.test.ts b/web/src/server/crawlPageHtmlRoute.test.ts
new file mode 100644
index 00000000..693397fe
--- /dev/null
+++ b/web/src/server/crawlPageHtmlRoute.test.ts
@@ -0,0 +1,80 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+import { NextRequest } from 'next/server';
+import { localRequest } from '@/server/testHelpers/routeTestUtils';
+
+const withReportDbMock = vi.fn();
+
+vi.mock('@/server/reportDb', () => ({
+ withReportDb: (fn: (client: unknown) => Promise) => withReportDbMock(fn),
+}));
+
+describe('/api/crawl/page-html', () => {
+ beforeEach(() => {
+ withReportDbMock.mockReset();
+ vi.resetModules();
+ });
+
+ it('GET returns runs with stats', async () => {
+ withReportDbMock.mockImplementation(async (fn) =>
+ fn({
+ query: vi.fn(),
+ }),
+ );
+ const loadMod = await import('@/lib/loadReportDb');
+ vi.spyOn(loadMod, 'listCrawlPageHtmlRuns').mockResolvedValue([
+ {
+ crawl_run_id: 3,
+ start_url: 'https://example.com',
+ created_at: '2026-06-01',
+ page_count: 12,
+ total_bytes: 4096,
+ },
+ ]);
+
+ const { GET } = await import('../../app/api/crawl/page-html/route');
+ const res = await GET(localRequest('/api/crawl/page-html'));
+ expect(res.status).toBe(200);
+ const body = await res.json();
+ expect(body.runs).toHaveLength(1);
+ expect(body.runs[0].crawl_run_id).toBe(3);
+ expect(body.runs[0].page_count).toBe(12);
+ });
+
+ it('DELETE removes HTML for a crawl run', async () => {
+ withReportDbMock.mockImplementation(async (fn) => fn({}));
+ const loadMod = await import('@/lib/loadReportDb');
+ vi.spyOn(loadMod, 'deletePageHtmlForRun').mockResolvedValue(5);
+
+ const { DELETE } = await import('../../app/api/crawl/page-html/route');
+ const res = await DELETE(
+ localRequest('/api/crawl/page-html', {
+ method: 'DELETE',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ crawlRunId: 7 }),
+ }),
+ );
+ expect(res.status).toBe(200);
+ const body = await res.json();
+ expect(body.ok).toBe(true);
+ expect(body.deletedPages).toBe(5);
+ expect(body.crawlRunId).toBe(7);
+ });
+
+ it('DELETE requires crawlRunId', async () => {
+ const { DELETE } = await import('../../app/api/crawl/page-html/route');
+ const res = await DELETE(
+ localRequest('/api/crawl/page-html', {
+ method: 'DELETE',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({}),
+ }),
+ );
+ expect(res.status).toBe(400);
+ });
+
+ it('rejects non-local hosts', async () => {
+ const { GET } = await import('../../app/api/crawl/page-html/route');
+ const res = await GET(new NextRequest('http://192.168.1.5:3000/api/crawl/page-html'));
+ expect(res.status).toBe(403);
+ });
+});
diff --git a/web/src/server/jobsRoute.test.ts b/web/src/server/jobsRoute.test.ts
index 38904a99..9f36d72f 100644
--- a/web/src/server/jobsRoute.test.ts
+++ b/web/src/server/jobsRoute.test.ts
@@ -1,25 +1,21 @@
import { describe, expect, it, vi, beforeEach } from 'vitest';
import { localRequest, remoteRequest } from '@/server/testHelpers/routeTestUtils';
-const reconcileMock = vi.fn();
-const listJobsMock = vi.fn();
-const activeJobMock = vi.fn();
+const listForApiMock = vi.fn();
-vi.mock('@/server/pipelineJobsDb', () => ({
- reconcileStaleRunningJobs: (...args: unknown[]) => reconcileMock(...args),
- listRecentPipelineJobs: (...args: unknown[]) => listJobsMock(...args),
- getActiveRunningJob: (...args: unknown[]) => activeJobMock(...args),
+vi.mock('@/server/pipelineJobs', () => ({
+ listPipelineJobsForApi: (...args: unknown[]) => listForApiMock(...args),
}));
describe('jobs route', () => {
beforeEach(() => {
- reconcileMock.mockReset();
- listJobsMock.mockReset();
- activeJobMock.mockReset();
+ listForApiMock.mockReset();
vi.resetModules();
- reconcileMock.mockResolvedValue(0);
- listJobsMock.mockResolvedValue([{ id: 'j1', status: 'completed' }]);
- activeJobMock.mockResolvedValue(null);
+ listForApiMock.mockResolvedValue({
+ jobs: [{ id: 'j1', status: 'completed' }],
+ active: null,
+ reconciled: 0,
+ });
});
it('returns 403 for non-local host', async () => {
diff --git a/web/src/server/pipelineJobs.test.ts b/web/src/server/pipelineJobs.test.ts
new file mode 100644
index 00000000..6ec54fd9
--- /dev/null
+++ b/web/src/server/pipelineJobs.test.ts
@@ -0,0 +1,151 @@
+import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
+import { PIPELINE_LOG_MAX, PIPELINE_LOG_TRIM } from '@/server/pipelineJobsDb';
+
+const tryClaimMock = vi.fn();
+const reconcileMock = vi.fn();
+const appendLogMock = vi.fn();
+const finishMock = vi.fn();
+const getJobFromDbMock = vi.fn();
+const getActiveMock = vi.fn();
+const listRecentMock = vi.fn();
+const markOrphanMock = vi.fn();
+
+vi.mock('@/server/pipelineJobsDb', async (importOriginal) => {
+ const actual = await importOriginal();
+ return {
+ ...actual,
+ tryClaimRunningPipelineJob: (...args: unknown[]) => tryClaimMock(...args),
+ reconcileStaleRunningJobs: (...args: unknown[]) => reconcileMock(...args),
+ appendPipelineJobLog: (...args: unknown[]) => appendLogMock(...args),
+ finishPipelineJob: (...args: unknown[]) => finishMock(...args),
+ getPipelineJobFromDb: (...args: unknown[]) => getJobFromDbMock(...args),
+ getActiveRunningJob: (...args: unknown[]) => getActiveMock(...args),
+ listRecentPipelineJobs: (...args: unknown[]) => listRecentMock(...args),
+ markRunningJobOrphaned: (...args: unknown[]) => markOrphanMock(...args),
+ };
+});
+
+const spawnMock = vi.fn();
+
+vi.mock('child_process', () => ({
+ spawn: (...args: unknown[]) => spawnMock(...args),
+}));
+
+function makeProc() {
+ const handlers: Record void>> = {};
+ return {
+ stdout: { on: vi.fn() },
+ stderr: { on: vi.fn() },
+ killed: false,
+ kill: vi.fn(),
+ on(event: string, fn: (arg?: unknown) => void) {
+ handlers[event] = handlers[event] || [];
+ handlers[event].push(fn);
+ },
+ emit(event: string, arg?: unknown) {
+ for (const fn of handlers[event] || []) fn(arg);
+ },
+ };
+}
+
+describe('pipelineJobs', () => {
+ beforeEach(() => {
+ vi.resetModules();
+ delete process.env.DATABASE_URL;
+ delete globalThis.__websiteProfilingPipelineJobs;
+ delete globalThis.__websiteProfilingPipelineProcesses;
+ tryClaimMock.mockReset();
+ reconcileMock.mockReset();
+ appendLogMock.mockReset();
+ finishMock.mockReset();
+ getJobFromDbMock.mockReset();
+ getActiveMock.mockReset();
+ listRecentMock.mockReset();
+ markOrphanMock.mockReset();
+ spawnMock.mockReset();
+ reconcileMock.mockResolvedValue(0);
+ listRecentMock.mockResolvedValue([]);
+ getActiveMock.mockResolvedValue(null);
+ markOrphanMock.mockResolvedValue(false);
+ finishMock.mockResolvedValue(undefined);
+ appendLogMock.mockResolvedValue(false);
+ });
+
+ afterEach(() => {
+ delete process.env.DATABASE_URL;
+ });
+
+ it('startPipelineJobAsync throws when in-memory job already running (no DB)', async () => {
+ const { startPipelineJobAsync } = await import('@/server/pipelineJobs');
+ const proc = makeProc();
+ spawnMock.mockReturnValue(proc);
+
+ const id = await startPipelineJobAsync('crawl', null, {});
+ expect(id).toBeTruthy();
+
+ await expect(startPipelineJobAsync('crawl', null, {})).rejects.toThrow(
+ /already running/i,
+ );
+ });
+
+ it('startPipelineJobAsync rejects when atomic claim fails', async () => {
+ process.env.DATABASE_URL = 'postgres://local/test';
+ tryClaimMock.mockResolvedValueOnce(true).mockResolvedValueOnce(false);
+ const proc = makeProc();
+ spawnMock.mockReturnValue(proc);
+
+ const { startPipelineJobAsync } = await import('@/server/pipelineJobs');
+ const first = await startPipelineJobAsync('crawl', null, {});
+ expect(first).toBeTruthy();
+ proc.emit('close', 0);
+ await new Promise((r) => setTimeout(r, 0));
+
+ await expect(startPipelineJobAsync('crawl', null, {})).rejects.toThrow(
+ /already running/i,
+ );
+ expect(tryClaimMock).toHaveBeenCalledTimes(2);
+ expect(spawnMock).toHaveBeenCalledTimes(1);
+ });
+
+ it('sets logTruncated when in-memory log exceeds cap', async () => {
+ const proc = makeProc();
+ spawnMock.mockReturnValue(proc);
+
+ const { startPipelineJobAsync, getJobSync } = await import('@/server/pipelineJobs');
+ const id = await startPipelineJobAsync('crawl', null, {});
+ const dataCall = proc.stdout.on.mock.calls.find((call) => call[0] === 'data');
+ const dataHandler = dataCall?.[1] as ((c: Buffer) => void) | undefined;
+ expect(dataHandler).toBeDefined();
+ dataHandler?.(Buffer.from('x'.repeat(PIPELINE_LOG_MAX + 1)));
+
+ const updated = getJobSync(id);
+ expect(updated?.log.length).toBeLessThanOrEqual(PIPELINE_LOG_TRIM);
+ expect(updated?.logTruncated).toBe(true);
+ });
+
+ it('listPipelineJobsForApi reconciles orphan jobs without live process', async () => {
+ process.env.DATABASE_URL = 'postgres://local/test';
+ const startedAt = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+ reconcileMock.mockResolvedValue(0);
+ getActiveMock
+ .mockResolvedValueOnce({
+ id: 'job-orphan',
+ jobType: 'crawl',
+ status: 'running',
+ propertyId: null,
+ startedAt,
+ finishedAt: null,
+ exitCode: null,
+ error: null,
+ })
+ .mockResolvedValueOnce(null);
+ markOrphanMock.mockResolvedValue(true);
+ listRecentMock.mockResolvedValue([]);
+
+ const { listPipelineJobsForApi } = await import('@/server/pipelineJobs');
+ const result = await listPipelineJobsForApi(5);
+ expect(markOrphanMock).toHaveBeenCalledWith('job-orphan');
+ expect(result.reconciled).toBe(1);
+ expect(result.active).toBeNull();
+ });
+});
diff --git a/web/src/server/pipelineJobs.ts b/web/src/server/pipelineJobs.ts
index ab7a63e3..689eac4c 100644
--- a/web/src/server/pipelineJobs.ts
+++ b/web/src/server/pipelineJobs.ts
@@ -5,14 +5,20 @@ import { randomUUID } from 'crypto';
import { getPipelineSpawnEnv } from '@/server/pipelineSpawnEnv';
import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython';
import { buildPipelineJobErrorMessage } from '@/lib/pipelineJobErrorMessage';
+import { logPipelineDbError } from '@/lib/pipelineDebug';
import {
appendPipelineJobLog,
cancelPipelineJobInDb,
finishPipelineJob,
+ getActiveRunningJob,
getPipelineJobFromDb,
- insertPipelineJob,
- isAnyPipelineJobRunning,
+ listRecentPipelineJobs,
+ markRunningJobOrphaned,
+ PIPELINE_LOG_MAX,
+ PIPELINE_LOG_TRIM,
reconcileStaleRunningJobs,
+ tryClaimRunningPipelineJob,
+ type PipelineJobListItem,
} from '@/server/pipelineJobsDb';
import type { PipelineJob, PipelineJobEntry, PipelineJobStore } from '@/types/api';
@@ -22,6 +28,7 @@ function isDbJobsEnabled(): boolean {
const WEB_CWD = process.cwd();
const DEFAULT_REPO_ROOT = process.env.WEBSITE_PROFILING_ROOT || path.resolve(WEB_CWD, '..');
+const ORPHAN_JOB_MINUTES = Number(process.env.PIPELINE_JOB_ORPHAN_MINUTES || '5');
const ALLOWED_COMMANDS = new Set([
null,
@@ -38,11 +45,6 @@ const ALLOWED_COMMANDS = new Set([
'google',
]);
-/**
- * Next may bundle this module into separate server chunks per API route, so module-level
- * `Map` instances are not shared. Persist store on globalThis so POST /api/run and GET
- * /api/jobs/[id] always see the same jobs (also survives dev Fast Refresh better).
- */
function getStore(): PipelineJobStore {
if (!globalThis.__websiteProfilingPipelineJobs) {
globalThis.__websiteProfilingPipelineJobs = {
@@ -62,6 +64,14 @@ function getProcessMap(): Map {
const CANCELLED_MESSAGE = 'Cancelled by user';
+function trimInMemoryLog(entry: PipelineJobEntry, chunk: string): void {
+ entry.log += chunk;
+ if (entry.log.length > PIPELINE_LOG_MAX) {
+ entry.log = entry.log.slice(-PIPELINE_LOG_TRIM);
+ entry.logTruncated = true;
+ }
+}
+
function markJobFinished(
id: string,
entry: PipelineJobEntry,
@@ -77,7 +87,9 @@ function markJobFinished(
getStore().running = false;
getProcessMap().delete(id);
if (isDbJobsEnabled()) {
- void finishPipelineJob(id, status, exitCode, error).catch(() => {});
+ void finishPipelineJob(id, status, exitCode, error, entry.logTruncated).catch((err) =>
+ logPipelineDbError('finishPipelineJob', err),
+ );
}
}
@@ -103,13 +115,6 @@ function resolveRepoRoot(override: string | undefined | null): string {
return normalized;
}
-/**
- * Resolve and validate an absolute config file path.
- *
- * Allowed locations:
- * - Under repoRoot (always)
- * - Under DATA_DIR — the data volume (/data) in Docker
- */
function validateConfigPath(absPath: string, repoRoot: string): string {
const normalized = path.resolve(absPath);
@@ -127,40 +132,70 @@ export interface StartPipelineJobOptions {
propertyId?: number | null;
}
-/**
- * Start a pipeline job. When configAbsPath is omitted (the normal UI flow),
- * Python picks up settings from the pipeline_config table via DATABASE_URL.
- */
-export async function assertNoRunningJob(): Promise {
- if (isDbJobsEnabled()) {
- await reconcileStaleRunningJobs();
- if (await isAnyPipelineJobRunning()) {
- throw new Error('An audit job is already running');
+function hasLiveProcess(jobId: string): boolean {
+ const proc = getProcessMap().get(jobId);
+ return Boolean(proc && !proc.killed);
+}
+
+async function reconcileOrphanedActiveJob(active: PipelineJobListItem): Promise {
+ if (hasLiveProcess(active.id)) return false;
+ const started = new Date(active.startedAt).getTime();
+ if (Number.isNaN(started) || Date.now() - started < ORPHAN_JOB_MINUTES * 60 * 1000) {
+ return false;
+ }
+ const updated = await markRunningJobOrphaned(active.id);
+ if (updated) {
+ getStore().running = false;
+ const entry = getStore().jobs.get(active.id);
+ if (entry && !entry.finished) {
+ markJobFinished(active.id, entry, 'error', -1, 'Job process not found (server restarted)');
}
- return;
}
- if (getStore().running) {
- throw new Error('An audit job is already running');
+ return updated;
+}
+
+export interface PipelineJobsListResult {
+ jobs: PipelineJobListItem[];
+ active: PipelineJobListItem | null;
+ reconciled: number;
+}
+
+/** List jobs for GET /api/jobs with stale + orphan reconciliation. */
+export async function listPipelineJobsForApi(limit: number): Promise {
+ let reconciled = await reconcileStaleRunningJobs();
+ let active = await getActiveRunningJob();
+ if (active) {
+ const orphanReconciled = await reconcileOrphanedActiveJob(active);
+ if (orphanReconciled) {
+ reconciled += 1;
+ active = await getActiveRunningJob();
+ }
}
+ const jobs = await listRecentPipelineJobs(limit);
+ return { jobs, active, reconciled };
}
-export function startPipelineJob(
+/**
+ * Start a pipeline job. When configAbsPath is omitted (the normal UI flow),
+ * Python picks up settings from the pipeline_config table via DATABASE_URL.
+ */
+export async function startPipelineJobAsync(
command: string | null | undefined,
configAbsPath: string | null | undefined,
options: StartPipelineJobOptions = {},
-): string {
+): Promise {
if (command != null && command !== '' && !ALLOWED_COMMANDS.has(command)) {
throw new Error('Invalid command');
}
+
const store = getStore();
- if (!isDbJobsEnabled() && store.running) {
+ if (store.running) {
throw new Error('An audit job is already running');
}
const repoRoot = resolveRepoRoot(options.repoRoot);
const pythonExe = sanitizePython(options.python, repoRoot);
- // Validate config path only when explicitly provided
let cfgPath: string | null = null;
if (configAbsPath != null && String(configAbsPath).trim() !== '') {
cfgPath = validateConfigPath(String(configAbsPath), repoRoot);
@@ -170,23 +205,31 @@ export function startPipelineJob(
}
const id = randomUUID();
+ const jobType = command?.split(/\s+/)[0] || 'full';
+
+ if (isDbJobsEnabled()) {
+ const claimed = await tryClaimRunningPipelineJob(
+ id,
+ jobType,
+ options.propertyId ?? null,
+ null,
+ );
+ if (!claimed) {
+ throw new Error('An audit job is already running');
+ }
+ }
+
const entry: PipelineJobEntry = {
status: 'running',
exitCode: null,
log: '',
+ logTruncated: false,
};
store.jobs.set(id, entry);
store.running = true;
- const jobType = command?.split(/\s+/)[0] || 'full';
- if (isDbJobsEnabled()) {
- void insertPipelineJob(id, jobType, options.propertyId ?? null, null).catch(() => {});
- }
-
- // When no explicit config path is given, Python reads from PostgreSQL via DATABASE_URL.
const args = ['-m', 'src'];
if (cfgPath) args.push('--config', cfgPath);
- // Support multi-word commands like "keywords --enrich-google"
if (command) args.push(...command.split(/\s+/).filter(Boolean));
const proc = spawn(pythonExe, args, {
@@ -198,12 +241,13 @@ export function startPipelineJob(
const append = (chunk: Buffer | string): void => {
const text = chunk.toString();
- entry.log += text;
- if (entry.log.length > 256_000) {
- entry.log = entry.log.slice(-200_000);
- }
+ trimInMemoryLog(entry, text);
if (isDbJobsEnabled()) {
- void appendPipelineJobLog(id, text).catch(() => {});
+ void appendPipelineJobLog(id, text)
+ .then((truncated) => {
+ if (truncated) entry.logTruncated = true;
+ })
+ .catch((err) => logPipelineDbError('appendPipelineJobLog', err));
}
};
@@ -233,17 +277,45 @@ export function startPipelineJob(
return id;
}
+/** @deprecated Use startPipelineJobAsync */
+export async function assertNoRunningJob(): Promise {
+ if (isDbJobsEnabled()) {
+ await reconcileStaleRunningJobs();
+ const active = await getActiveRunningJob();
+ if (active) {
+ await reconcileOrphanedActiveJob(active);
+ const stillActive = await getActiveRunningJob();
+ if (stillActive) throw new Error('An audit job is already running');
+ }
+ return;
+ }
+ if (getStore().running) {
+ throw new Error('An audit job is already running');
+ }
+}
+
export async function getJob(id: string): Promise {
+ const memory = getStore().jobs.get(id);
if (isDbJobsEnabled()) {
const fromDb = await getPipelineJobFromDb(id);
- if (fromDb) return fromDb;
- // Fall back to in-memory for jobs started in this process before DB insert completes.
- return getStore().jobs.get(id) ?? null;
+ if (fromDb) {
+ if (memory) {
+ return {
+ ...fromDb,
+ log: memory.log.length >= fromDb.log.length ? memory.log : fromDb.log,
+ logTruncated: memory.logTruncated || fromDb.logTruncated,
+ status: memory.finished ? memory.status : fromDb.status,
+ exitCode: memory.finished ? memory.exitCode : fromDb.exitCode,
+ error: memory.error ?? fromDb.error,
+ };
+ }
+ return fromDb;
+ }
+ return memory ?? null;
}
- return getStore().jobs.get(id) ?? null;
+ return memory ?? null;
}
-/** Sync read from in-memory cache only (legacy). */
export function getJobSync(id: string): PipelineJob | null {
return getStore().jobs.get(id) ?? null;
}
@@ -254,10 +326,6 @@ export interface CancelPipelineJobResult {
error?: string;
}
-/**
- * Stop a running pipeline job. Kills the child process when this server instance
- * spawned it; otherwise marks the DB row cancelled (best effort after restart).
- */
export async function cancelPipelineJob(id: string): Promise {
const trimmed = id.trim();
if (!trimmed) {
@@ -272,9 +340,13 @@ export async function cancelPipelineJob(id: string): Promise {});
+ void appendPipelineJobLog(trimmed, cancelLine)
+ .then((truncated) => {
+ if (truncated) entry.logTruncated = true;
+ })
+ .catch((err) => logPipelineDbError('appendPipelineJobLog', err));
}
try {
proc.kill();
diff --git a/web/src/server/pipelineJobsDb.ts b/web/src/server/pipelineJobsDb.ts
index 6a5491d6..73ad2512 100644
--- a/web/src/server/pipelineJobsDb.ts
+++ b/web/src/server/pipelineJobsDb.ts
@@ -3,40 +3,107 @@ import type { PoolClient } from 'pg';
import { withDb } from '@/server/db';
import type { PipelineJob } from '@/types/api';
-const LOG_MAX = 256_000;
-const LOG_TRIM = 200_000;
+export const PIPELINE_LOG_MAX = 256_000;
+export const PIPELINE_LOG_TRIM = 200_000;
+
+const STALE_JOB_HOURS = Number(process.env.PIPELINE_JOB_STALE_HOURS || '1');
export function hashConfig(configPath: string | null): string | null {
if (!configPath) return null;
return createHash('sha256').update(configPath).digest('hex').slice(0, 16);
}
-export async function insertPipelineJob(
+function trimPipelineLog(log: string): { log: string; truncated: boolean } {
+ if (log.length <= PIPELINE_LOG_MAX) {
+ return { log, truncated: false };
+ }
+ return { log: log.slice(-PIPELINE_LOG_TRIM), truncated: true };
+}
+
+async function reconcileStaleRunningJobsWithClient(client: PoolClient): Promise {
+ const cur = await client.query<{ id: string }>(
+ `UPDATE pipeline_jobs
+ SET status = 'error',
+ error_text = COALESCE(error_text, 'Job interrupted (server restart or timeout)'),
+ finished_at = now()
+ WHERE status = 'running'
+ AND started_at < now() - ($1::text || ' hours')::interval
+ RETURNING id::text`,
+ [String(STALE_JOB_HOURS)],
+ );
+ return cur.rowCount ?? 0;
+}
+
+/** Mark jobs stuck in running state as error (e.g. after server restart). */
+export async function reconcileStaleRunningJobs(): Promise {
+ return withDb(async (client) => reconcileStaleRunningJobsWithClient(client));
+}
+
+/**
+ * Atomically claim the single running pipeline slot.
+ * Reconciles stale jobs in the same transaction before insert.
+ */
+export async function tryClaimRunningPipelineJob(
id: string,
jobType: string,
propertyId: number | null,
configHash: string | null,
-): Promise {
- await withDb(async (client) => {
- await client.query(
- `INSERT INTO pipeline_jobs (id, job_type, status, property_id, config_hash)
- VALUES ($1::uuid, $2, 'running', $3, $4)`,
- [id, jobType, propertyId, configHash],
+): Promise {
+ return withDb(async (client) => {
+ await client.query('BEGIN');
+ try {
+ await reconcileStaleRunningJobsWithClient(client);
+ const cur = await client.query<{ id: string }>(
+ `INSERT INTO pipeline_jobs (id, job_type, status, property_id, config_hash)
+ SELECT $1::uuid, $2, 'running', $3, $4
+ WHERE NOT EXISTS (SELECT 1 FROM pipeline_jobs WHERE status = 'running')
+ RETURNING id::text`,
+ [id, jobType, propertyId, configHash],
+ );
+ await client.query('COMMIT');
+ return (cur.rowCount ?? 0) > 0;
+ } catch (e) {
+ await client.query('ROLLBACK');
+ throw e;
+ }
+ });
+}
+
+export async function markRunningJobOrphaned(
+ id: string,
+ message = 'Job process not found (server restarted)',
+): Promise {
+ return withDb(async (client) => {
+ const cur = await client.query<{ id: string }>(
+ `UPDATE pipeline_jobs
+ SET status = 'error',
+ error_text = $2,
+ exit_code = -1,
+ finished_at = now()
+ WHERE id = $1::uuid AND status = 'running'
+ RETURNING id::text`,
+ [id, message],
);
+ return (cur.rowCount ?? 0) > 0;
});
}
-export async function appendPipelineJobLog(id: string, chunk: string): Promise {
- await withDb(async (client) => {
- const cur = await client.query<{ log_text: string }>(
- `SELECT log_text FROM pipeline_jobs WHERE id = $1::uuid FOR UPDATE`,
+export async function appendPipelineJobLog(id: string, chunk: string): Promise {
+ return withDb(async (client) => {
+ const cur = await client.query<{ log_text: string; log_truncated: boolean }>(
+ `SELECT log_text, log_truncated FROM pipeline_jobs WHERE id = $1::uuid FOR UPDATE`,
[id],
);
const row = cur.rows[0];
- if (!row) return;
- let log = (row.log_text || '') + chunk;
- if (log.length > LOG_MAX) log = log.slice(-LOG_TRIM);
- await client.query(`UPDATE pipeline_jobs SET log_text = $2 WHERE id = $1::uuid`, [id, log]);
+ if (!row) return false;
+ const combined = (row.log_text || '') + chunk;
+ const { log, truncated } = trimPipelineLog(combined);
+ const logTruncated = row.log_truncated || truncated;
+ await client.query(
+ `UPDATE pipeline_jobs SET log_text = $2, log_truncated = $3 WHERE id = $1::uuid`,
+ [id, log, logTruncated],
+ );
+ return logTruncated;
});
}
@@ -64,13 +131,23 @@ export async function finishPipelineJob(
status: 'success' | 'error',
exitCode: number | null,
error?: string,
+ logTruncated?: boolean,
): Promise {
await withDb(async (client) => {
+ if (logTruncated === undefined) {
+ await client.query(
+ `UPDATE pipeline_jobs
+ SET status = $2, exit_code = $3, error_text = $4, finished_at = now()
+ WHERE id = $1::uuid`,
+ [id, status, exitCode, error ?? null],
+ );
+ return;
+ }
await client.query(
`UPDATE pipeline_jobs
- SET status = $2, exit_code = $3, error_text = $4, finished_at = now()
+ SET status = $2, exit_code = $3, error_text = $4, finished_at = now(), log_truncated = $5
WHERE id = $1::uuid`,
- [id, status, exitCode, error ?? null],
+ [id, status, exitCode, error ?? null, logTruncated],
);
});
}
@@ -82,8 +159,9 @@ export async function getPipelineJobFromDb(id: string): Promise(
- `SELECT status, exit_code, log_text, error_text FROM pipeline_jobs WHERE id = $1::uuid`,
+ `SELECT status, exit_code, log_text, error_text, log_truncated FROM pipeline_jobs WHERE id = $1::uuid`,
[id],
);
const row = cur.rows[0];
@@ -94,6 +172,7 @@ export async function getPipelineJobFromDb(id: string): Promise {
- return withDb(async (client) => {
- const cur = await client.query<{ id: string }>(
- `UPDATE pipeline_jobs
- SET status = 'error',
- error_text = COALESCE(error_text, 'Job interrupted (server restart or timeout)'),
- finished_at = now()
- WHERE status = 'running'
- AND started_at < now() - ($1::text || ' hours')::interval
- RETURNING id::text`,
- [String(STALE_JOB_HOURS)],
- );
- return cur.rowCount ?? 0;
- });
-}
-
export async function listRecentPipelineJobs(limit = 20): Promise {
return withDb(async (client) => {
const cur = await client.query<{
diff --git a/web/src/strings.json b/web/src/strings.json
index 1b5010e7..0bd0f082 100644
--- a/web/src/strings.json
+++ b/web/src/strings.json
@@ -200,6 +200,9 @@
"stopJobFailed": "Could not stop the audit: {message}",
"outputLabel": "Audit run log",
"outputTitle": "Audit run log",
+ "logTruncatedBanner": "Older log lines were trimmed to save memory. Download log for the tail snapshot.",
+ "presetSaveFailed": "Could not save crawl preset for this property.",
+ "resumeJobFailed": "Could not resume the active audit job.",
"setupStepsAria": "Run audit setup steps",
"consoleFilterHint": "Site Audit run",
"statusLabel": "Status",
@@ -213,6 +216,27 @@
"browserCrawlBannerTitle": "Headless browser not available",
"browserCrawlBannerHint": "JavaScript and Auto crawl modes need Playwright Python packages and Chromium. Run: pip install -r requirements.txt. Ensure Chrome or Chromium is on PATH or set CHROME_PATH.",
"browserCrawlChecking": "Checking browser availability…",
+ "storedHtml": {
+ "title": "Stored raw HTML",
+ "hint": "Delete persisted page HTML for a specific crawl run to free database space. Crawl results, word counts, and audit reports are kept.",
+ "loading": "Loading stored HTML…",
+ "empty": "No stored HTML in recent crawl runs.",
+ "loadFailed": "Could not load stored HTML stats.",
+ "deleteFailed": "Could not delete stored HTML.",
+ "colRun": "Crawl run",
+ "colSite": "Start URL",
+ "colPages": "Pages",
+ "colSize": "Size",
+ "colAction": "Action",
+ "runIdLabel": "Run #{id}",
+ "delete": "Delete",
+ "deleteTitle": "Delete stored HTML for crawl run #{id}",
+ "deleteConfirm": "Delete all stored HTML for crawl run #{id}? Crawl metrics and reports stay intact.",
+ "confirmDelete": "Delete HTML",
+ "cancel": "Cancel",
+ "deleting": "Deleting…",
+ "refresh": "Refresh list"
+ },
"saveSettings": "Save settings",
"saveAndClose": "Save & close",
"saving": "Saving…",
@@ -306,6 +330,7 @@
"saving": "Saving…",
"saved": "Ops settings saved.",
"saveFailed": "Could not save ops settings.",
+ "loadFailed": "Could not load ops settings.",
"cronEndpointsHint": "Wire cron: curl -X POST http://localhost:3000/api/schedule/check and /api/alerts/check?propertyId={id}",
"gscLinksMissing": "No GSC Links CSV imported yet — upload from Search Console → Links for backlink velocity and competitor gap.",
"gscLinksStale": "GSC Links import is {days} days old — re-import weekly for accurate velocity."
@@ -1990,6 +2015,7 @@
"baselineLabel": "Baseline (older)",
"needTwoReports": "Run at least two full site audits (not crawl-only) for this domain to compare snapshots.",
"pickBaseline": "Select a baseline report to compare against the current report.",
+ "exportCsvFailed": "Could not export issue diff CSV.",
"noDifferences": "No URL-level changes between these two audits.",
"noneInCategory": "None in this category.",
"urlChangeListsUnavailable": "Per-URL change lists are missing from one or both audits. Site-wide metrics below may still apply. Run a full site audit to enable URL-level comparison.",
@@ -2126,6 +2152,7 @@
"browserPageErrorsLine": "{pages} page(s) had uncaught JavaScript errors during rendering.",
"viewJavaScriptErrors": "View JavaScript errors"
},
+ "historyTrendUnavailable": "Could not load health trend vs prior run.",
"googleStaleWarning": "Search Console & Analytics data is older than 7 days. Refresh from Integrations.",
"googlePartialWarning": "Search Console or Analytics data is missing from the last sync.",
"mlErrors": "AI insights reported {count} error{plural}",
diff --git a/web/src/types/api.ts b/web/src/types/api.ts
index 66f75c23..fe4e7629 100644
--- a/web/src/types/api.ts
+++ b/web/src/types/api.ts
@@ -23,6 +23,7 @@ export interface PipelineJob {
exitCode: number | null;
log: string;
error?: string;
+ logTruncated?: boolean;
}
/** In-memory job entry (server only). */
@@ -81,6 +82,7 @@ export interface JobStatusResponse {
exitCode: number | null;
log: string;
error: string | null;
+ logTruncated?: boolean;
}
export interface RunPostResponse {
diff --git a/web/src/types/report.ts b/web/src/types/report.ts
index ad940bcf..16352d9f 100644
--- a/web/src/types/report.ts
+++ b/web/src/types/report.ts
@@ -763,6 +763,16 @@ export interface CrawlRunSummary {
discovery_mode?: string;
}
+/** Stored raw HTML footprint for one crawl run (crawl_page_html). */
+export interface CrawlPageHtmlRunRow {
+ crawl_run_id: number;
+ start_url: string;
+ created_at: string;
+ render_mode?: string;
+ page_count: number;
+ total_bytes: number;
+}
+
export interface PortfolioCrawlConfig {
pages_crawled?: number;
max_pages_configured?: number;
diff --git a/web/src/views/CompareReports.tsx b/web/src/views/CompareReports.tsx
index 5257949d..5caaa229 100644
--- a/web/src/views/CompareReports.tsx
+++ b/web/src/views/CompareReports.tsx
@@ -343,11 +343,14 @@ export default function CompareReports({ searchQuery = '' }: ViewProps) {
a.click();
URL.revokeObjectURL(url);
})
- .catch(() => {});
+ .catch(() => setCopyHint(vc.exportCsvFailed));
}}
>
Export issue diff (CSV)
+ {copyHint ? (
+ {copyHint}
+ ) : null}
) : null}
{reportList.length >= 2 && compareReportId == null && !loading && !error ? (
@@ -91,7 +91,7 @@ WebsiteProfiling/
│ ├── integrations/ # Google Search Console, GA4, Bing, CrUX
│ ├── llm/ # AI enrich + chat agent
│ ├── tools/ # Exports, audit query tools, MCP helpers
-│ ├── mcp/ # MCP server (221 read-only tools)
+│ ├── mcp/ # MCP server (340 read-only tools, domain bundles)
│ ├── db/ # PostgreSQL storage layer
│ ├── commands/ # CLI subcommands
│ ├── cli.py # Pipeline entrypoint
@@ -147,6 +147,10 @@ Open [http://localhost:3000/home](http://localhost:3000/home).
./local-run stop # stop Postgres container
```
+`requirements.txt` pins direct Python dependencies to versions verified by `./local-test python`. Re-run the full test suite after intentional upgrades.
+
+Pipeline jobs: stuck `running` rows are reconciled after **1 hour** by default (`PIPELINE_JOB_STALE_HOURS`). Orphan jobs with no live server process are cleared after **5 minutes** (`PIPELINE_JOB_ORPHAN_MINUTES`). Increase `PIPELINE_JOB_STALE_HOURS` for crawls that routinely run longer than an hour.
+
**Tests**
```bash
@@ -182,7 +186,7 @@ Google Search Console / Analytics: connect via **Integrations** (gear icon) in t
| **Ollama** | Local daemon at `http://127.0.0.1:11434`. Chat UI lists installed models plus the live Ollama cloud catalog (billing: free local, account free tier, Pro). Native tool calling when supported; otherwise ReAct fallback. Pick the model in-chat without leaving the page. |
| **OpenAI** / **Anthropic** | API key in AI settings; native tool calling with streaming. |
-The agent uses the same **221 read-only audit tools** as the MCP server (`docs/MCP.md`). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`).
+The agent uses the same **340 read-only audit tools** as the MCP server (`docs/MCP.md`), with **dynamic routing** (~45 tools per turn plus router meta-tools). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`).
Production: `docker-compose.prod.yml` (set `POSTGRES_PASSWORD`, `AUTH_SECRET`).
diff --git a/alembic/versions/014_pipeline_job_log_truncated.py b/alembic/versions/014_pipeline_job_log_truncated.py
new file mode 100644
index 00000000..55cb6ec1
--- /dev/null
+++ b/alembic/versions/014_pipeline_job_log_truncated.py
@@ -0,0 +1,27 @@
+"""Add log_truncated flag to pipeline_jobs.
+
+Revision ID: 014_pipeline_log_truncated
+Revises: 013_crawl_discovery_edges
+"""
+from __future__ import annotations
+
+from alembic import op
+
+revision = "014_pipeline_log_truncated"
+down_revision = "013_crawl_discovery_edges"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.execute("""
+ ALTER TABLE pipeline_jobs
+ ADD COLUMN IF NOT EXISTS log_truncated BOOLEAN NOT NULL DEFAULT false;
+ """)
+
+
+def downgrade() -> None:
+ op.execute("""
+ ALTER TABLE pipeline_jobs
+ DROP COLUMN IF EXISTS log_truncated;
+ """)
diff --git a/alembic/versions/015_crawl_page_html.py b/alembic/versions/015_crawl_page_html.py
new file mode 100644
index 00000000..8c894ad8
--- /dev/null
+++ b/alembic/versions/015_crawl_page_html.py
@@ -0,0 +1,38 @@
+"""Add crawl_page_html table for per-URL raw HTML storage.
+
+Revision ID: 015_crawl_page_html
+Revises: 014_pipeline_log_truncated
+"""
+from __future__ import annotations
+
+from alembic import op
+
+revision = "015_crawl_page_html"
+down_revision = "014_pipeline_log_truncated"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.execute("""
+ CREATE TABLE crawl_page_html (
+ crawl_run_id BIGINT NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE,
+ url TEXT NOT NULL,
+ html TEXT NOT NULL,
+ status TEXT,
+ content_type TEXT,
+ fetch_method TEXT NOT NULL DEFAULT 'static',
+ byte_length INTEGER NOT NULL DEFAULT 0,
+ captured_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ PRIMARY KEY (crawl_run_id, url)
+ );
+ CREATE INDEX IF NOT EXISTS idx_crawl_page_html_run
+ ON crawl_page_html (crawl_run_id);
+ """)
+
+
+def downgrade() -> None:
+ op.execute("""
+ DROP INDEX IF EXISTS idx_crawl_page_html_run;
+ DROP TABLE IF EXISTS crawl_page_html;
+ """)
diff --git a/docs/MCP.md b/docs/MCP.md
index 2458bfa0..b39b8bab 100644
--- a/docs/MCP.md
+++ b/docs/MCP.md
@@ -12,8 +12,47 @@ export PYTHONPATH=src
## Cursor configuration
+Site Audit exposes **domain-scoped MCP servers** (like Cursor optional plugins). Connect only the bundles you need instead of loading all 340 tools in one server.
+
+| `WP_MCP_DOMAIN` | Typical tools | Use when |
+|-----------------|---------------|----------|
+| `core` (default) | Router, workflows, insight (~12) | General chat, tool search, coverage |
+| `crawl` | Crawl, on-page, schema, accessibility | Technical crawl audits |
+| `google` | Google, insight, CTR, keywords | GSC/GA4 analysis |
+| `links` | Links, backlinks, indexation | Link architecture |
+| `full` | All 340 tools | Debugging / legacy single-server setup |
+
Add to `.cursor/mcp.json` (or Cursor MCP settings):
+```json
+{
+ "mcpServers": {
+ "site-audit-core": {
+ "command": "python",
+ "args": ["-m", "website_profiling.mcp"],
+ "env": {
+ "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling",
+ "PYTHONPATH": "src",
+ "WP_MCP_DOMAIN": "core",
+ "WP_PROPERTY_ID": "1"
+ }
+ },
+ "site-audit-google": {
+ "command": "python",
+ "args": ["-m", "website_profiling.mcp"],
+ "env": {
+ "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling",
+ "PYTHONPATH": "src",
+ "WP_MCP_DOMAIN": "google",
+ "WP_PROPERTY_ID": "1"
+ }
+ }
+ }
+}
+```
+
+Single-server legacy setup (all tools):
+
```json
{
"mcpServers": {
@@ -23,6 +62,7 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings):
"env": {
"DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling",
"PYTHONPATH": "src",
+ "WP_MCP_DOMAIN": "full",
"WP_PROPERTY_ID": "1"
}
}
@@ -41,9 +81,14 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings):
| `audit://property/{id}/report/latest` | Payload key index (counts, not full blob) |
| `audit://property/{id}/report/{report_id}` | Payload key index for a specific report |
| `audit://glossary` | Excerpt from `docs/GLOSSARY.md` |
-| `audit://tools` | Tool catalog grouped by SEO domain |
+| `audit://tools` | Tool catalog for the connected `WP_MCP_DOMAIN` server |
+| `audit://domains` | Available MCP domain bundles and tool groupings |
-## Tools (221 read-only + export)
+## Tools (340 read-only + export)
+
+### Router and insight (Tier 0 — `WP_MCP_DOMAIN=core`)
+
+`search_audit_tools`, `list_tool_domains`, `get_data_coverage_report`, `run_insight_workflow`, `run_technical_workflow`, `run_keyword_workflow`, `run_domain_agent`, `get_landing_page_blended_table`, `get_opportunity_matrix`, `get_traffic_health_check`, `get_landing_page_full_diagnosis`, `get_issue_to_traffic_map`
### Export and deliverables
@@ -99,11 +144,11 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu
### Keywords
-`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `get_serp_feature_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`, `list_keywords_ctr_opportunity`, `expand_keywords`, `generate_content_brief`
+`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `get_serp_feature_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`, `list_keywords_ctr_opportunity`, `expand_keywords`, `generate_content_brief`, `get_brand_keyword_split`, `list_keywords_by_intent`
### Google and CTR
-`get_google_summary`, `get_google_integration_status`, `get_gsc_top_queries`, `get_gsc_top_pages`, `get_gsc_ctr_opportunity_pages`, `get_ga4_summary`, `get_ga4_page_metrics`, `get_gsc_page_query_slice`, `get_gsc_url_inspection`, `get_gsc_index_coverage`, `analyze_serp_snippet_for_url`
+`get_google_summary`, `get_google_integration_status`, `get_gsc_top_queries`, `get_gsc_top_pages`, `get_gsc_ctr_opportunity_pages`, `get_ga4_summary`, `get_ga4_page_metrics`, `get_gsc_page_query_slice`, `get_gsc_url_inspection`, `get_gsc_index_coverage`, `analyze_serp_snippet_for_url`, `get_gsc_daily_trend`, `get_ga4_daily_trend`, `get_ga4_by_device`, `get_ga4_by_channel`, `get_gsc_page_queries`
### Backlinks
@@ -163,6 +208,8 @@ Already available: `validate_rich_results`, `get_gsc_url_inspection`, `export_si
The same tools power **AI Chat** at [http://localhost:3000/chat](http://localhost:3000/chat). Enable AI in Run audit → AI settings.
+In-app chat uses **dynamic tool routing**: each turn loads Tier 0 router tools plus a domain-scoped subset (~45 tools), not the full catalog. Set `CHAT_TOOL_MODE=full` to load all tools for debugging.
+
## Ollama note
When the local Ollama daemon supports native tools (most current models, including Ollama cloud refs like `minimax-m3:cloud`), chat uses Ollama’s `/api/chat` tool format. Older or tool-less models fall back to JSON ReAct parsing. OpenAI and Anthropic always use native tool calling with streaming in the chat UI.
diff --git a/input.txt.example b/input.txt.example
index a6a9c56e..7494b553 100644
--- a/input.txt.example
+++ b/input.txt.example
@@ -15,6 +15,11 @@ allow_external = false
store_outlinks = true
store_content_excerpt = true
content_excerpt_max_chars = 4096
+store_page_html = false
+max_stored_html_bytes = 2097152
+run_content_analysis = false
+content_analysis_strategy = main_only
+content_analysis_workers = 4
preserve_crawl_history = true
crawl_stream_to_db = false
crawl_exclude_urls =
diff --git a/pipeline-config.example.txt b/pipeline-config.example.txt
index a43055d3..d1957b9e 100644
--- a/pipeline-config.example.txt
+++ b/pipeline-config.example.txt
@@ -16,6 +16,11 @@ allow_external = false
store_outlinks = true
store_content_excerpt = true
content_excerpt_max_chars = 4096
+store_page_html = false
+max_stored_html_bytes = 2097152
+run_content_analysis = false
+content_analysis_strategy = main_only
+content_analysis_workers = 4
preserve_crawl_history = true
crawl_stream_to_db = false
crawl_exclude_urls =
diff --git a/requirements.txt b/requirements.txt
index 9138a6e1..002458fe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,21 +1,21 @@
-requests>=2.28.0
-beautifulsoup4>=4.11.0
-lxml>=4.9.0
-pandas>=1.5.0
-tqdm>=4.64.0
-networkx>=2.8.0
-python-Wappalyzer>=0.3.1
+requests==2.34.2
+beautifulsoup4==4.14.3
+lxml==6.1.1
+pandas==3.0.3
+tqdm==4.67.3
+networkx==3.6.1
+python-Wappalyzer==0.3.1
# Local content analysis (duplicates, language)
-rapidfuzz>=3.0.0
-langdetect>=1.0.9
+rapidfuzz==3.14.5
+langdetect==1.0.9
# Google Search Console + GA4 integration (optional; required for `python -m src google`)
-google-auth>=2.0.0
-google-auth-oauthlib>=1.0.0
-google-api-python-client>=2.0.0
-google-analytics-data>=0.18.0
-google-analytics-admin>=0.22.0
+google-auth==2.53.0
+google-auth-oauthlib==1.4.0
+google-api-python-client==2.197.0
+google-analytics-data==0.23.0
+google-analytics-admin==0.30.0
# Keywords Explorer — Google Suggest + Wikipedia + Datamuse (all free, no auth needed)
# requests already listed above
@@ -23,28 +23,28 @@ google-analytics-admin>=0.22.0
# pytrends>=4.9,<5
# PostgreSQL
-psycopg[binary,pool]>=3.2
-sqlalchemy>=2.0.0
-alembic>=1.13
+psycopg[binary,pool]==3.3.4
+sqlalchemy==2.0.50
+alembic==1.18.4
# Audit export (PDF)
-reportlab>=4.0.0
+reportlab==4.5.1
# JavaScript rendering crawl (headless Chromium via Playwright)
-playwright>=1.49.0
+playwright==1.60.0
# LLM providers for AI enrichment (configure via web UI AI tab)
-httpx>=0.27.0
-openai>=1.0.0
-anthropic>=0.25.0
+httpx==0.28.1
+openai==2.41.0
+anthropic==0.107.0
# Spell-check / HTML validation extras
-pyspellchecker>=0.8.1
-html5lib>=1.1
+pyspellchecker==0.9.0
+html5lib==1.1
# MCP server for Cursor / Claude Desktop
-mcp>=1.0.0
+mcp~=1.0.0
# Dev / test
-pytest>=7.0.0
-pytest-cov>=5.0.0
+pytest==9.0.3
+pytest-cov==7.1.0
diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1
index ef9c7297..adc865d2 100644
--- a/scripts/local-test.ps1
+++ b/scripts/local-test.ps1
@@ -237,6 +237,10 @@ function Invoke-PytestReporting {
tests/test_optional_audits.py `
tests/test_property_profile.py `
tests/test_reporting_gaps.py `
+ tests/test_text_content_analysis.py `
+ tests/test_builder_image_buckets.py `
+ tests/test_pipeline_report_pool_unit.py `
+ tests/test_reporting_builder_modules.py `
--cov=website_profiling.reporting `
--cov-config=.coveragerc.reporting `
--cov-report=term-missing `
@@ -272,6 +276,7 @@ function Invoke-PytestTools {
tests/test_export_sitemap.py `
tests/test_mcp_registry.py `
tests/test_mcp_resources.py `
+ tests/test_tools_gate100_coverage.py `
tests/test_tools_branch_coverage.py `
--cov=website_profiling.tools `
--cov-config=.coveragerc.tools `
diff --git a/scripts/local-test.sh b/scripts/local-test.sh
index 3d32f7e5..357f1d53 100755
--- a/scripts/local-test.sh
+++ b/scripts/local-test.sh
@@ -126,6 +126,10 @@ run_pytest_reporting() {
tests/test_optional_audits.py \
tests/test_property_profile.py \
tests/test_reporting_gaps.py \
+ tests/test_text_content_analysis.py \
+ tests/test_builder_image_buckets.py \
+ tests/test_pipeline_report_pool_unit.py \
+ tests/test_reporting_builder_modules.py \
--cov=website_profiling.reporting \
--cov-config=.coveragerc.reporting \
--cov-report=term-missing \
@@ -149,6 +153,7 @@ run_pytest_tools() {
tests/test_audit_tools_links_extras.py \
tests/test_audit_tools_expansion.py \
tests/test_audit_tools_expansion_coverage.py \
+ tests/test_audit_tools_batch100_coverage.py \
tests/test_export_custom_coverage.py \
tests/test_export_artifacts_coverage.py \
tests/test_export_compare_coverage.py \
@@ -161,6 +166,9 @@ run_pytest_tools() {
tests/test_export_sitemap.py \
tests/test_mcp_registry.py \
tests/test_mcp_resources.py \
+ tests/test_router_tools.py \
+ tests/test_tool_selector.py \
+ tests/test_tools_gate100_coverage.py \
tests/test_tools_branch_coverage.py \
--cov=website_profiling.tools \
--cov-config=.coveragerc.tools \
diff --git a/src/website_profiling/analysis/log_parser.py b/src/website_profiling/analysis/log_parser.py
index 63999bc0..0ec4d519 100644
--- a/src/website_profiling/analysis/log_parser.py
+++ b/src/website_profiling/analysis/log_parser.py
@@ -15,6 +15,8 @@ def parse_access_log_lines(lines: list[str]) -> dict[str, Any]:
"""Return hit counts and URL sets from access log lines."""
url_hits: Counter[str] = Counter()
status_hits: Counter[str] = Counter()
+ paths_5xx: Counter[str] = Counter()
+ googlebot_path_hits: Counter[str] = Counter()
googlebot_hits = 0
parsed_lines = 0
@@ -29,16 +31,23 @@ def parse_access_log_lines(lines: list[str]) -> dict[str, Any]:
path, status, ua = m.group(1), m.group(2), m.group(3).lower()
url_hits[path] += 1
status_hits[status] += 1
+ if status.startswith("5"):
+ paths_5xx[path] += 1
if "googlebot" in ua:
googlebot_hits += 1
+ googlebot_path_hits[path] += 1
top_urls = [{"path": p, "hits": c} for p, c in url_hits.most_common(100)]
+ paths_5xx_rows = [{"path": p, "hits": c} for p, c in paths_5xx.most_common(100)]
+ googlebot_paths = [{"path": p, "hits": c} for p, c in googlebot_path_hits.most_common(100)]
return {
"parsed_lines": parsed_lines,
"unique_paths": len(url_hits),
"googlebot_hits": googlebot_hits,
"status_counts": dict(status_hits),
"top_paths": top_urls,
+ "paths_5xx": paths_5xx_rows,
+ "googlebot_paths": googlebot_paths,
}
diff --git a/src/website_profiling/commands/config_resolve.py b/src/website_profiling/commands/config_resolve.py
index 690a8a20..5468b51d 100644
--- a/src/website_profiling/commands/config_resolve.py
+++ b/src/website_profiling/commands/config_resolve.py
@@ -268,6 +268,7 @@ def build_parser() -> argparse.ArgumentParser:
nargs="?",
choices=[
"crawl",
+ "content_analysis",
"report",
"plot",
"lighthouse",
diff --git a/src/website_profiling/commands/pipeline_cmd.py b/src/website_profiling/commands/pipeline_cmd.py
index 24c00679..d50078fa 100644
--- a/src/website_profiling/commands/pipeline_cmd.py
+++ b/src/website_profiling/commands/pipeline_cmd.py
@@ -125,6 +125,10 @@ def run(cfg: dict, args: argparse.Namespace) -> None:
use_database = True
run_crawl = args.command == "crawl" or (args.command is None and get_bool(cfg, "run_crawl", True))
+ run_content_analysis = (
+ args.command == "content_analysis"
+ or (args.command is None and get_bool(cfg, "run_content_analysis", False))
+ )
run_report = args.command == "report" or (args.command is None and get_bool(cfg, "run_report", True))
run_plot = args.command == "plot" or (args.command is None and get_bool(cfg, "run_plot", False))
run_lighthouse = args.command is None and get_bool(cfg, "run_lighthouse", False)
@@ -132,12 +136,14 @@ def run(cfg: dict, args: argparse.Namespace) -> None:
lighthouse_max_pages = _cfg_int(cfg, "lighthouse_max_pages", 20)
if args.command is None and (
- run_crawl or run_lighthouse or run_lighthouse_on_pages or run_report or run_plot
+ run_crawl or run_content_analysis or run_lighthouse or run_lighthouse_on_pages or run_report or run_plot
):
emit_phase_start("config", message="Resolving pipeline configuration")
steps = []
if run_crawl:
steps.append("crawl")
+ if run_content_analysis:
+ steps.append("content-analysis")
if run_lighthouse_on_pages:
steps.append("lighthouse-on-pages")
elif run_lighthouse:
@@ -154,6 +160,11 @@ def run(cfg: dict, args: argparse.Namespace) -> None:
if run_crawl:
phase_results.append(run_pipeline_phase("crawl", lambda: _run_crawl(cfg, use_database)))
+ if run_content_analysis and use_database:
+ phase_results.append(
+ run_pipeline_phase("content_analysis", lambda: _run_content_analysis(cfg, use_database))
+ )
+
if run_lighthouse_on_pages and use_database:
phase_results.append(
run_pipeline_phase(
@@ -215,6 +226,11 @@ def _run_crawl(cfg: dict, use_database: bool) -> None:
preserve_crawl_history = get_bool(cfg, "preserve_crawl_history", True)
store_content_excerpt = get_bool(cfg, "store_content_excerpt", False)
content_excerpt_max_chars = _cfg_int(cfg, "content_excerpt_max_chars", 4096)
+ store_page_html = get_bool(cfg, "store_page_html", False)
+ max_stored_html_bytes = _cfg_int(cfg, "max_stored_html_bytes", 2_097_152)
+ run_content_analysis = get_bool(cfg, "run_content_analysis", False)
+ content_analysis_strategy = (cfg.get("content_analysis_strategy") or "main_only").strip()
+ content_analysis_workers = _cfg_int(cfg, "content_analysis_workers", 4)
crawl_stream_to_db = get_bool(cfg, "crawl_stream_to_db", False)
property_id = active_property_id_from_cfg(cfg)
render_mode = _normalize_render_mode(cfg)
@@ -258,6 +274,11 @@ def _run_crawl(cfg: dict, use_database: bool) -> None:
preserve_crawl_history=preserve_crawl_history,
store_content_excerpt=store_content_excerpt,
content_excerpt_max_chars=content_excerpt_max_chars,
+ store_page_html=store_page_html,
+ max_stored_html_bytes=max_stored_html_bytes,
+ run_content_analysis=run_content_analysis,
+ content_analysis_strategy=content_analysis_strategy,
+ content_analysis_workers=content_analysis_workers,
crawl_stream_to_db=crawl_stream_to_db,
property_id=property_id,
render_mode=render_mode,
@@ -289,6 +310,33 @@ def _run_crawl(cfg: dict, use_database: bool) -> None:
console_print("Crawl results: PostgreSQL")
+def _run_content_analysis(cfg: dict, use_database: bool) -> None:
+ if not use_database:
+ console_print("[Content analysis] Skipped (database required).", flush=True)
+ return
+ if not get_bool(cfg, "store_page_html", False):
+ console_print(
+ "[Content analysis] Skipped: enable store_page_html to persist HTML for analysis.",
+ flush=True,
+ )
+ return
+
+ from ..content_analysis import run_content_analysis
+
+ store_content_excerpt = get_bool(cfg, "store_content_excerpt", False)
+ excerpt_max = _cfg_int(cfg, "content_excerpt_max_chars", 4096)
+ strategy = (cfg.get("content_analysis_strategy") or "main_only").strip().lower()
+ workers = _cfg_int(cfg, "content_analysis_workers", 4)
+
+ console_print("[Content analysis] Starting...", flush=True)
+ run_content_analysis(
+ excerpt_max_chars=excerpt_max if store_content_excerpt else 0,
+ strategy=strategy,
+ workers=workers,
+ )
+ console_print("[Content analysis] Done.", flush=True)
+
+
def _run_lighthouse_on_pages(cfg: dict, lighthouse_max_pages: int) -> None:
from ..db import db_session, get_latest_crawl_run_id, read_crawl
from ..lighthouse.runner import run_lighthouse_on_pages as do_lighthouse_on_pages
diff --git a/src/website_profiling/common.py b/src/website_profiling/common.py
index b19c528b..899ebe09 100644
--- a/src/website_profiling/common.py
+++ b/src/website_profiling/common.py
@@ -1,575 +1,57 @@
"""
-Shared helpers for crawler and report/plot scripts.
+Shared helpers for crawler and report/plot scripts (re-export facade).
"""
-import json
-import os
-import warnings
-from urllib.parse import urljoin, urldefrag, urlparse
-import urllib.robotparser as robotparser
-import ast
-import math
-
-import pandas as pd
-from bs4 import BeautifulSoup
-
-
-def load_dataframe(path: str) -> pd.DataFrame:
- """Load a DataFrame from CSV or JSON (by extension)."""
- if not os.path.isfile(path):
- raise FileNotFoundError(path)
- path_lower = path.lower()
- if path_lower.endswith(".json"):
- return pd.read_json(path, orient="records")
- return pd.read_csv(path)
-
-
-def save_dataframe(df: pd.DataFrame, path: str) -> None:
- """Save a DataFrame to CSV or JSON (by extension). Uses default_handler for JSON to avoid numpy types."""
- path_lower = path.lower()
- if path_lower.endswith(".json"):
- df.to_json(path, orient="records", indent=2, date_format="iso", default_handler=str)
- else:
- df.to_csv(path, index=False)
-
-
-def load_edges(path: str) -> list[tuple[str, str]]:
- """Load edge list from CSV or JSON (by extension). Returns list of (from_url, to_url)."""
- if not os.path.isfile(path):
- return []
- path_lower = path.lower()
- try:
- if path_lower.endswith(".json"):
- with open(path, "r", encoding="utf-8") as f:
- data = json.load(f)
- if isinstance(data, list) and data and isinstance(data[0], dict):
- return [(str(o.get("from", "")), str(o.get("to", ""))) for o in data if o.get("from") and o.get("to")]
- return []
- edf = pd.read_csv(path)
- if {"from", "to"}.issubset(edf.columns):
- return [(str(a).rstrip("/"), str(b).rstrip("/")) for a, b in edf[["from", "to"]].values]
- except Exception:
- pass
- return []
-
-
-def save_edges(edges: list[tuple[str, str]], path: str) -> None:
- """Save edge list to CSV or JSON (by extension)."""
- path_lower = path.lower()
- if path_lower.endswith(".json"):
- data = [{"from": a, "to": b} for a, b in edges]
- with open(path, "w", encoding="utf-8") as f:
- json.dump(data, f, indent=2)
- else:
- pd.DataFrame(edges, columns=["from", "to"]).to_csv(path, index=False)
-
-
-_TRACKING_PARAM_PREFIXES = ("utm_",)
-_FACET_PARAM_NAMES = frozenset({"sort", "filter", "page", "offset", "limit"})
-
-
-def strip_crawl_query_params(url: str, ignore_params: list[str] | None = None) -> str:
- """Remove tracking and facet query params for crawl deduplication."""
- parsed = urlparse(url)
- if not parsed.query:
- return url.rstrip("/")
- ignore = {p.lower() for p in (ignore_params or [])}
- parts = []
- for pair in parsed.query.split("&"):
- if not pair:
- continue
- key = pair.split("=", 1)[0].lower()
- if key in ignore:
- continue
- if any(key.startswith(p) for p in _TRACKING_PARAM_PREFIXES):
- continue
- if key in _FACET_PARAM_NAMES:
- continue
- parts.append(pair)
- query = "&".join(parts)
- rebuilt = parsed._replace(query=query).geturl()
- return rebuilt.rstrip("/")
-
-
-def normalize_link(
- base: str,
- href: str,
- strip_params: bool = True,
- ignore_params: list[str] | None = None,
-) -> str | None:
- if not href:
- return None
- href = href.strip()
- if href.startswith(("mailto:", "javascript:", "tel:", "data:")):
- return None
- joined = urljoin(base, href)
- joined, _ = urldefrag(joined)
- parsed = urlparse(joined)
- if parsed.scheme not in ("http", "https"):
- return None
- out = joined.rstrip("/")
- if strip_params:
- out = strip_crawl_query_params(out, ignore_params)
- return out
-
-
-def _parse_rel_flags(rel_raw: str) -> tuple[bool, bool, bool]:
- parts = {p.strip().lower() for p in (rel_raw or "").split() if p.strip()}
- return ("nofollow" in parts, "sponsored" in parts, "ugc" in parts)
-
-
-def _anchor_text_from_tag(a) -> str:
- parts: list[str] = []
- for child in a.children:
- if getattr(child, "name", None) == "img":
- parts.append("[image]")
- elif isinstance(child, str):
- t = child.strip()
- if t:
- parts.append(t)
- text = " ".join(parts).strip() or a.get_text(separator=" ", strip=True)
- return (text or "")[:500]
-
-
-def parse_link_edges(base_url: str, html_text: str) -> tuple[str, list[dict]]:
- """Extract title and rich outbound link records from HTML."""
- soup = BeautifulSoup(html_text, "lxml")
- title_tag = (
- soup.title.string.strip()
- if soup.title and soup.title.string
- else ""
- )
- start_netloc = urlparse(base_url).netloc
- edges: list[dict] = []
- for a in soup.find_all("a", href=True):
- ln = normalize_link(base_url, a["href"])
- if not ln:
- continue
- rel_raw = a.get("rel") or ""
- if isinstance(rel_raw, list):
- rel_str = " ".join(str(x) for x in rel_raw)
- else:
- rel_str = str(rel_raw)
- nofollow, sponsored, ugc = _parse_rel_flags(rel_str)
- link_type = "internal" if urlparse(ln).netloc == start_netloc else "external"
- edges.append({
- "to_url": ln.rstrip("/"),
- "anchor_text": _anchor_text_from_tag(a),
- "rel": rel_str.strip(),
- "is_nofollow": nofollow,
- "is_sponsored": sponsored,
- "is_ugc": ugc,
- "link_type": link_type,
- })
- return title_tag, edges
-
-
-def parse_links(base_url: str, html_text: str) -> tuple[str, set[str]]:
- """Extract page title and set of absolute links from HTML. Returns (title, links)."""
- title, edges = parse_link_edges(base_url, html_text)
- return title, {e["to_url"] for e in edges}
-
-
-def parse_seo(base_url: str, html_text: str) -> tuple[str, int, str, int, str]:
- """
- Extract SEO-related fields from HTML.
- Returns (meta_description, meta_description_len, h1_text, h1_count, canonical_url).
- """
- soup = BeautifulSoup(html_text, "lxml")
- meta_desc = ""
- meta = soup.find("meta", attrs={"name": "description"})
- if meta and meta.get("content"):
- meta_desc = (meta["content"] or "").strip()
- if not meta_desc:
- og = soup.find("meta", attrs={"property": "og:description"})
- if og and og.get("content"):
- meta_desc = (og["content"] or "").strip()
- meta_desc_len = len(meta_desc)
-
- h1_tags = soup.find_all("h1")
- h1_count = len(h1_tags)
- h1_text = (h1_tags[0].get_text(separator=" ", strip=True) if h1_tags else "") or ""
-
- canonical_url = ""
- link_canonical = soup.find("link", attrs={"rel": "canonical"})
- if link_canonical and link_canonical.get("href"):
- canonical_url = normalize_link(base_url, link_canonical["href"]) or ""
-
- return meta_desc, meta_desc_len, h1_text, h1_count, canonical_url
-
-
-def parse_seo_extended(html_text: str, base_url: str) -> dict:
- """
- Extract extended SEO/accessibility/performance-related fields from HTML.
- Returns a dict with: viewport_present, viewport_content, noindex, has_schema,
- heading_sequence, images_without_alt, images_total, img_without_lazy, img_without_dimensions,
- aria_count, mixed_content_count.
- """
- soup = BeautifulSoup(html_text, "lxml")
- out = {
- "viewport_present": False,
- "viewport_content": "",
- "noindex": False,
- "has_schema": False,
- "heading_sequence": [],
- "heading_text": [],
- "images_without_alt": 0,
- "images_total": 0,
- "img_without_lazy": 0,
- "img_without_dimensions": 0,
- "aria_count": 0,
- "mixed_content_count": 0,
- }
- # Viewport
- viewport = soup.find("meta", attrs={"name": "viewport"})
- if viewport and viewport.get("content"):
- out["viewport_present"] = True
- out["viewport_content"] = (viewport["content"] or "").strip()
- # noindex
- robots = soup.find("meta", attrs={"name": "robots"})
- if robots and robots.get("content"):
- content = (robots["content"] or "").lower()
- out["noindex"] = "noindex" in content
- # Structured data: JSON-LD or microdata
- if soup.find("script", type="application/ld+json"):
- out["has_schema"] = True
- if soup.find(attrs={"itemscope": True}):
- out["has_schema"] = True
- # Heading order (h1..h6 tag names) and visible heading copy (for keywords / fingerprints)
- for tag in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
- if tag.name:
- out["heading_sequence"].append(tag.name)
- text = (tag.get_text(separator=" ", strip=True) or "").strip()
- if text:
- out["heading_text"].append(text)
- # Images: alt, lazy, dimensions
- base_scheme = urlparse(base_url).scheme.lower()
- for img in soup.find_all("img"):
- out["images_total"] += 1
- if not img.get("alt") and not img.get("aria-label"):
- out["images_without_alt"] += 1
- loading = (img.get("loading") or "").strip().lower()
- if loading != "lazy":
- out["img_without_lazy"] += 1
- if not img.get("width") and not img.get("height"):
- out["img_without_dimensions"] += 1
- src = img.get("src") or ""
- if base_scheme == "https" and src.strip().lower().startswith("http://"):
- out["mixed_content_count"] += 1
- # ARIA: count elements with any aria- attribute
- for el in soup.find_all(True):
- if getattr(el, "attrs", None) and any(k.startswith("aria-") for k in el.attrs):
- out["aria_count"] += 1
- # Mixed content: links and other src/href
- for tag in soup.find_all(True):
- for attr in ("href", "src", "srcset"):
- val = tag.get(attr)
- if not val or base_scheme != "https":
- continue
- val = str(val).strip().lower()
- if val.startswith("http://"):
- out["mixed_content_count"] += 1
- elif attr == "srcset":
- for part in val.split(","):
- part = part.strip().split()[0] if part.strip() else ""
- if part.startswith("http://"):
- out["mixed_content_count"] += 1
- return out
-
-
-_STOP_WORDS = frozenset({
- "the", "and", "for", "that", "this", "with", "from", "your", "have", "are",
- "was", "were", "been", "will", "would", "could", "should", "about", "which",
- "their", "there", "what", "when", "where", "more", "some", "than", "them",
- "other", "into", "over", "also", "just", "after", "before", "only", "then",
- "very", "most", "each", "such", "like", "does", "here", "because", "being",
- "well", "while", "these", "those", "both", "many", "much", "even", "back",
- "through", "still", "between", "every", "under", "last", "long", "great",
- "make", "same", "come", "take", "know", "they", "page", "site", "home",
- "click", "read", "view", "next", "menu", "main", "skip", "content", "link",
- "http", "https", "www", "html", "class", "none", "true", "false", "null",
-})
-
-
-def _count_syllables(word: str) -> int:
- word = word.lower().strip()
- if len(word) <= 3:
- return 1
- vowels = "aeiouy"
- count = 0
- prev_vowel = False
- for ch in word:
- is_vowel = ch in vowels
- if is_vowel and not prev_vowel:
- count += 1
- prev_vowel = is_vowel
- if word.endswith("e") and count > 1:
- count -= 1
- return max(1, count)
-
-
-def parse_content_text(soup, raw_html: str, excerpt_max_chars: int = 0) -> dict:
- """Extract content analytics: word count, reading level, content-to-HTML ratio, top keywords.
-
- excerpt_max_chars: when > 0, strip script/style from body and store a whitespace-normalized
- plain-text excerpt (truncated) in ``content_excerpt`` for analysis / AI / UI.
- """
- import re
- from collections import Counter
-
- body = soup.find("body")
- if body:
- for tag in body.find_all(["script", "style", "noscript"]):
- tag.decompose()
- body_text = body.get_text(separator=" ", strip=True) if body else ""
- words = [w for w in re.findall(r"[a-zA-Z]+", body_text) if len(w) >= 2]
- word_count = len(words)
-
- sentences = [s.strip() for s in re.split(r"[.!?]+", body_text) if len(s.strip()) > 5]
- sentence_count = max(1, len(sentences))
-
- total_syllables = sum(_count_syllables(w) for w in words) if words else 0
-
- reading_level = 0.0
- if word_count > 30:
- reading_level = (
- 0.39 * (word_count / sentence_count)
- + 11.8 * (total_syllables / max(1, word_count))
- - 15.59
- )
- reading_level = max(0.0, min(18.0, round(reading_level, 1)))
-
- html_len = max(1, len(raw_html))
- content_html_ratio = round(len(body_text) / html_len * 100, 1)
-
- keyword_words = [w.lower() for w in words if len(w) >= 4 and w.lower() not in _STOP_WORDS]
- top_keywords = Counter(keyword_words).most_common(10)
- max_kw = top_keywords[0][1] if top_keywords else 0
- kw_rows = []
- for w, c in top_keywords:
- score = round(100 * c / max_kw) if max_kw else 0
- kw_rows.append({"word": w, "count": c, "score": int(score)})
-
- excerpt = ""
- if excerpt_max_chars and excerpt_max_chars > 0 and body_text:
- excerpt = re.sub(r"\s+", " ", body_text.strip())
- if len(excerpt) > excerpt_max_chars:
- excerpt = excerpt[: excerpt_max_chars].rsplit(" ", 1)[0].strip() or excerpt[:excerpt_max_chars]
-
- return {
- "word_count": word_count,
- "reading_level": reading_level,
- "content_html_ratio": content_html_ratio,
- "top_keywords": json.dumps(kw_rows),
- "content_excerpt": excerpt,
- }
-
-
-def parse_social_meta(soup) -> dict:
- """Extract Open Graph and Twitter Card meta tags."""
- def _meta_content(attrs: dict) -> str:
- tag = soup.find("meta", attrs=attrs)
- return (tag.get("content") or "").strip() if tag else ""
-
- return {
- "og_title": _meta_content({"property": "og:title"}),
- "og_description": _meta_content({"property": "og:description"}),
- "og_image": _meta_content({"property": "og:image"}),
- "og_type": _meta_content({"property": "og:type"}),
- "twitter_card": _meta_content({"name": "twitter:card"}),
- "twitter_title": _meta_content({"name": "twitter:title"}),
- "twitter_image": _meta_content({"name": "twitter:image"}),
- }
-
-
-_TECH_PATTERNS = [
- ("WordPress", "html", "/wp-content/"),
- ("WordPress", "html", "/wp-includes/"),
- ("Drupal", "meta_generator", "Drupal"),
- ("Joomla", "meta_generator", "Joomla"),
- ("Shopify", "html", "cdn.shopify.com"),
- ("Squarespace", "html", "squarespace.com"),
- ("Wix", "html", "wix.com"),
- ("Next.js", "html", "__NEXT_DATA__"),
- ("Next.js", "html", "_next/static"),
- ("Nuxt.js", "html", "__NUXT__"),
- ("Gatsby", "html", "gatsby-"),
- ("React", "html", "data-reactroot"),
- ("React", "html", "__REACT_DEVTOOLS"),
- ("React", "html", "react.production.min"),
- ("Vue.js", "html", "__vue"),
- ("Vue.js", "html", "vue.min.js"),
- ("Angular", "html", "ng-version"),
- ("Angular", "html", "ng-app"),
- ("Svelte", "html", "svelte"),
- ("jQuery", "html", "jquery"),
- ("Bootstrap", "html", "bootstrap"),
- ("Tailwind CSS", "html", "tailwindcss"),
- ("Google Analytics", "html", "google-analytics.com/analytics.js"),
- ("Google Analytics", "html", "googletagmanager.com/gtag"),
- ("Google Tag Manager", "html", "googletagmanager.com/gtm.js"),
- ("Facebook Pixel", "html", "connect.facebook.net"),
- ("Hotjar", "html", "hotjar.com"),
- ("Google Fonts", "html", "fonts.googleapis.com"),
- ("Font Awesome", "html", "fontawesome"),
- ("Cloudflare", "header", "cf-ray"),
- ("Nginx", "header_server", "nginx"),
- ("Apache", "header_server", "apache"),
- ("LiteSpeed", "header_server", "litespeed"),
- ("Vercel", "header_server", "vercel"),
- ("Netlify", "header_server", "netlify"),
- ("Amazon CloudFront", "header", "x-amz-cf-id"),
- ("AWS", "header_server", "amazons3"),
-]
-
-# Module-level cache for Wappalyzer instance (avoids reloading technologies file per page).
-_wappalyzer_instance = None
-_wappalyzer_disabled = False
-
-
-def _is_wappalyzer_regex_warning(msg: str) -> bool:
- lower = msg.lower()
- return "compiling regex" in lower and "unbalanced parenthesis" in lower
-
-
-def detect_tech_wappalyzer(
- url: str,
- html: str,
- headers: dict,
- soup,
- wappalyzer=None,
-) -> str:
- """
- Detect technologies using python-Wappalyzer from existing HTML and headers.
- Returns JSON list of tech names. On any failure, falls back to parse_tech_stack(soup, headers, url).
- """
- global _wappalyzer_instance, _wappalyzer_disabled
- if _wappalyzer_disabled:
- return parse_tech_stack(soup, headers, url)
- try:
- from Wappalyzer import Wappalyzer, WebPage
- except ImportError:
- return parse_tech_stack(soup, headers, url)
- try:
- with warnings.catch_warnings(record=True) as caught:
- warnings.simplefilter("always")
- instance = wappalyzer if wappalyzer is not None else _wappalyzer_instance
- if instance is None:
- instance = Wappalyzer.latest()
- if wappalyzer is None:
- _wappalyzer_instance = instance
- webpage = WebPage(url, html=html, headers=headers)
- detected = instance.analyze(webpage)
- if any(_is_wappalyzer_regex_warning(str(w.message)) for w in caught):
- _wappalyzer_disabled = True
- _wappalyzer_instance = None
- return parse_tech_stack(soup, headers, url)
- return json.dumps(sorted(detected))
- except Exception:
- return parse_tech_stack(soup, headers, url)
-
-
-def parse_tech_stack(soup, headers: dict, url: str) -> str:
- """Detect technologies from HTML patterns and HTTP headers. Returns JSON list of tech names."""
- detected = set()
- html_str = str(soup).lower()
- meta_gen = soup.find("meta", attrs={"name": "generator"})
- generator = (meta_gen.get("content") or "").strip().lower() if meta_gen else ""
- server_header = (headers.get("Server") or headers.get("server") or "").lower()
-
- for name, source, pattern in _TECH_PATTERNS:
- pat = pattern.lower()
- if source == "html" and pat in html_str:
- detected.add(name)
- elif source == "meta_generator" and pat in generator:
- detected.add(name)
- elif source == "header":
- for v in headers.values():
- if isinstance(v, str) and pat in v.lower():
- detected.add(name)
- break
- elif source == "header_server" and pat in server_header:
- detected.add(name)
-
- return json.dumps(sorted(detected))
-
-
-def parse_resources(html_text: str, base_url: str) -> dict:
- """
- Extract script/link resource counts and total sizes (same-origin only, no fetch).
- Returns dict: script_count, link_stylesheet_count, script_urls, stylesheet_urls
- (URLs for optional later HEAD/GET). Does not fetch; caller may fetch with limit.
- """
- soup = BeautifulSoup(html_text, "lxml")
- parsed_base = urlparse(base_url)
- script_urls = []
- for s in soup.find_all("script", src=True):
- url = normalize_link(base_url, s["src"])
- if url and urlparse(url).netloc == parsed_base.netloc:
- script_urls.append(url)
- stylesheet_urls = []
- for link in soup.find_all("link", rel=lambda r: r and "stylesheet" in (r.lower() if isinstance(r, str) else "")):
- url = link.get("href") and normalize_link(base_url, link["href"])
- if url and urlparse(url).netloc == parsed_base.netloc:
- stylesheet_urls.append(url)
- return {
- "script_count": len(script_urls),
- "link_stylesheet_count": len(stylesheet_urls),
- "script_urls": script_urls,
- "stylesheet_urls": stylesheet_urls,
- }
-
-
-def _is_empty(raw) -> bool:
- if raw is None:
- return True
- if isinstance(raw, float) and math.isnan(raw):
- return True
- if raw == "":
- return True
- return False
-
-
-def parse_links_serialized(raw) -> list[str]:
- """
- Parse a serialized list of URLs from CSV/DataFrame (string list repr, comma-separated, or list).
- """
- if _is_empty(raw):
- return []
- if isinstance(raw, list):
- return [str(x).strip().rstrip("/") for x in raw if x]
- s = str(raw).strip()
- if not s:
- return []
- if s.startswith("[") and s.endswith("]"):
- try:
- v = ast.literal_eval(s)
- if isinstance(v, (list, tuple)):
- return [str(x).strip().rstrip("/") for x in v if x]
- except Exception:
- pass
- return [t.strip().rstrip("/") for t in s.split(",") if t.strip()]
-
-
-def load_robots(start_url: str):
- """Load robots.txt for the given URL; returns RobotFileParser or None on error."""
- parsed = urlparse(start_url)
- robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
- rp = robotparser.RobotFileParser()
- rp.set_url(robots_url)
- try:
- rp.read()
- return rp
- except Exception:
- return None
-
-
-# Column names that may contain serialized outlink lists (for building edges from crawl CSV)
-LINK_COLUMN_NAMES = (
- "links",
- "edges",
- "outlinks",
- "outlink_targets",
- "targets",
- "link_targets",
- "links_list",
+from .parsing import tech as _tech
+from .parsing.content import parse_content_text, parse_social_meta
+from .parsing.io import load_dataframe, load_edges, save_dataframe, save_edges
+from .parsing.links import (
+ LINK_COLUMN_NAMES,
+ normalize_link,
+ parse_link_edges,
+ parse_links,
+ parse_links_serialized,
+ strip_crawl_query_params,
+ _is_empty,
)
+from .parsing.robots import load_robots
+from .parsing.seo import parse_resources, parse_seo, parse_seo_extended
+from .parsing.tech import parse_tech_stack, _is_wappalyzer_regex_warning
+
+_wappalyzer_instance = _tech._wappalyzer_instance
+_wappalyzer_disabled = _tech._wappalyzer_disabled
+
+
+def detect_tech_wappalyzer(url, html, headers, soup, wappalyzer=None):
+ """Detect technologies; syncs wappalyzer module state with this facade for tests."""
+ _tech._wappalyzer_disabled = _wappalyzer_disabled
+ _tech._wappalyzer_instance = _wappalyzer_instance
+ result = _tech.detect_tech_wappalyzer(url, html, headers, soup, wappalyzer)
+ globals()["_wappalyzer_disabled"] = _tech._wappalyzer_disabled
+ globals()["_wappalyzer_instance"] = _tech._wappalyzer_instance
+ return result
+
+
+__all__ = [
+ "load_dataframe",
+ "save_dataframe",
+ "load_edges",
+ "save_edges",
+ "strip_crawl_query_params",
+ "normalize_link",
+ "parse_link_edges",
+ "parse_links",
+ "parse_seo",
+ "parse_seo_extended",
+ "parse_content_text",
+ "parse_social_meta",
+ "detect_tech_wappalyzer",
+ "parse_tech_stack",
+ "parse_resources",
+ "parse_links_serialized",
+ "load_robots",
+ "LINK_COLUMN_NAMES",
+ "_is_wappalyzer_regex_warning",
+ "_is_empty",
+ "_wappalyzer_disabled",
+ "_wappalyzer_instance",
+]
diff --git a/src/website_profiling/content_analysis/__init__.py b/src/website_profiling/content_analysis/__init__.py
new file mode 100644
index 00000000..71e1aded
--- /dev/null
+++ b/src/website_profiling/content_analysis/__init__.py
@@ -0,0 +1,7 @@
+"""Post-crawl content analysis from stored HTML."""
+from __future__ import annotations
+
+from .page import analyze_page_html
+from .pipeline import run_content_analysis
+
+__all__ = ["analyze_page_html", "run_content_analysis"]
diff --git a/src/website_profiling/content_analysis/batch.py b/src/website_profiling/content_analysis/batch.py
new file mode 100644
index 00000000..7e77a686
--- /dev/null
+++ b/src/website_profiling/content_analysis/batch.py
@@ -0,0 +1,78 @@
+"""Batch content analysis for a crawl run."""
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Iterator
+
+from psycopg import Connection
+
+from ..db.html_store import read_page_html_for_run
+from .page import ContentStrategy, analyze_page_html
+
+_PAGE_BATCH = 500
+
+
+def iter_html_pages(conn: Connection, crawl_run_id: int) -> Iterator[dict[str, Any]]:
+ offset = 0
+ while True:
+ chunk = list(read_page_html_for_run(conn, crawl_run_id, limit=_PAGE_BATCH, offset=offset))
+ if not chunk:
+ break
+ for row in chunk:
+ yield row
+ if len(chunk) < _PAGE_BATCH:
+ break
+ offset += _PAGE_BATCH
+
+
+def _analyze_row(
+ row: dict[str, Any],
+ *,
+ excerpt_max_chars: int,
+ strategy: ContentStrategy,
+) -> dict[str, Any] | None:
+ html = row.get("html")
+ url = row.get("url")
+ if not url or not html:
+ return None
+ fields = analyze_page_html(
+ str(html),
+ excerpt_max_chars=excerpt_max_chars,
+ strategy=strategy,
+ )
+ return {"url": str(url).rstrip("/"), **fields}
+
+
+def analyze_run_html(
+ conn: Connection,
+ crawl_run_id: int,
+ *,
+ excerpt_max_chars: int = 0,
+ strategy: ContentStrategy = "main_only",
+ workers: int = 4,
+) -> list[dict[str, Any]]:
+ """Analyze all stored HTML for a crawl run; returns merge payloads keyed by url."""
+ rows = list(iter_html_pages(conn, crawl_run_id))
+ if not rows:
+ return []
+
+ worker_count = max(1, int(workers))
+ if worker_count == 1 or len(rows) == 1:
+ out: list[dict[str, Any]] = []
+ for row in rows:
+ merged = _analyze_row(row, excerpt_max_chars=excerpt_max_chars, strategy=strategy)
+ if merged:
+ out.append(merged)
+ return out
+
+ results: list[dict[str, Any]] = []
+ with ThreadPoolExecutor(max_workers=worker_count) as pool:
+ futures = [
+ pool.submit(_analyze_row, row, excerpt_max_chars=excerpt_max_chars, strategy=strategy)
+ for row in rows
+ ]
+ for fut in as_completed(futures):
+ merged = fut.result()
+ if merged:
+ results.append(merged)
+ return results
diff --git a/src/website_profiling/content_analysis/constants.py b/src/website_profiling/content_analysis/constants.py
new file mode 100644
index 00000000..d1d7b7c8
--- /dev/null
+++ b/src/website_profiling/content_analysis/constants.py
@@ -0,0 +1,17 @@
+"""Shared constants for content analysis."""
+from __future__ import annotations
+
+STOP_WORDS = frozenset({
+ "the", "and", "for", "that", "this", "with", "from", "your", "have", "are",
+ "was", "were", "been", "will", "would", "could", "should", "about", "which",
+ "their", "there", "what", "when", "where", "more", "some", "than", "them",
+ "other", "into", "over", "also", "just", "after", "before", "only", "then",
+ "very", "most", "each", "such", "like", "does", "here", "because", "being",
+ "well", "while", "these", "those", "both", "many", "much", "even", "back",
+ "through", "still", "between", "every", "under", "last", "long", "great",
+ "make", "same", "come", "take", "know", "they", "page", "site", "home",
+ "click", "read", "view", "next", "menu", "main", "skip", "content", "link",
+ "http", "https", "www", "html", "class", "none", "true", "false", "null",
+})
+
+CHROME_TAGS = ("nav", "footer", "header", "aside")
diff --git a/src/website_profiling/content_analysis/dom_cleanup.py b/src/website_profiling/content_analysis/dom_cleanup.py
new file mode 100644
index 00000000..b35f554b
--- /dev/null
+++ b/src/website_profiling/content_analysis/dom_cleanup.py
@@ -0,0 +1,16 @@
+"""Remove non-content DOM nodes before text extraction."""
+from __future__ import annotations
+
+from bs4 import BeautifulSoup
+
+from .constants import CHROME_TAGS
+
+
+def cleanup_dom(soup: BeautifulSoup) -> BeautifulSoup:
+ for tag in soup.find_all(["script", "style", "noscript"]):
+ tag.decompose()
+ for tag in soup.find_all(CHROME_TAGS):
+ tag.decompose()
+ for tag in soup.find_all(attrs={"aria-hidden": "true"}):
+ tag.decompose()
+ return soup
diff --git a/src/website_profiling/content_analysis/excerpt.py b/src/website_profiling/content_analysis/excerpt.py
new file mode 100644
index 00000000..ae33a75e
--- /dev/null
+++ b/src/website_profiling/content_analysis/excerpt.py
@@ -0,0 +1,14 @@
+"""Plain-text excerpt generation."""
+from __future__ import annotations
+
+import re
+
+
+def build_excerpt(body_text: str, max_chars: int) -> str:
+ if not max_chars or max_chars <= 0 or not body_text:
+ return ""
+ excerpt = re.sub(r"\s+", " ", body_text.strip())
+ if len(excerpt) <= max_chars:
+ return excerpt
+ truncated = excerpt[:max_chars].rsplit(" ", 1)[0].strip()
+ return truncated or excerpt[:max_chars]
diff --git a/src/website_profiling/content_analysis/html_loader.py b/src/website_profiling/content_analysis/html_loader.py
new file mode 100644
index 00000000..9e34a2f5
--- /dev/null
+++ b/src/website_profiling/content_analysis/html_loader.py
@@ -0,0 +1,8 @@
+"""Load BeautifulSoup from raw HTML."""
+from __future__ import annotations
+
+from bs4 import BeautifulSoup
+
+
+def load_soup(raw_html: str) -> BeautifulSoup:
+ return BeautifulSoup(raw_html or "", "lxml")
diff --git a/src/website_profiling/content_analysis/html_ratio.py b/src/website_profiling/content_analysis/html_ratio.py
new file mode 100644
index 00000000..f14f399b
--- /dev/null
+++ b/src/website_profiling/content_analysis/html_ratio.py
@@ -0,0 +1,7 @@
+"""Content-to-HTML size ratio."""
+from __future__ import annotations
+
+
+def content_html_ratio(body_text: str, raw_html: str) -> float:
+ html_len = max(1, len(raw_html or ""))
+ return round(len(body_text or "") / html_len * 100, 1)
diff --git a/src/website_profiling/content_analysis/keywords.py b/src/website_profiling/content_analysis/keywords.py
new file mode 100644
index 00000000..b6368c3c
--- /dev/null
+++ b/src/website_profiling/content_analysis/keywords.py
@@ -0,0 +1,18 @@
+"""Per-page keyword extraction."""
+from __future__ import annotations
+
+import json
+from collections import Counter
+
+from .constants import STOP_WORDS
+
+
+def top_keywords_json(words: list[str], *, limit: int = 10) -> str:
+ keyword_words = [w.lower() for w in words if len(w) >= 4 and w.lower() not in STOP_WORDS]
+ top_keywords = Counter(keyword_words).most_common(limit)
+ max_kw = top_keywords[0][1] if top_keywords else 0
+ kw_rows = []
+ for w, c in top_keywords:
+ score = round(100 * c / max_kw) if max_kw else 0
+ kw_rows.append({"word": w, "count": c, "score": int(score)})
+ return json.dumps(kw_rows)
diff --git a/src/website_profiling/content_analysis/main_content.py b/src/website_profiling/content_analysis/main_content.py
new file mode 100644
index 00000000..0a00e956
--- /dev/null
+++ b/src/website_profiling/content_analysis/main_content.py
@@ -0,0 +1,25 @@
+"""Select the primary content root element from a page."""
+from __future__ import annotations
+
+from typing import Literal
+
+from bs4 import BeautifulSoup, Tag
+
+ContentStrategy = Literal["main_only", "full_body"]
+
+
+def find_main_content(soup: BeautifulSoup, strategy: ContentStrategy = "main_only") -> Tag | BeautifulSoup:
+ if strategy == "full_body":
+ return soup.find("body") or soup
+
+ candidates: list[Tag | None] = [
+ soup.find("main"),
+ soup.find("article"),
+ soup.find(attrs={"role": "main"}),
+ soup.find(id="content"),
+ soup.find(class_="content"),
+ ]
+ for el in candidates:
+ if el is not None and (el.get_text(separator=" ", strip=True) or "").strip():
+ return el
+ return soup.find("body") or soup
diff --git a/src/website_profiling/content_analysis/page.py b/src/website_profiling/content_analysis/page.py
new file mode 100644
index 00000000..6384dc4a
--- /dev/null
+++ b/src/website_profiling/content_analysis/page.py
@@ -0,0 +1,45 @@
+"""Per-page content analysis orchestration."""
+from __future__ import annotations
+
+from typing import Literal
+
+from .dom_cleanup import cleanup_dom
+from .excerpt import build_excerpt
+from .html_loader import load_soup
+from .html_ratio import content_html_ratio
+from .keywords import top_keywords_json
+from .main_content import find_main_content
+from .reading_level import flesch_kincaid_grade
+from .text_extract import extract_text
+from .tokenize import count_words, tokenize_words
+
+ContentStrategy = Literal["main_only", "full_body"]
+
+CONTENT_FIELDS = (
+ "word_count",
+ "reading_level",
+ "content_html_ratio",
+ "top_keywords",
+ "content_excerpt",
+)
+
+
+def analyze_page_html(
+ raw_html: str,
+ *,
+ excerpt_max_chars: int = 0,
+ strategy: ContentStrategy = "main_only",
+) -> dict:
+ """Analyze stored HTML and return crawl row content fields."""
+ soup = load_soup(raw_html)
+ cleaned = cleanup_dom(soup)
+ root = find_main_content(cleaned, strategy=strategy)
+ body_text = extract_text(root)
+ words = tokenize_words(body_text)
+ return {
+ "word_count": count_words(words),
+ "reading_level": flesch_kincaid_grade(words, body_text),
+ "content_html_ratio": content_html_ratio(body_text, raw_html),
+ "top_keywords": top_keywords_json(words),
+ "content_excerpt": build_excerpt(body_text, excerpt_max_chars),
+ }
diff --git a/src/website_profiling/content_analysis/pipeline.py b/src/website_profiling/content_analysis/pipeline.py
new file mode 100644
index 00000000..97f7a3d6
--- /dev/null
+++ b/src/website_profiling/content_analysis/pipeline.py
@@ -0,0 +1,50 @@
+"""Pipeline entrypoint for post-crawl content analysis."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from ..console_io import console_print
+from ..db import db_session, get_latest_crawl_run_id
+from ..db.crawl_store import merge_crawl_result_fields_batch
+from ..progress import emit_phase_done, emit_phase_start, emit_progress
+from .batch import analyze_run_html
+from .page import ContentStrategy
+
+
+def run_content_analysis(
+ crawl_run_id: Optional[int] = None,
+ *,
+ excerpt_max_chars: int = 0,
+ strategy: str = "main_only",
+ workers: int = 4,
+) -> dict[str, Any]:
+ """Analyze stored HTML for a crawl run and merge metrics into crawl_results."""
+ emit_phase_start("content_analysis", message="Analyzing stored page HTML")
+ console_print(" Content analysis: reading stored HTML...", flush=True)
+
+ strat: ContentStrategy = "full_body" if strategy == "full_body" else "main_only"
+ summary: dict[str, Any] = {"crawl_run_id": None, "pages_analyzed": 0, "strategy": strat}
+
+ with db_session() as conn:
+ run_id = crawl_run_id if crawl_run_id is not None else get_latest_crawl_run_id(conn)
+ if run_id is None:
+ console_print(" Content analysis skipped: no crawl run in database.", flush=True)
+ emit_phase_done("content_analysis")
+ return summary
+
+ summary["crawl_run_id"] = int(run_id)
+ emit_progress("content_analysis", "analyze", message="Analyzing page HTML")
+ updates = analyze_run_html(
+ conn,
+ int(run_id),
+ excerpt_max_chars=excerpt_max_chars,
+ strategy=strat,
+ workers=workers,
+ )
+ if updates:
+ merge_crawl_result_fields_batch(conn, int(run_id), updates)
+ summary["pages_analyzed"] = len(updates)
+
+ console_print(f" Content analysis complete ({summary['pages_analyzed']} pages).", flush=True)
+ emit_phase_done("content_analysis")
+ return summary
diff --git a/src/website_profiling/content_analysis/reading_level.py b/src/website_profiling/content_analysis/reading_level.py
new file mode 100644
index 00000000..92e6876a
--- /dev/null
+++ b/src/website_profiling/content_analysis/reading_level.py
@@ -0,0 +1,39 @@
+"""Reading level (Flesch-Kincaid grade) helpers."""
+from __future__ import annotations
+
+import re
+
+
+def count_syllables(word: str) -> int:
+ word = word.lower().strip()
+ if len(word) <= 3:
+ return 1
+ vowels = "aeiouy"
+ count = 0
+ prev_vowel = False
+ for ch in word:
+ is_vowel = ch in vowels
+ if is_vowel and not prev_vowel:
+ count += 1
+ prev_vowel = is_vowel
+ if word.endswith("e") and count > 1:
+ count -= 1
+ return max(1, count)
+
+
+def split_sentences(body_text: str) -> list[str]:
+ return [s.strip() for s in re.split(r"[.!?]+", body_text or "") if len(s.strip()) > 5]
+
+
+def flesch_kincaid_grade(words: list[str], body_text: str) -> float:
+ word_count = len(words)
+ if word_count <= 30:
+ return 0.0
+ sentence_count = max(1, len(split_sentences(body_text)))
+ total_syllables = sum(count_syllables(w) for w in words)
+ reading_level = (
+ 0.39 * (word_count / sentence_count)
+ + 11.8 * (total_syllables / max(1, word_count))
+ - 15.59
+ )
+ return max(0.0, min(18.0, round(reading_level, 1)))
diff --git a/src/website_profiling/content_analysis/text_extract.py b/src/website_profiling/content_analysis/text_extract.py
new file mode 100644
index 00000000..e65cccdf
--- /dev/null
+++ b/src/website_profiling/content_analysis/text_extract.py
@@ -0,0 +1,8 @@
+"""Plain-text extraction from a DOM subtree."""
+from __future__ import annotations
+
+from bs4 import BeautifulSoup, Tag
+
+
+def extract_text(root: Tag | BeautifulSoup) -> str:
+ return root.get_text(separator=" ", strip=True) if root is not None else ""
diff --git a/src/website_profiling/content_analysis/tokenize.py b/src/website_profiling/content_analysis/tokenize.py
new file mode 100644
index 00000000..b5bc86db
--- /dev/null
+++ b/src/website_profiling/content_analysis/tokenize.py
@@ -0,0 +1,12 @@
+"""Word tokenization for content metrics."""
+from __future__ import annotations
+
+import re
+
+
+def tokenize_words(body_text: str) -> list[str]:
+ return [w for w in re.findall(r"[a-zA-Z]+", body_text or "") if len(w) >= 2]
+
+
+def count_words(tokens: list[str]) -> int:
+ return len(tokens)
diff --git a/src/website_profiling/crawl/config.py b/src/website_profiling/crawl/config.py
new file mode 100644
index 00000000..2acbffe2
--- /dev/null
+++ b/src/website_profiling/crawl/config.py
@@ -0,0 +1,132 @@
+"""Crawler configuration dataclass."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .discovery import follow_links_for_mode, normalize_discovery_mode
+
+DEFAULT_USER_AGENT = "WebsiteProfilingCrawler/1.0"
+MOBILE_USER_AGENT = (
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) "
+ "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
+)
+
+
+def resolve_crawl_user_agent(
+ preset: str | None, custom: str | None, default: str | None = None
+) -> str:
+ p = (preset or "default").strip().lower()
+ if p == "mobile":
+ return MOBILE_USER_AGENT
+ if p == "custom" and custom and str(custom).strip():
+ return str(custom).strip()
+ return (default or DEFAULT_USER_AGENT).strip() or DEFAULT_USER_AGENT
+
+
+@dataclass
+class CrawlConfig:
+ start_url: str
+ max_pages: Optional[int] = None
+ concurrency: int = 6
+ timeout: int = 12
+ ignore_robots: bool = False
+ allow_external: bool = False
+ max_depth: Optional[int] = None
+ user_agent: Optional[str] = None
+ polite_delay: float = 0.0
+ store_outlinks: bool = False
+ exclude_urls: Optional[list[str]] = None
+ use_wappalyzer: bool = True
+ store_content_excerpt: bool = False
+ content_excerpt_max_chars: int = 4096
+ store_page_html: bool = False
+ max_stored_html_bytes: int = 2_097_152
+ run_content_analysis: bool = False
+ content_analysis_strategy: str = "main_only"
+ content_analysis_workers: int = 4
+ render_mode: str = "static"
+ js_concurrency: int = 3
+ js_timeout: int = 30
+ js_wait_until: str = "domcontentloaded"
+ js_extra_wait_ms: int = 1500
+ js_block_resources: bool = True
+ capture_console: bool = True
+ js_console_levels: str = "error,warning"
+ capture_failed_requests: bool = False
+ console_max_per_page: int = 20
+ custom_extraction_regex: str = ""
+ crawl_ignore_params: Optional[list[str]] = None
+ discovery_mode: str = "spider"
+ crawl_url_list: Optional[list[str]] = None
+ crawl_user_agent_preset: str = "default"
+ crawl_user_agent_custom: str = ""
+ crawl_auth_username: str = ""
+ crawl_auth_password: str = ""
+ crawl_extra_headers: str = ""
+ crawl_cookies: str = ""
+ crawl_robots_txt_override: str = ""
+ custom_extractors: Optional[list[dict]] = None
+ enable_axe: bool = False
+
+ @classmethod
+ def from_kwargs(cls, **kwargs: object) -> CrawlConfig:
+ """Build config from Crawler keyword arguments (unknown keys ignored)."""
+ fields = {f.name for f in cls.__dataclass_fields__.values()}
+ return cls(**{k: v for k, v in kwargs.items() if k in fields})
+
+ def normalized(self) -> CrawlConfig:
+ """Apply normalized derived fields in-place."""
+ self.start_url = self.start_url.rstrip("/")
+ self.discovery_mode = normalize_discovery_mode(self.discovery_mode)
+ self.render_mode = (self.render_mode or "static").strip().lower()
+ self.js_concurrency = max(1, int(self.js_concurrency))
+ self.max_pages = (
+ self.max_pages if (self.max_pages is not None and self.max_pages > 0) else float("inf")
+ )
+ self.max_depth = None if self.max_depth is None else int(self.max_depth)
+ self.polite_delay = max(0.0, float(self.polite_delay))
+ self.exclude_urls = list(self.exclude_urls) if self.exclude_urls else []
+ self.store_content_excerpt = bool(self.store_content_excerpt)
+ self.content_excerpt_max_chars = max(0, int(self.content_excerpt_max_chars or 0))
+ self.store_page_html = bool(self.store_page_html)
+ self.max_stored_html_bytes = max(1, int(self.max_stored_html_bytes or 2_097_152))
+ self.run_content_analysis = bool(self.run_content_analysis)
+ strat = (self.content_analysis_strategy or "main_only").strip().lower()
+ self.content_analysis_strategy = strat if strat in ("main_only", "full_body") else "main_only"
+ self.content_analysis_workers = max(1, int(self.content_analysis_workers or 4))
+ self.custom_extraction_regex = (self.custom_extraction_regex or "").strip()
+ self.custom_extractors = list(self.custom_extractors or [])
+ self.crawl_ignore_params = list(self.crawl_ignore_params or [])
+ self.crawl_url_list = [
+ u.rstrip("/") for u in (self.crawl_url_list or []) if u and str(u).strip()
+ ]
+ self.user_agent = resolve_crawl_user_agent(
+ self.crawl_user_agent_preset,
+ self.crawl_user_agent_custom,
+ self.user_agent,
+ )
+ return self
+
+ @property
+ def defer_content_analysis(self) -> bool:
+ return self.store_page_html and self.run_content_analysis
+
+ @property
+ def effective_concurrency(self) -> int:
+ if self.render_mode == "javascript":
+ return self.js_concurrency
+ return max(1, int(self.concurrency))
+
+ @property
+ def follow_links(self) -> bool:
+ return follow_links_for_mode(self.discovery_mode)
+
+ @property
+ def fetcher_render_mode(self) -> str:
+ if self.render_mode == "javascript":
+ return "javascript"
+ if self.render_mode == "auto":
+ return "auto"
+ return "static"
diff --git a/src/website_profiling/crawl/crawler.py b/src/website_profiling/crawl/crawler.py
index 39b0ac51..a00b4c9d 100644
--- a/src/website_profiling/crawl/crawler.py
+++ b/src/website_profiling/crawl/crawler.py
@@ -1,73 +1,47 @@
"""
Website crawler: threaded, respects robots.txt, returns DataFrame and optional CSV.
"""
+from __future__ import annotations
+
import json
-import threading
import time
from concurrent.futures import ThreadPoolExecutor
-from queue import Queue
from typing import Optional
-from urllib.parse import urlparse
-
-
-def _url_matches_exclude(url: str, exclude_urls: list[str]) -> bool:
- """True if url equals or is under any exclude prefix (trailing-slash normalized)."""
- if not exclude_urls:
- return False
- u = url.rstrip("/")
- for prefix in exclude_urls:
- p = prefix.strip().rstrip("/")
- if not p:
- continue
- if u == p or u.startswith(p + "/"):
- return True
- return False
import pandas as pd
import requests
from tqdm.auto import tqdm
from ..console_io import console_print
-from ..common import (
- detect_tech_wappalyzer,
- load_robots,
- normalize_link,
- parse_content_text,
- parse_link_edges,
- parse_resources,
- parse_seo,
- parse_seo_extended,
- parse_social_meta,
- parse_tech_stack,
+from ..common import strip_crawl_query_params
+from .config import (
+ DEFAULT_USER_AGENT,
+ MOBILE_USER_AGENT,
+ CrawlConfig,
+ resolve_crawl_user_agent,
)
-from ..analysis.page import analyze_html
-from .discovery import (
- follow_links_for_mode,
- normalize_discovery_mode,
- seed_sitemap_for_mode,
-)
-from .extraction import parse_extractors_config, run_extractors
+from .db_writer import CrawlDbWriter, _CrawlDbWriter
+from .discovery import normalize_discovery_mode
from .fetchers import build_fetcher
+from .sitemap import discover_sitemap_urls
from .fetchers.base import FetchResult
-from .fetchers.browser_diagnostics import merge_browser_into_page_analysis
from .fetchers.hybrid import HybridFetcher
-from .fetchers.spa_heuristics import needs_js_render_after_parse
-from .sitemap import discover_sitemap_urls
-
-DEFAULT_USER_AGENT = "WebsiteProfilingCrawler/1.0"
-MOBILE_USER_AGENT = (
- "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) "
- "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
-)
+from .frontier import CrawlFrontier, url_matches_exclude
+from .page_record import PageRecordBuilder
+from .schema import crawl_dataframe_columns, empty_crawl_row
+# Re-export for backward compatibility.
+_url_matches_exclude = url_matches_exclude
-def resolve_crawl_user_agent(preset: str | None, custom: str | None, default: str | None = None) -> str:
- p = (preset or "default").strip().lower()
- if p == "mobile":
- return MOBILE_USER_AGENT
- if p == "custom" and custom and str(custom).strip():
- return str(custom).strip()
- return (default or DEFAULT_USER_AGENT).strip() or DEFAULT_USER_AGENT
+__all__ = [
+ "Crawler",
+ "run_crawler",
+ "resolve_crawl_user_agent",
+ "DEFAULT_USER_AGENT",
+ "MOBILE_USER_AGENT",
+ "_url_matches_exclude",
+ "_CrawlDbWriter",
+]
class Crawler:
@@ -87,6 +61,11 @@ def __init__(
use_wappalyzer: bool = True,
store_content_excerpt: bool = False,
content_excerpt_max_chars: int = 4096,
+ store_page_html: bool = False,
+ max_stored_html_bytes: int = 2_097_152,
+ run_content_analysis: bool = False,
+ content_analysis_strategy: str = "main_only",
+ content_analysis_workers: int = 4,
render_mode: str = "static",
js_concurrency: int = 3,
js_timeout: int = 30,
@@ -110,331 +89,202 @@ def __init__(
crawl_robots_txt_override: str = "",
custom_extractors: Optional[list[dict]] = None,
enable_axe: bool = False,
+ *,
+ config: Optional[CrawlConfig] = None,
):
- self.start_url = start_url.rstrip("/")
- self.start_netloc = urlparse(self.start_url).netloc
- self.discovery_mode = normalize_discovery_mode(discovery_mode)
- self.follow_links = follow_links_for_mode(self.discovery_mode)
- self.crawl_url_list = [u.rstrip("/") for u in (crawl_url_list or []) if u and str(u).strip()]
+ if config is None:
+ config = CrawlConfig.from_kwargs(
+ start_url=start_url,
+ max_pages=max_pages,
+ concurrency=concurrency,
+ timeout=timeout,
+ ignore_robots=ignore_robots,
+ allow_external=allow_external,
+ max_depth=max_depth,
+ user_agent=user_agent,
+ polite_delay=polite_delay,
+ store_outlinks=store_outlinks,
+ exclude_urls=exclude_urls,
+ use_wappalyzer=use_wappalyzer,
+ store_content_excerpt=store_content_excerpt,
+ content_excerpt_max_chars=content_excerpt_max_chars,
+ store_page_html=store_page_html,
+ max_stored_html_bytes=max_stored_html_bytes,
+ run_content_analysis=run_content_analysis,
+ content_analysis_strategy=content_analysis_strategy,
+ content_analysis_workers=content_analysis_workers,
+ render_mode=render_mode,
+ js_concurrency=js_concurrency,
+ js_timeout=js_timeout,
+ js_wait_until=js_wait_until,
+ js_extra_wait_ms=js_extra_wait_ms,
+ js_block_resources=js_block_resources,
+ capture_console=capture_console,
+ js_console_levels=js_console_levels,
+ capture_failed_requests=capture_failed_requests,
+ console_max_per_page=console_max_per_page,
+ custom_extraction_regex=custom_extraction_regex,
+ crawl_ignore_params=crawl_ignore_params,
+ discovery_mode=discovery_mode,
+ crawl_url_list=crawl_url_list,
+ crawl_user_agent_preset=crawl_user_agent_preset,
+ crawl_user_agent_custom=crawl_user_agent_custom,
+ crawl_auth_username=crawl_auth_username,
+ crawl_auth_password=crawl_auth_password,
+ crawl_extra_headers=crawl_extra_headers,
+ crawl_cookies=crawl_cookies,
+ crawl_robots_txt_override=crawl_robots_txt_override,
+ custom_extractors=custom_extractors,
+ enable_axe=enable_axe,
+ )
+ config.normalized()
+ self.config = config
+
+ self.start_url = config.start_url
+ self.discovery_mode = config.discovery_mode
+ self.follow_links = config.follow_links
+ self.crawl_url_list = config.crawl_url_list
self.link_edges_accum: list[dict] = []
- self.render_mode = (render_mode or "static").strip().lower()
- self.js_concurrency = max(1, int(js_concurrency))
- effective_concurrency = (
- self.js_concurrency
- if self.render_mode == "javascript"
- else max(1, int(concurrency))
- )
- self.max_pages = (
- max_pages if (max_pages is not None and max_pages > 0) else float("inf")
+ self.render_mode = config.render_mode
+ self.max_pages = config.max_pages
+ self.concurrency = config.effective_concurrency
+ self.timeout = config.timeout
+ self.polite_delay = config.polite_delay
+ self.store_outlinks = config.store_outlinks
+ self.exclude_urls = config.exclude_urls
+ self.crawl_ignore_params = config.crawl_ignore_params
+ self.custom_extraction_regex = config.custom_extraction_regex
+ self.custom_extractors = config.custom_extractors
+ self.store_page_html = config.store_page_html
+ self.max_stored_html_bytes = config.max_stored_html_bytes
+ self._html_buffer: list[dict] = []
+ self._db_writer: Optional[CrawlDbWriter] = None
+
+ self.page_builder = PageRecordBuilder(
+ use_wappalyzer=config.use_wappalyzer,
+ store_content_excerpt=config.store_content_excerpt,
+ content_excerpt_max_chars=config.content_excerpt_max_chars,
+ defer_content_analysis=config.defer_content_analysis,
+ custom_extraction_regex=config.custom_extraction_regex,
+ custom_extractors=config.custom_extractors,
)
- self.concurrency = effective_concurrency
- self.timeout = timeout
- self.ignore_robots = ignore_robots
- self.allow_external = allow_external
- self.max_depth = None if max_depth is None else int(max_depth)
- self.user_agent = resolve_crawl_user_agent(
- crawl_user_agent_preset, crawl_user_agent_custom, user_agent
+
+ self.frontier = CrawlFrontier(
+ config.start_url,
+ allow_external=config.allow_external,
+ max_depth=config.max_depth,
+ exclude_urls=config.exclude_urls,
+ follow_links=config.follow_links,
+ ignore_robots=config.ignore_robots,
+ user_agent=config.user_agent or DEFAULT_USER_AGENT,
+ crawl_robots_txt_override=config.crawl_robots_txt_override,
)
- self.polite_delay = max(0.0, float(polite_delay))
- self.store_outlinks = store_outlinks
- self.exclude_urls = list(exclude_urls) if exclude_urls else []
- self.use_wappalyzer = use_wappalyzer
- self.store_content_excerpt = bool(store_content_excerpt)
- self.content_excerpt_max_chars = max(0, int(content_excerpt_max_chars or 0))
- self._wappalyzer_instance = None
- self.custom_extraction_regex = (custom_extraction_regex or "").strip()
- self.custom_extractors = list(custom_extractors or [])
- self.crawl_ignore_params = list(crawl_ignore_params or [])
+ self.queue = self.frontier.queue
+ self.depths = self.frontier.depths
+ self.visited = self.frontier.visited
+ self.lock = self.frontier.lock
- self.queue = Queue()
- self.depths: dict[str, int] = {}
- self.visited = set()
- self.results = []
- self.lock = threading.Lock()
+ self.results: list[dict] = []
self.session = requests.Session()
- self.session.headers.update({"User-Agent": self.user_agent})
- if crawl_auth_username:
- self.session.auth = (crawl_auth_username, crawl_auth_password or "")
- for line in (crawl_extra_headers or "").replace("\r", "").split("\n"):
+ self.session.headers.update({"User-Agent": config.user_agent})
+ if config.crawl_auth_username:
+ self.session.auth = (config.crawl_auth_username, config.crawl_auth_password or "")
+ for line in (config.crawl_extra_headers or "").replace("\r", "").split("\n"):
if ":" in line:
key, val = line.split(":", 1)
k, v = key.strip(), val.strip()
if k:
self.session.headers[k] = v
- if crawl_cookies and str(crawl_cookies).strip():
- self.session.headers["Cookie"] = str(crawl_cookies).strip()
- self.rp = None
- if not self.ignore_robots:
- override = (crawl_robots_txt_override or "").strip()
- if override:
- import io
- import urllib.robotparser as robotparser
+ if config.crawl_cookies and str(config.crawl_cookies).strip():
+ self.session.headers["Cookie"] = str(config.crawl_cookies).strip()
- self.rp = robotparser.RobotFileParser()
- self.rp.parse(override.splitlines())
- else:
- self.rp = load_robots(self.start_url)
self.fetcher = build_fetcher(
- render_mode="javascript" if self.render_mode == "javascript" else ("auto" if self.render_mode == "auto" else "static"),
- timeout=timeout,
- user_agent=self.user_agent,
+ render_mode=config.fetcher_render_mode,
+ timeout=config.timeout,
+ user_agent=config.user_agent,
session=self.session,
- js_concurrency=self.js_concurrency,
- js_timeout=js_timeout,
- js_wait_until=js_wait_until,
- js_extra_wait_ms=js_extra_wait_ms,
- js_block_resources=js_block_resources,
- capture_console=capture_console,
- js_console_levels=js_console_levels,
- capture_failed_requests=capture_failed_requests,
- console_max_per_page=console_max_per_page,
- run_axe=enable_axe,
+ js_concurrency=config.js_concurrency,
+ js_timeout=config.js_timeout,
+ js_wait_until=config.js_wait_until,
+ js_extra_wait_ms=config.js_extra_wait_ms,
+ js_block_resources=config.js_block_resources,
+ capture_console=config.capture_console,
+ js_console_levels=config.js_console_levels,
+ capture_failed_requests=config.capture_failed_requests,
+ console_max_per_page=config.console_max_per_page,
+ run_axe=config.enable_axe,
)
self._hybrid_fetcher = (
self.fetcher if isinstance(self.fetcher, HybridFetcher) else None
)
- self._seed_initial_urls(timeout)
+ self.frontier.seed_initial_urls(
+ discovery_mode=config.discovery_mode,
+ crawl_url_list=config.crawl_url_list,
+ timeout=config.timeout,
+ session=self.session,
+ )
- def _enqueue_seed(self, url: str, depth: int = 0) -> None:
- u = url.rstrip("/")
- if _url_matches_exclude(u, self.exclude_urls):
- return
- if not self.allow_external and not self.same_domain(u):
- return
- if u in self.depths:
- return
- self.queue.put(u)
- self.depths[u] = depth
+ @property
+ def rp(self):
+ return self.frontier.rp
- def _seed_initial_urls(self, timeout: int) -> None:
- mode = self.discovery_mode
- if mode in ("list", "hybrid"):
- for url in self.crawl_url_list:
- self._enqueue_seed(url, 0)
- if mode in ("spider", "hybrid"):
- self._enqueue_seed(self.start_url, 0)
- if seed_sitemap_for_mode(mode):
- self._seed_sitemap_urls(timeout)
+ @rp.setter
+ def rp(self, value) -> None:
+ self.frontier.rp = value
- def same_domain(self, url):
- return urlparse(url).netloc == self.start_netloc
+ def same_domain(self, url: str) -> bool:
+ return self.frontier.same_domain(url)
- def allowed_by_robots(self, url):
- if self.ignore_robots or not self.rp:
- return True
- try:
- return self.rp.can_fetch(self.user_agent, url)
- except Exception:
- return True
+ def allowed_by_robots(self, url: str) -> bool:
+ return self.frontier.allowed_by_robots(url)
- def _seed_sitemap_urls(self, timeout: int) -> None:
- try:
- seeds = discover_sitemap_urls(
- self.start_url,
- timeout=timeout,
- session=self.session,
- )
- except Exception:
- return
- for url in seeds:
- self._enqueue_seed(url, 0)
+ def _queue_contains(self, item: str) -> bool:
+ return self.frontier.queue_contains(item)
- def fetch(self, url) -> FetchResult:
+ def fetch(self, url: str) -> FetchResult:
return self.fetcher.fetch(url)
- def _empty_seo(self, url: str, headers_dict: Optional[dict] = None, redirect_chain_length: int = 0) -> dict:
- """Default SEO/performance fields when no HTML or error."""
- h = headers_dict or {}
- return {
- "response_time_ms": "",
- "content_length": 0,
- "final_url": url,
- "meta_description": "",
- "meta_description_len": 0,
- "h1": "",
- "h1_count": 0,
- "canonical_url": "",
- "viewport_present": False,
- "viewport_content": "",
- "noindex": False,
- "has_schema": False,
- "heading_sequence": "",
- "heading_text": "",
- "images_without_alt": 0,
- "images_total": 0,
- "img_without_lazy": 0,
- "img_without_dimensions": 0,
- "aria_count": 0,
- "mixed_content_count": 0,
- "redirect_chain_length": redirect_chain_length,
- "cache_control": h.get("Cache-Control", ""),
- "etag": h.get("ETag", ""),
- "x_robots_tag": h.get("X-Robots-Tag", ""),
- "strict_transport_security": h.get("Strict-Transport-Security", ""),
- "x_content_type_options": h.get("X-Content-Type-Options", ""),
- "x_frame_options": h.get("X-Frame-Options", ""),
- "content_security_policy": h.get("Content-Security-Policy", ""),
- "script_count": 0,
- "link_stylesheet_count": 0,
- "total_js_bytes": 0,
- "total_css_bytes": 0,
- "word_count": 0,
- "reading_level": 0.0,
- "content_html_ratio": 0.0,
- "top_keywords": "[]",
- "content_excerpt": "",
- "og_title": "",
- "og_description": "",
- "og_image": "",
- "og_type": "",
- "twitter_card": "",
- "twitter_title": "",
- "twitter_image": "",
- "tech_stack": "[]",
- "depth": None,
- "page_analysis": "{}",
- }
-
- def _parse_page_content(
+ def _capture_page_html(
self,
url: str,
- text: str,
- final_url: str,
- headers_dict: dict,
- redirect_chain_length: int,
- ) -> dict:
- """Extract title, links, and SEO/content fields from HTML."""
- ext = self._empty_seo(url, headers_dict, redirect_chain_length)
- title, link_edge_rows = parse_link_edges(url, text)
- links = {e["to_url"] for e in link_edge_rows}
- meta_description, meta_description_len, h1_text, h1_count, canonical_url = (
- parse_seo(url, text)
+ text: str | None,
+ status: object,
+ content_type: str | None,
+ fetch_method: str,
+ ) -> None:
+ from .html_capture import build_page_html_record
+
+ record = build_page_html_record(
+ url=url,
+ html=text or "",
+ status=status,
+ content_type=content_type,
+ fetch_method=fetch_method,
+ max_bytes=self.max_stored_html_bytes,
+ enabled=self.store_page_html,
)
- seo_ext = parse_seo_extended(text, final_url or url)
- ext["viewport_present"] = seo_ext.get("viewport_present", False)
- ext["viewport_content"] = seo_ext.get("viewport_content", "")
- ext["noindex"] = seo_ext.get("noindex", False)
- if (headers_dict.get("X-Robots-Tag") or "").lower().find("noindex") >= 0:
- ext["noindex"] = True
- ext["has_schema"] = seo_ext.get("has_schema", False)
- ext["heading_sequence"] = ",".join(seo_ext.get("heading_sequence") or [])
- ext["heading_text"] = " | ".join(seo_ext.get("heading_text") or [])
- ext["images_without_alt"] = seo_ext.get("images_without_alt", 0)
- ext["images_total"] = seo_ext.get("images_total", 0)
- ext["img_without_lazy"] = seo_ext.get("img_without_lazy", 0)
- ext["img_without_dimensions"] = seo_ext.get("img_without_dimensions", 0)
- ext["aria_count"] = seo_ext.get("aria_count", 0)
- ext["mixed_content_count"] = seo_ext.get("mixed_content_count", 0)
- res_res = parse_resources(text, final_url or url)
- ext["script_count"] = res_res.get("script_count", 0)
- ext["link_stylesheet_count"] = res_res.get("link_stylesheet_count", 0)
- from bs4 import BeautifulSoup as _BS
-
- _soup = _BS(text, "lxml")
- excerpt_max = self.content_excerpt_max_chars if self.store_content_excerpt else 0
- ct_data = parse_content_text(_soup, text, excerpt_max_chars=excerpt_max)
- ext["word_count"] = ct_data.get("word_count", 0)
- ext["reading_level"] = ct_data.get("reading_level", 0.0)
- ext["content_html_ratio"] = ct_data.get("content_html_ratio", 0.0)
- ext["top_keywords"] = ct_data.get("top_keywords", "[]")
- ext["content_excerpt"] = ct_data.get("content_excerpt") or ""
- social = parse_social_meta(_soup)
- ext["og_title"] = social.get("og_title", "")
- ext["og_description"] = social.get("og_description", "")
- ext["og_image"] = social.get("og_image", "")
- ext["og_type"] = social.get("og_type", "")
- ext["twitter_card"] = social.get("twitter_card", "")
- ext["twitter_title"] = social.get("twitter_title", "")
- ext["twitter_image"] = social.get("twitter_image", "")
- if self.use_wappalyzer:
- ext["tech_stack"] = detect_tech_wappalyzer(
- final_url or url, text, headers_dict, _soup, self._wappalyzer_instance
- )
+ if record is None:
+ return
+ if self._db_writer is not None:
+ self._db_writer.enqueue_html(record)
else:
- ext["tech_stack"] = parse_tech_stack(_soup, headers_dict, final_url or url)
- ext["page_analysis"] = json.dumps(
- analyze_html(text, final_url or url, final_url or url, canonical_url)
- )
- return {
- "title": title,
- "links": links,
- "link_edges": link_edge_rows,
- "meta_description": meta_description,
- "meta_description_len": meta_description_len,
- "h1_text": h1_text,
- "h1_count": h1_count,
- "canonical_url": canonical_url,
- "ext": ext,
- }
-
- def _maybe_refetch_after_parse(
- self,
- url: str,
- result: FetchResult,
- *,
- link_count: int,
- same_domain_link_count: int,
- ) -> FetchResult:
- """Post-parse auto-mode fallback when static HTML has too few links."""
- if self.render_mode != "auto" or self._hybrid_fetcher is None:
- return result
- if result.fetch_method != "static":
- return result
- if not needs_js_render_after_parse(
- result,
- link_count=link_count,
- same_domain_link_count=same_domain_link_count,
- ):
- return result
- rendered = self._hybrid_fetcher.refetch_rendered(url)
- if rendered.status == 200 and rendered.text:
- return rendered
- return result
+ self._html_buffer.append(record)
- @staticmethod
- def _sync_from_fetch_result(
- result: FetchResult,
- url: str,
- *,
- text: Optional[str],
- fetch_method: str,
- final_url: str,
- content_length: int,
- response_time_ms: Optional[int],
- headers_dict: dict,
- redirect_chain_length: int,
- status: Optional[int],
- ct: Optional[str],
- ) -> dict:
- """Copy all FetchResult fields after a post-parse browser refetch."""
- return {
- "text": result.text,
- "fetch_method": result.fetch_method,
- "final_url": result.final_url or url,
- "content_length": result.content_length or content_length,
- "response_time_ms": result.response_time_ms,
- "headers_dict": result.headers_dict or headers_dict,
- "redirect_chain_length": result.redirect_chain_length,
- "status": result.status,
- "ct": result.content_type,
- }
-
- def worker(self, url):
+ def worker(self, url: str) -> dict:
if not self.allowed_by_robots(url):
- out = {
- "url": url,
- "status": "blocked_by_robots",
- "content_type": "",
- "title": "",
- "outlinks": 0,
- "fetch_method": "static",
- **self._empty_seo(url),
- }
- if self.store_outlinks:
- out["outlink_targets"] = "[]"
- return out
+ return PageRecordBuilder.build_robots_blocked_row(
+ url, store_outlinks=self.store_outlinks
+ )
result = self.fetch(url)
+ if result.status is None:
+ return PageRecordBuilder.build_fetch_error_row(
+ url,
+ result,
+ fetch_method=result.fetch_method,
+ store_outlinks=self.store_outlinks,
+ )
+
status = result.status
ct = result.content_type
text = result.text
@@ -445,59 +295,33 @@ def worker(self, url):
redirect_chain_length = result.redirect_chain_length
fetch_method = result.fetch_method
- if status is None:
- out = {
- "url": url,
- "status": "error",
- "content_type": "",
- "title": "",
- "outlinks": 0,
- "fetch_method": fetch_method,
- **self._empty_seo(url, headers_dict, redirect_chain_length),
- }
- if self.store_outlinks:
- out["outlink_targets"] = "[]"
- if result.browser_diagnostics:
- out["page_analysis"] = merge_browser_into_page_analysis(
- None, result.browser_diagnostics
- )
- return out
-
title = ""
outlinks_count = 0
- outlink_list = []
+ outlink_list: list[str] = []
meta_description = ""
meta_description_len = 0
h1_text = ""
h1_count = 0
canonical_url = ""
- ext = self._empty_seo(url, headers_dict, redirect_chain_length)
+ ext = self.page_builder.empty_ext(url, headers_dict, redirect_chain_length)
if text:
- parsed = self._parse_page_content(
+ parsed = self.page_builder.parse_page_content(
url, text, final_url or url, headers_dict, redirect_chain_length
)
links = parsed["links"]
same_domain_link_count = sum(1 for link in links if self.same_domain(link))
- result = self._maybe_refetch_after_parse(
+ result = PageRecordBuilder.maybe_refetch_after_parse(
url,
result,
+ render_mode=self.render_mode,
+ hybrid_fetcher=self._hybrid_fetcher,
link_count=len(links),
same_domain_link_count=same_domain_link_count,
)
if result.text and result.text != text:
- synced = self._sync_from_fetch_result(
- result,
- url,
- text=text,
- fetch_method=fetch_method,
- final_url=final_url,
- content_length=content_length,
- response_time_ms=response_time_ms,
- headers_dict=headers_dict,
- redirect_chain_length=redirect_chain_length,
- status=status,
- ct=ct,
+ synced = PageRecordBuilder.sync_from_fetch_result(
+ result, url, content_length=content_length, headers_dict=headers_dict
)
text = synced["text"]
fetch_method = synced["fetch_method"]
@@ -508,7 +332,7 @@ def worker(self, url):
redirect_chain_length = synced["redirect_chain_length"]
status = synced["status"]
ct = synced["ct"]
- parsed = self._parse_page_content(
+ parsed = self.page_builder.parse_page_content(
url, text, final_url, headers_dict, redirect_chain_length
)
links = parsed["links"]
@@ -523,8 +347,6 @@ def worker(self, url):
ext = parsed["ext"]
if self.crawl_ignore_params:
- from ..common import strip_crawl_query_params
-
links = [strip_crawl_query_params(l, self.crawl_ignore_params) for l in links]
link_edge_rows = parsed.get("link_edges") or []
@@ -533,23 +355,7 @@ def worker(self, url):
if self.store_outlinks:
outlink_list.append(link)
self.link_edges_accum.append({"from_url": url, **edge})
- if not self.follow_links:
- continue
- if _url_matches_exclude(link, self.exclude_urls):
- continue
- if not self.allow_external and not self.same_domain(link):
- continue
- cur_depth = self.depths.get(url, 0)
- if self.max_depth is not None and cur_depth >= self.max_depth:
- continue
- with self.lock:
- if (
- link not in self.visited
- and link not in self.depths
- and not self._queue_contains(link)
- ):
- self.queue.put(link)
- self.depths[link] = cur_depth + 1
+ self.frontier.try_enqueue_link(link, url)
ext["response_time_ms"] = response_time_ms if response_time_ms is not None else ""
ext["content_length"] = content_length or 0
@@ -566,31 +372,17 @@ def worker(self, url):
ext["x_content_type_options"] = headers_dict.get("X-Content-Type-Options", "")
ext["x_frame_options"] = headers_dict.get("X-Frame-Options", "")
ext["content_security_policy"] = headers_dict.get("Content-Security-Policy", "")
-
ext["depth"] = self.depths.get(url)
- if self.custom_extraction_regex and text:
- import re
-
- try:
- match = re.search(self.custom_extraction_regex, text)
- if match:
- ext["custom_extract"] = match.group(1) if match.lastindex else match.group(0)
- except re.error:
- pass
-
- if self.custom_extractors and text:
- fields = run_extractors(text, self.custom_extractors)
- if fields:
- ext["custom_fields"] = json.dumps(fields)
+ self.page_builder.apply_custom_extractions(ext, text)
if self.polite_delay:
time.sleep(self.polite_delay)
- if result.browser_diagnostics:
- ext["page_analysis"] = merge_browser_into_page_analysis(
- ext.get("page_analysis"), result.browser_diagnostics
- )
+ PageRecordBuilder.merge_browser_diagnostics(ext, result)
+
+ if text:
+ self._capture_page_html(url, text, status, ct, fetch_method)
res = {
"url": url,
@@ -605,18 +397,12 @@ def worker(self, url):
res["outlink_targets"] = json.dumps(list(outlink_list))
return res
- def _queue_contains(self, item):
- try:
- return item in list(self.queue.queue)
- except Exception:
- return False
-
def crawl(
self,
show_progress: bool = True,
stream_crawl_run_id: Optional[int] = None,
stream_batch_size: int = 500,
- ):
+ ) -> pd.DataFrame:
start_time = time.time()
from ..progress import CrawlProgressTracker, emit_phase_start
@@ -628,10 +414,17 @@ def crawl(
)
emit_phase_start("crawl", message="Crawling pages")
futures = []
- db_writer: Optional[_CrawlDbWriter] = None
+ db_writer: Optional[CrawlDbWriter] = None
pages_crawled = 0
+ self._db_writer = None
+ self._html_buffer = []
if stream_crawl_run_id is not None:
- db_writer = _CrawlDbWriter(stream_crawl_run_id, stream_batch_size)
+ db_writer = _CrawlDbWriter(
+ stream_crawl_run_id,
+ stream_batch_size,
+ store_page_html=self.store_page_html,
+ )
+ self._db_writer = db_writer
db_writer.start()
use_tqdm = show_progress and stream_crawl_run_id is None
pbar = tqdm(
@@ -650,12 +443,10 @@ def crawl(
and len(self.results) + len(futures) < self.max_pages
):
url = self.queue.get()
- if _url_matches_exclude(url, self.exclude_urls):
+ if self.frontier.should_skip_dequeued(url):
+ continue
+ if not self.frontier.mark_visited(url):
continue
- with self.lock:
- if url in self.visited:
- continue
- self.visited.add(url)
futures.append(ex.submit(self.worker, url))
remaining = []
@@ -664,60 +455,7 @@ def crawl(
try:
res = f.result()
except Exception:
- res = {
- "url": None,
- "status": "error",
- "content_type": "",
- "title": "",
- "outlinks": 0,
- "response_time_ms": "",
- "content_length": 0,
- "final_url": "",
- "meta_description": "",
- "meta_description_len": 0,
- "h1": "",
- "h1_count": 0,
- "canonical_url": "",
- "viewport_present": False,
- "viewport_content": "",
- "noindex": False,
- "has_schema": False,
- "heading_sequence": "",
- "heading_text": "",
- "images_without_alt": 0,
- "images_total": 0,
- "img_without_lazy": 0,
- "img_without_dimensions": 0,
- "aria_count": 0,
- "mixed_content_count": 0,
- "redirect_chain_length": 0,
- "cache_control": "",
- "etag": "",
- "x_robots_tag": "",
- "strict_transport_security": "",
- "x_content_type_options": "",
- "x_frame_options": "",
- "content_security_policy": "",
- "script_count": 0,
- "link_stylesheet_count": 0,
- "total_js_bytes": 0,
- "total_css_bytes": 0,
- "word_count": 0,
- "reading_level": 0.0,
- "content_html_ratio": 0.0,
- "top_keywords": "[]",
- "content_excerpt": "",
- "og_title": "",
- "og_description": "",
- "og_image": "",
- "og_type": "",
- "twitter_card": "",
- "twitter_title": "",
- "twitter_image": "",
- "tech_stack": "[]",
- "depth": None,
- "page_analysis": "{}",
- }
+ res = empty_crawl_row(status="error")
if self.store_outlinks:
res["outlink_targets"] = "[]"
self.results.append(res)
@@ -729,10 +467,7 @@ def crawl(
db_writer.enqueue(res)
if use_tqdm:
pbar.update(1)
- progress_tracker.maybe_emit(
- pages_crawled,
- page_url,
- )
+ progress_tracker.maybe_emit(pages_crawled, page_url)
else:
remaining.append(f)
futures = remaining
@@ -755,117 +490,15 @@ def crawl(
db_writer.finish()
db_writer.join()
db_writer.raise_if_failed()
+ self._db_writer = None
elapsed = time.time() - start_time
df = pd.DataFrame(self.results)
if df.empty:
- cols = [
- "url",
- "status",
- "content_type",
- "title",
- "outlinks",
- "response_time_ms",
- "content_length",
- "final_url",
- "meta_description",
- "meta_description_len",
- "h1",
- "h1_count",
- "canonical_url",
- "viewport_present",
- "viewport_content",
- "noindex",
- "has_schema",
- "heading_sequence",
- "heading_text",
- "images_without_alt",
- "images_total",
- "img_without_lazy",
- "img_without_dimensions",
- "aria_count",
- "mixed_content_count",
- "redirect_chain_length",
- "cache_control",
- "etag",
- "x_robots_tag",
- "strict_transport_security",
- "x_content_type_options",
- "x_frame_options",
- "content_security_policy",
- "script_count",
- "link_stylesheet_count",
- "total_js_bytes",
- "total_css_bytes",
- "word_count",
- "reading_level",
- "content_html_ratio",
- "top_keywords",
- "content_excerpt",
- "og_title",
- "og_description",
- "og_image",
- "og_type",
- "twitter_card",
- "twitter_title",
- "twitter_image",
- "tech_stack",
- "depth",
- "page_analysis",
- "fetch_method",
- ]
- if self.store_outlinks:
- cols.append("outlink_targets")
- df = pd.DataFrame(columns=cols)
+ df = pd.DataFrame(columns=crawl_dataframe_columns(store_outlinks=self.store_outlinks))
df["crawl_time_s"] = elapsed
return df
-class _CrawlDbWriter(threading.Thread):
- """Background thread: batch-insert crawl rows via PostgreSQL connection pool."""
-
- def __init__(self, crawl_run_id: int, batch_size: int = 500) -> None:
- super().__init__(daemon=True)
- self.crawl_run_id = crawl_run_id
- self.batch_size = max(50, batch_size)
- self._queue: Queue = Queue()
- self._error: Optional[BaseException] = None
-
- def enqueue(self, record: dict) -> None:
- self._queue.put(record)
-
- def finish(self) -> None:
- self._queue.put(None)
-
- def run(self) -> None:
- from ..db import db_session
- from ..db.crawl_store import _crawl_rows_from_df, write_crawl_batch
-
- buffer: list[dict] = []
- try:
- while True:
- item = self._queue.get()
- if item is None:
- if buffer:
- chunk = pd.DataFrame(buffer)
- with db_session() as conn:
- rows = _crawl_rows_from_df(chunk, self.crawl_run_id)
- write_crawl_batch(conn, rows, self.crawl_run_id, commit=True)
- break
- buffer.append(item)
- if len(buffer) >= self.batch_size:
- chunk = pd.DataFrame(buffer)
- buffer = []
- with db_session() as conn:
- rows = _crawl_rows_from_df(chunk, self.crawl_run_id)
- write_crawl_batch(conn, rows, self.crawl_run_id, commit=True)
- except BaseException as e:
- self._error = e
-
- def raise_if_failed(self) -> None:
- if self._error is not None:
- raise self._error
-
-
def run_crawler(
start_url: str,
max_pages: Optional[int] = None,
@@ -883,6 +516,11 @@ def run_crawler(
preserve_crawl_history: bool = True,
store_content_excerpt: bool = False,
content_excerpt_max_chars: int = 4096,
+ store_page_html: bool = False,
+ max_stored_html_bytes: int = 2_097_152,
+ run_content_analysis: bool = False,
+ content_analysis_strategy: str = "main_only",
+ content_analysis_workers: int = 4,
crawl_stream_to_db: bool = False,
property_id: Optional[int] = None,
render_mode: str = "static",
@@ -932,6 +570,11 @@ def run_crawler(
exclude_urls=exclude_urls,
store_content_excerpt=store_content_excerpt,
content_excerpt_max_chars=content_excerpt_max_chars,
+ store_page_html=store_page_html,
+ max_stored_html_bytes=max_stored_html_bytes,
+ run_content_analysis=run_content_analysis,
+ content_analysis_strategy=content_analysis_strategy,
+ content_analysis_workers=content_analysis_workers,
render_mode=render_mode,
js_concurrency=js_concurrency,
js_timeout=js_timeout,
@@ -998,7 +641,6 @@ def run_crawler(
with db_session() as conn:
write_link_edges(conn, crawler.link_edges_accum, crawl_run_id=run_id)
if output_db and not df.empty and stream_run_id is None:
- import sys
console_print(" Writing crawl results to DB...", flush=True)
from ..db import backup_db_if_exists, create_crawl_run, db_session, read_historical_data, restore_historical_data, write_crawl
from ..db.storage import ensure_crawl_tables_cleared
@@ -1022,6 +664,11 @@ def run_crawler(
discovery_mode=disc_label,
)
write_crawl(conn, df, crawl_run_id=run_id)
+ html_buffer = getattr(crawler, "_html_buffer", None) or []
+ if getattr(crawler, "store_page_html", False) and html_buffer:
+ from ..db.html_store import write_page_html_batch
+
+ write_page_html_batch(conn, html_buffer, run_id, commit=True)
if crawler.link_edges_accum:
from ..db.crawl_store import write_link_edges
diff --git a/src/website_profiling/crawl/db_writer.py b/src/website_profiling/crawl/db_writer.py
new file mode 100644
index 00000000..302ba7fc
--- /dev/null
+++ b/src/website_profiling/crawl/db_writer.py
@@ -0,0 +1,84 @@
+"""Background thread: batch-insert crawl rows and optional HTML via PostgreSQL connection pool."""
+
+from __future__ import annotations
+
+import threading
+from queue import Queue
+from typing import Optional
+
+import pandas as pd
+
+
+class CrawlDbWriter(threading.Thread):
+ """Background thread: batch-insert crawl rows and optional page HTML."""
+
+ def __init__(self, crawl_run_id: int, batch_size: int = 500, *, store_page_html: bool = False) -> None:
+ super().__init__(daemon=True)
+ self.crawl_run_id = crawl_run_id
+ self.batch_size = max(50, batch_size)
+ self.store_page_html = bool(store_page_html)
+ self._queue: Queue = Queue()
+ self._error: Optional[BaseException] = None
+
+ def enqueue(self, record: dict) -> None:
+ self._queue.put(("crawl", record))
+
+ def enqueue_html(self, record: dict) -> None:
+ if not self.store_page_html:
+ return
+ self._queue.put(("html", record))
+
+ def finish(self) -> None:
+ self._queue.put(None)
+
+ def _flush_crawl(self, buffer: list[dict]) -> None:
+ if not buffer:
+ return
+ from ..db import db_session
+ from ..db.crawl_store import _crawl_rows_from_df, write_crawl_batch
+
+ chunk = pd.DataFrame(buffer)
+ with db_session() as conn:
+ rows = _crawl_rows_from_df(chunk, self.crawl_run_id)
+ write_crawl_batch(conn, rows, self.crawl_run_id, commit=True)
+
+ def _flush_html(self, buffer: list[dict]) -> None:
+ if not buffer:
+ return
+ from ..db import db_session
+ from ..db.html_store import write_page_html_batch
+
+ with db_session() as conn:
+ write_page_html_batch(conn, buffer, self.crawl_run_id, commit=True)
+
+ def run(self) -> None:
+ crawl_buffer: list[dict] = []
+ html_buffer: list[dict] = []
+ try:
+ while True:
+ item = self._queue.get()
+ if item is None:
+ self._flush_crawl(crawl_buffer)
+ self._flush_html(html_buffer)
+ break
+ kind, payload = item
+ if kind == "html":
+ html_buffer.append(payload)
+ if len(html_buffer) >= self.batch_size:
+ self._flush_html(html_buffer)
+ html_buffer = []
+ else:
+ crawl_buffer.append(payload)
+ if len(crawl_buffer) >= self.batch_size:
+ self._flush_crawl(crawl_buffer)
+ crawl_buffer = []
+ except BaseException as e:
+ self._error = e
+
+ def raise_if_failed(self) -> None:
+ if self._error is not None:
+ raise self._error
+
+
+# Backward-compatible alias for tests and internal imports.
+_CrawlDbWriter = CrawlDbWriter
diff --git a/src/website_profiling/crawl/frontier.py b/src/website_profiling/crawl/frontier.py
new file mode 100644
index 00000000..cb1e65ac
--- /dev/null
+++ b/src/website_profiling/crawl/frontier.py
@@ -0,0 +1,156 @@
+"""Crawl frontier: URL queue, depth tracking, and link discovery."""
+
+from __future__ import annotations
+
+import threading
+from queue import Queue
+from typing import Optional
+from urllib.parse import urlparse
+
+import requests
+
+from ..common import load_robots
+from . import sitemap
+from .discovery import seed_sitemap_for_mode
+
+
+def url_matches_exclude(url: str, exclude_urls: list[str]) -> bool:
+ """True if url equals or is under any exclude prefix (trailing-slash normalized)."""
+ if not exclude_urls:
+ return False
+ u = url.rstrip("/")
+ for prefix in exclude_urls:
+ p = prefix.strip().rstrip("/")
+ if not p:
+ continue
+ if u == p or u.startswith(p + "/"):
+ return True
+ return False
+
+
+class CrawlFrontier:
+ """Manages crawl queue, visited set, depth limits, and robots rules."""
+
+ def __init__(
+ self,
+ start_url: str,
+ *,
+ allow_external: bool = False,
+ max_depth: Optional[int] = None,
+ exclude_urls: Optional[list[str]] = None,
+ follow_links: bool = True,
+ ignore_robots: bool = False,
+ user_agent: str = "",
+ crawl_robots_txt_override: str = "",
+ ) -> None:
+ self.start_url = start_url.rstrip("/")
+ self.start_netloc = urlparse(self.start_url).netloc
+ self.allow_external = allow_external
+ self.max_depth = max_depth
+ self.exclude_urls = list(exclude_urls) if exclude_urls else []
+ self.follow_links = follow_links
+ self.user_agent = user_agent
+ self.queue: Queue = Queue()
+ self.depths: dict[str, int] = {}
+ self.visited: set[str] = set()
+ self.lock = threading.Lock()
+ self.rp = None
+ if not ignore_robots:
+ override = (crawl_robots_txt_override or "").strip()
+ if override:
+ import urllib.robotparser as robotparser
+
+ self.rp = robotparser.RobotFileParser()
+ self.rp.parse(override.splitlines())
+ else:
+ self.rp = load_robots(self.start_url)
+
+ def same_domain(self, url: str) -> bool:
+ return urlparse(url).netloc == self.start_netloc
+
+ def allowed_by_robots(self, url: str) -> bool:
+ if not self.rp:
+ return True
+ try:
+ return self.rp.can_fetch(self.user_agent, url)
+ except Exception:
+ return True
+
+ def queue_contains(self, item: str) -> bool:
+ try:
+ return item in list(self.queue.queue)
+ except Exception:
+ return False
+
+ def enqueue_seed(self, url: str, depth: int = 0) -> None:
+ u = url.rstrip("/")
+ if url_matches_exclude(u, self.exclude_urls):
+ return
+ if not self.allow_external and not self.same_domain(u):
+ return
+ if u in self.depths:
+ return
+ self.queue.put(u)
+ self.depths[u] = depth
+
+ def seed_initial_urls(
+ self,
+ *,
+ discovery_mode: str,
+ crawl_url_list: list[str],
+ timeout: int,
+ session: requests.Session,
+ ) -> None:
+ mode = discovery_mode
+ if mode in ("list", "hybrid"):
+ for url in crawl_url_list:
+ self.enqueue_seed(url, 0)
+ if mode in ("spider", "hybrid"):
+ self.enqueue_seed(self.start_url, 0)
+ if seed_sitemap_for_mode(mode):
+ self.seed_sitemap_urls(timeout, session)
+
+ def seed_sitemap_urls(self, timeout: int, session: requests.Session) -> None:
+ try:
+ seeds = sitemap.discover_sitemap_urls(
+ self.start_url,
+ timeout=timeout,
+ session=session,
+ )
+ except Exception:
+ return
+ for url in seeds:
+ self.enqueue_seed(url, 0)
+
+ def try_enqueue_link(self, link: str, from_url: str) -> bool:
+ """Enqueue a discovered link if frontier rules allow. Returns True if enqueued."""
+ if not self.follow_links:
+ return False
+ if url_matches_exclude(link, self.exclude_urls):
+ return False
+ if not self.allow_external and not self.same_domain(link):
+ return False
+ cur_depth = self.depths.get(from_url, 0)
+ if self.max_depth is not None and cur_depth >= self.max_depth:
+ return False
+ with self.lock:
+ if (
+ link not in self.visited
+ and link not in self.depths
+ and not self.queue_contains(link)
+ ):
+ self.queue.put(link)
+ self.depths[link] = cur_depth + 1
+ return True
+ return False
+
+ def mark_visited(self, url: str) -> bool:
+ """Mark URL visited; return False if already visited."""
+ with self.lock:
+ if url in self.visited:
+ return False
+ self.visited.add(url)
+ return True
+
+ def should_skip_dequeued(self, url: str) -> bool:
+ return url_matches_exclude(url, self.exclude_urls)
diff --git a/src/website_profiling/crawl/html_capture.py b/src/website_profiling/crawl/html_capture.py
new file mode 100644
index 00000000..021947ee
--- /dev/null
+++ b/src/website_profiling/crawl/html_capture.py
@@ -0,0 +1,60 @@
+"""Helpers for deciding whether and how to persist fetched HTML."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+
+def _is_html_content_type(content_type: str | None) -> bool:
+ ct = (content_type or "").lower()
+ return "text/html" in ct or "application/xhtml+xml" in ct
+
+
+def should_store_page_html(
+ *,
+ enabled: bool,
+ status: Any,
+ content_type: str | None,
+ html: str | None,
+ max_bytes: int,
+) -> bool:
+ """True when HTML should be persisted for a crawl URL."""
+ if not enabled or not html or not str(html).strip():
+ return False
+ try:
+ if int(status) != 200:
+ return False
+ except (TypeError, ValueError):
+ return False
+ if not _is_html_content_type(content_type):
+ return False
+ return len(str(html).encode("utf-8")) <= max(1, int(max_bytes))
+
+
+def build_page_html_record(
+ *,
+ url: str,
+ html: str,
+ status: Any,
+ content_type: str | None,
+ fetch_method: str,
+ max_bytes: int,
+ enabled: bool = True,
+) -> Optional[dict[str, Any]]:
+ """Build a side-channel HTML record, or None if storage should be skipped."""
+ if not should_store_page_html(
+ enabled=enabled,
+ status=status,
+ content_type=content_type,
+ html=html,
+ max_bytes=max_bytes,
+ ):
+ return None
+ text = str(html)
+ return {
+ "url": str(url or "").rstrip("/"),
+ "html": text,
+ "status": str(status),
+ "content_type": str(content_type or ""),
+ "fetch_method": str(fetch_method or "static").strip() or "static",
+ "byte_length": len(text.encode("utf-8")),
+ }
diff --git a/src/website_profiling/crawl/page_record.py b/src/website_profiling/crawl/page_record.py
new file mode 100644
index 00000000..c76322a1
--- /dev/null
+++ b/src/website_profiling/crawl/page_record.py
@@ -0,0 +1,229 @@
+"""Build crawl page records from fetched HTML."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Optional
+
+from ..analysis.page import analyze_html
+from ..common import (
+ detect_tech_wappalyzer,
+ parse_content_text,
+ parse_link_edges,
+ parse_resources,
+ parse_seo,
+ parse_seo_extended,
+ parse_social_meta,
+ parse_tech_stack,
+)
+from .extraction import run_extractors
+from .fetchers.base import FetchResult
+from .fetchers.browser_diagnostics import merge_browser_into_page_analysis
+from .fetchers.hybrid import HybridFetcher
+from .fetchers.spa_heuristics import needs_js_render_after_parse
+from .schema import empty_crawl_row, empty_crawl_row_ext
+
+
+class PageRecordBuilder:
+ """Extract SEO/content fields and assemble crawl result rows."""
+
+ def __init__(
+ self,
+ *,
+ use_wappalyzer: bool = True,
+ store_content_excerpt: bool = False,
+ content_excerpt_max_chars: int = 4096,
+ defer_content_analysis: bool = False,
+ custom_extraction_regex: str = "",
+ custom_extractors: Optional[list[dict]] = None,
+ ) -> None:
+ self.use_wappalyzer = use_wappalyzer
+ self.store_content_excerpt = store_content_excerpt
+ self.content_excerpt_max_chars = content_excerpt_max_chars
+ self.defer_content_analysis = defer_content_analysis
+ self.custom_extraction_regex = custom_extraction_regex
+ self.custom_extractors = list(custom_extractors or [])
+ self._wappalyzer_instance = None
+
+ def empty_ext(
+ self,
+ url: str,
+ headers_dict: Optional[dict] = None,
+ redirect_chain_length: int = 0,
+ ) -> dict[str, Any]:
+ return empty_crawl_row_ext(url, headers_dict, redirect_chain_length)
+
+ def parse_page_content(
+ self,
+ url: str,
+ text: str,
+ final_url: str,
+ headers_dict: dict,
+ redirect_chain_length: int,
+ ) -> dict[str, Any]:
+ """Extract title, links, and SEO/content fields from HTML."""
+ ext = self.empty_ext(url, headers_dict, redirect_chain_length)
+ title, link_edge_rows = parse_link_edges(url, text)
+ links = {e["to_url"] for e in link_edge_rows}
+ meta_description, meta_description_len, h1_text, h1_count, canonical_url = parse_seo(
+ url, text
+ )
+ seo_ext = parse_seo_extended(text, final_url or url)
+ ext["viewport_present"] = seo_ext.get("viewport_present", False)
+ ext["viewport_content"] = seo_ext.get("viewport_content", "")
+ ext["noindex"] = seo_ext.get("noindex", False)
+ if (headers_dict.get("X-Robots-Tag") or "").lower().find("noindex") >= 0:
+ ext["noindex"] = True
+ ext["has_schema"] = seo_ext.get("has_schema", False)
+ ext["heading_sequence"] = ",".join(seo_ext.get("heading_sequence") or [])
+ ext["heading_text"] = " | ".join(seo_ext.get("heading_text") or [])
+ ext["images_without_alt"] = seo_ext.get("images_without_alt", 0)
+ ext["images_total"] = seo_ext.get("images_total", 0)
+ ext["img_without_lazy"] = seo_ext.get("img_without_lazy", 0)
+ ext["img_without_dimensions"] = seo_ext.get("img_without_dimensions", 0)
+ ext["aria_count"] = seo_ext.get("aria_count", 0)
+ ext["mixed_content_count"] = seo_ext.get("mixed_content_count", 0)
+ res_res = parse_resources(text, final_url or url)
+ ext["script_count"] = res_res.get("script_count", 0)
+ ext["link_stylesheet_count"] = res_res.get("link_stylesheet_count", 0)
+ from bs4 import BeautifulSoup as _BS
+
+ _soup = _BS(text, "lxml")
+ excerpt_max = self.content_excerpt_max_chars if self.store_content_excerpt else 0
+ if not self.defer_content_analysis:
+ ct_data = parse_content_text(_soup, text, excerpt_max_chars=excerpt_max)
+ ext["word_count"] = ct_data.get("word_count", 0)
+ ext["reading_level"] = ct_data.get("reading_level", 0.0)
+ ext["content_html_ratio"] = ct_data.get("content_html_ratio", 0.0)
+ ext["top_keywords"] = ct_data.get("top_keywords", "[]")
+ ext["content_excerpt"] = ct_data.get("content_excerpt") or ""
+ social = parse_social_meta(_soup)
+ ext["og_title"] = social.get("og_title", "")
+ ext["og_description"] = social.get("og_description", "")
+ ext["og_image"] = social.get("og_image", "")
+ ext["og_type"] = social.get("og_type", "")
+ ext["twitter_card"] = social.get("twitter_card", "")
+ ext["twitter_title"] = social.get("twitter_title", "")
+ ext["twitter_image"] = social.get("twitter_image", "")
+ if self.use_wappalyzer:
+ ext["tech_stack"] = detect_tech_wappalyzer(
+ final_url or url, text, headers_dict, _soup, self._wappalyzer_instance
+ )
+ else:
+ ext["tech_stack"] = parse_tech_stack(_soup, headers_dict, final_url or url)
+ ext["page_analysis"] = json.dumps(
+ analyze_html(text, final_url or url, final_url or url, canonical_url)
+ )
+ return {
+ "title": title,
+ "links": links,
+ "link_edges": link_edge_rows,
+ "meta_description": meta_description,
+ "meta_description_len": meta_description_len,
+ "h1_text": h1_text,
+ "h1_count": h1_count,
+ "canonical_url": canonical_url,
+ "ext": ext,
+ }
+
+ def apply_custom_extractions(self, ext: dict[str, Any], text: Optional[str]) -> None:
+ if self.custom_extraction_regex and text:
+ try:
+ match = re.search(self.custom_extraction_regex, text)
+ if match:
+ ext["custom_extract"] = match.group(1) if match.lastindex else match.group(0)
+ except re.error:
+ pass
+ if self.custom_extractors and text:
+ fields = run_extractors(text, self.custom_extractors)
+ if fields:
+ ext["custom_fields"] = json.dumps(fields)
+
+ @staticmethod
+ def maybe_refetch_after_parse(
+ url: str,
+ result: FetchResult,
+ *,
+ render_mode: str,
+ hybrid_fetcher: Optional[HybridFetcher],
+ link_count: int,
+ same_domain_link_count: int,
+ ) -> FetchResult:
+ """Post-parse auto-mode fallback when static HTML has too few links."""
+ if render_mode != "auto" or hybrid_fetcher is None:
+ return result
+ if result.fetch_method != "static":
+ return result
+ if not needs_js_render_after_parse(
+ result,
+ link_count=link_count,
+ same_domain_link_count=same_domain_link_count,
+ ):
+ return result
+ rendered = hybrid_fetcher.refetch_rendered(url)
+ if rendered.status == 200 and rendered.text:
+ return rendered
+ return result
+
+ @staticmethod
+ def sync_from_fetch_result(
+ result: FetchResult,
+ url: str,
+ *,
+ content_length: int,
+ headers_dict: dict,
+ ) -> dict[str, Any]:
+ """Copy FetchResult fields after a post-parse browser refetch."""
+ return {
+ "text": result.text,
+ "fetch_method": result.fetch_method,
+ "final_url": result.final_url or url,
+ "content_length": result.content_length or content_length,
+ "response_time_ms": result.response_time_ms,
+ "headers_dict": result.headers_dict or headers_dict,
+ "redirect_chain_length": result.redirect_chain_length,
+ "status": result.status,
+ "ct": result.content_type,
+ }
+
+ @staticmethod
+ def build_robots_blocked_row(url: str, *, store_outlinks: bool) -> dict[str, Any]:
+ row = empty_crawl_row(
+ url=url,
+ status="blocked_by_robots",
+ fetch_method="static",
+ )
+ if store_outlinks:
+ row["outlink_targets"] = "[]"
+ return row
+
+ @staticmethod
+ def build_fetch_error_row(
+ url: str,
+ result: FetchResult,
+ *,
+ fetch_method: str,
+ store_outlinks: bool,
+ ) -> dict[str, Any]:
+ row = empty_crawl_row(
+ url=url,
+ status="error",
+ fetch_method=fetch_method,
+ headers_dict=result.headers_dict or {},
+ redirect_chain_length=result.redirect_chain_length,
+ )
+ if store_outlinks:
+ row["outlink_targets"] = "[]"
+ if result.browser_diagnostics:
+ row["page_analysis"] = merge_browser_into_page_analysis(
+ None, result.browser_diagnostics
+ )
+ return row
+
+ @staticmethod
+ def merge_browser_diagnostics(ext: dict[str, Any], result: FetchResult) -> None:
+ if result.browser_diagnostics:
+ ext["page_analysis"] = merge_browser_into_page_analysis(
+ ext.get("page_analysis"), result.browser_diagnostics
+ )
diff --git a/src/website_profiling/crawl/schema.py b/src/website_profiling/crawl/schema.py
new file mode 100644
index 00000000..3d3cf077
--- /dev/null
+++ b/src/website_profiling/crawl/schema.py
@@ -0,0 +1,154 @@
+"""Crawl row schema: single source for DataFrame columns and default field values."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+# Core crawl columns (excluding optional outlink_targets).
+CRAWL_ROW_COLUMNS: list[str] = [
+ "url",
+ "status",
+ "content_type",
+ "title",
+ "outlinks",
+ "response_time_ms",
+ "content_length",
+ "final_url",
+ "meta_description",
+ "meta_description_len",
+ "h1",
+ "h1_count",
+ "canonical_url",
+ "viewport_present",
+ "viewport_content",
+ "noindex",
+ "has_schema",
+ "heading_sequence",
+ "heading_text",
+ "images_without_alt",
+ "images_total",
+ "img_without_lazy",
+ "img_without_dimensions",
+ "aria_count",
+ "mixed_content_count",
+ "redirect_chain_length",
+ "cache_control",
+ "etag",
+ "x_robots_tag",
+ "strict_transport_security",
+ "x_content_type_options",
+ "x_frame_options",
+ "content_security_policy",
+ "script_count",
+ "link_stylesheet_count",
+ "total_js_bytes",
+ "total_css_bytes",
+ "word_count",
+ "reading_level",
+ "content_html_ratio",
+ "top_keywords",
+ "content_excerpt",
+ "og_title",
+ "og_description",
+ "og_image",
+ "og_type",
+ "twitter_card",
+ "twitter_title",
+ "twitter_image",
+ "tech_stack",
+ "depth",
+ "page_analysis",
+ "fetch_method",
+]
+
+
+def empty_crawl_row_ext(
+ url: str,
+ headers_dict: Optional[dict] = None,
+ redirect_chain_length: int = 0,
+) -> dict[str, Any]:
+ """Default SEO/performance extension fields when no HTML or on error."""
+ h = headers_dict or {}
+ return {
+ "response_time_ms": "",
+ "content_length": 0,
+ "final_url": url,
+ "meta_description": "",
+ "meta_description_len": 0,
+ "h1": "",
+ "h1_count": 0,
+ "canonical_url": "",
+ "viewport_present": False,
+ "viewport_content": "",
+ "noindex": False,
+ "has_schema": False,
+ "heading_sequence": "",
+ "heading_text": "",
+ "images_without_alt": 0,
+ "images_total": 0,
+ "img_without_lazy": 0,
+ "img_without_dimensions": 0,
+ "aria_count": 0,
+ "mixed_content_count": 0,
+ "redirect_chain_length": redirect_chain_length,
+ "cache_control": h.get("Cache-Control", ""),
+ "etag": h.get("ETag", ""),
+ "x_robots_tag": h.get("X-Robots-Tag", ""),
+ "strict_transport_security": h.get("Strict-Transport-Security", ""),
+ "x_content_type_options": h.get("X-Content-Type-Options", ""),
+ "x_frame_options": h.get("X-Frame-Options", ""),
+ "content_security_policy": h.get("Content-Security-Policy", ""),
+ "script_count": 0,
+ "link_stylesheet_count": 0,
+ "total_js_bytes": 0,
+ "total_css_bytes": 0,
+ "word_count": 0,
+ "reading_level": 0.0,
+ "content_html_ratio": 0.0,
+ "top_keywords": "[]",
+ "content_excerpt": "",
+ "og_title": "",
+ "og_description": "",
+ "og_image": "",
+ "og_type": "",
+ "twitter_card": "",
+ "twitter_title": "",
+ "twitter_image": "",
+ "tech_stack": "[]",
+ "depth": None,
+ "page_analysis": "{}",
+ }
+
+
+def empty_crawl_row(
+ url: Optional[str] = None,
+ status: str | int = "error",
+ *,
+ content_type: str = "",
+ title: str = "",
+ outlinks: int = 0,
+ fetch_method: str = "static",
+ headers_dict: Optional[dict] = None,
+ redirect_chain_length: int = 0,
+ **overrides: Any,
+) -> dict[str, Any]:
+ """Build a full crawl result row with defaults; overrides merge on top."""
+ row: dict[str, Any] = {
+ "url": url,
+ "status": status,
+ "content_type": content_type,
+ "title": title,
+ "outlinks": outlinks,
+ "fetch_method": fetch_method,
+ **empty_crawl_row_ext(url or "", headers_dict, redirect_chain_length),
+ }
+ row.update(overrides)
+ return row
+
+
+def crawl_dataframe_columns(*, store_outlinks: bool = False) -> list[str]:
+ """Column list for an empty crawl DataFrame."""
+ cols = list(CRAWL_ROW_COLUMNS)
+ if store_outlinks:
+ cols.append("outlink_targets")
+ return cols
diff --git a/src/website_profiling/db/crawl_store.py b/src/website_profiling/db/crawl_store.py
index a4122d17..fc50d327 100644
--- a/src/website_profiling/db/crawl_store.py
+++ b/src/website_profiling/db/crawl_store.py
@@ -234,6 +234,10 @@ def write_crawl(conn: Connection, df: pd.DataFrame, crawl_run_id: Optional[int]
with conn.transaction():
if crawl_run_id is not None:
conn.execute("DELETE FROM crawl_results WHERE crawl_run_id = %s", (crawl_run_id,))
+ try:
+ conn.execute("DELETE FROM crawl_page_html WHERE crawl_run_id = %s", (crawl_run_id,))
+ except Exception:
+ pass
target_run_id = crawl_run_id
else:
conn.execute("DELETE FROM crawl_results")
@@ -251,6 +255,39 @@ def write_crawl(conn: Connection, df: pd.DataFrame, crawl_run_id: Optional[int]
_write_crawl_rows(conn, rows)
+_MERGE_FIELDS_BATCH_SIZE = 200
+_MERGE_FIELDS_SQL = """UPDATE crawl_results
+SET data = COALESCE(data, '{}'::jsonb) || %s::jsonb
+WHERE crawl_run_id = %s AND url = %s"""
+
+
+def merge_crawl_result_fields_batch(
+ conn: Connection,
+ crawl_run_id: int,
+ updates: list[dict[str, Any]],
+ *,
+ commit: bool = True,
+) -> int:
+ """Merge per-URL content fields into crawl_results.data JSONB. Returns rows updated."""
+ if not updates:
+ return 0
+ params: list[tuple] = []
+ for item in updates:
+ url = str(item.get("url") or "").rstrip("/")
+ if not url:
+ continue
+ fields = {k: _sanitize_for_json(v) for k, v in item.items() if k != "url"}
+ if not fields:
+ continue
+ params.append((_json_val(fields), crawl_run_id, url))
+ if not params:
+ return 0
+ _executemany(conn, _MERGE_FIELDS_SQL, params, page_size=_MERGE_FIELDS_BATCH_SIZE)
+ if commit:
+ conn.commit()
+ return len(params)
+
+
def read_crawl(conn: Connection, run_id: Optional[int] = None) -> pd.DataFrame:
try:
return _read_crawl_rows(conn, run_id, include_fetch_method=True)
diff --git a/src/website_profiling/db/historical.py b/src/website_profiling/db/historical.py
index 0879c28e..b1e62798 100644
--- a/src/website_profiling/db/historical.py
+++ b/src/website_profiling/db/historical.py
@@ -207,7 +207,7 @@ def _bulk(
def ensure_crawl_tables_cleared(conn: Connection) -> None:
"""Clear crawl-scoped tables before a non-append crawl (preserves reports, Google, etc.)."""
- conn.execute("TRUNCATE crawl_results, edges, nodes RESTART IDENTITY CASCADE")
+ conn.execute("TRUNCATE crawl_results, crawl_page_html, edges, nodes RESTART IDENTITY CASCADE")
conn.commit()
diff --git a/src/website_profiling/db/html_store.py b/src/website_profiling/db/html_store.py
new file mode 100644
index 00000000..5b5a96d5
--- /dev/null
+++ b/src/website_profiling/db/html_store.py
@@ -0,0 +1,112 @@
+"""Per-URL raw HTML storage for crawl runs."""
+from __future__ import annotations
+
+from typing import Any, Iterator, Optional
+
+from psycopg import Connection
+
+from ._common import _executemany, _now_iso
+
+_HTML_BATCH_SIZE = 200
+
+_HTML_UPSERT_SQL = """INSERT INTO crawl_page_html (
+ crawl_run_id, url, html, status, content_type, fetch_method, byte_length, captured_at
+) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+ON CONFLICT (crawl_run_id, url) DO UPDATE SET
+ html = EXCLUDED.html,
+ status = EXCLUDED.status,
+ content_type = EXCLUDED.content_type,
+ fetch_method = EXCLUDED.fetch_method,
+ byte_length = EXCLUDED.byte_length,
+ captured_at = EXCLUDED.captured_at"""
+
+
+def _normalize_url(url: str) -> str:
+ return str(url or "").rstrip("/")
+
+
+def _rows_from_records(records: list[dict[str, Any]], crawl_run_id: int) -> list[tuple]:
+ rows: list[tuple] = []
+ captured_at = _now_iso()
+ for rec in records:
+ url = _normalize_url(str(rec.get("url") or ""))
+ html = rec.get("html")
+ if not url or not html:
+ continue
+ status = str(rec.get("status") or "") if rec.get("status") is not None else None
+ content_type = str(rec.get("content_type") or "") if rec.get("content_type") is not None else None
+ fetch_method = str(rec.get("fetch_method") or "static").strip() or "static"
+ byte_length = int(rec.get("byte_length") or len(str(html).encode("utf-8")))
+ rows.append(
+ (crawl_run_id, url, str(html), status, content_type, fetch_method, byte_length, captured_at)
+ )
+ return rows
+
+
+def write_page_html_batch(
+ conn: Connection,
+ records: list[dict[str, Any]],
+ crawl_run_id: int,
+ *,
+ commit: bool = True,
+) -> None:
+ """Upsert HTML rows for a crawl run (each record: url, html, status, content_type, fetch_method, byte_length)."""
+ rows = _rows_from_records(records, crawl_run_id)
+ if not rows:
+ return
+ _executemany(conn, _HTML_UPSERT_SQL, rows, page_size=_HTML_BATCH_SIZE)
+ if commit:
+ conn.commit()
+
+
+def read_page_html(conn: Connection, crawl_run_id: int, url: str) -> Optional[dict[str, Any]]:
+ """Return stored HTML and metadata for one URL, or None."""
+ norm = _normalize_url(url)
+ if not norm:
+ return None
+ try:
+ cur = conn.execute(
+ """SELECT url, html, status, content_type, fetch_method, byte_length, captured_at
+ FROM crawl_page_html
+ WHERE crawl_run_id = %s AND url = %s""",
+ (crawl_run_id, norm),
+ )
+ row = cur.fetchone()
+ if row is None:
+ return None
+ return dict(row)
+ except Exception:
+ return None
+
+
+def read_page_html_for_run(
+ conn: Connection,
+ crawl_run_id: int,
+ *,
+ limit: int = 5000,
+ offset: int = 0,
+) -> Iterator[dict[str, Any]]:
+ """Yield stored HTML rows for a crawl run (paginated)."""
+ try:
+ cur = conn.execute(
+ """SELECT url, html, status, content_type, fetch_method, byte_length, captured_at
+ FROM crawl_page_html
+ WHERE crawl_run_id = %s
+ ORDER BY url
+ LIMIT %s OFFSET %s""",
+ (crawl_run_id, max(1, int(limit)), max(0, int(offset))),
+ )
+ for row in cur.fetchall():
+ yield dict(row)
+ except Exception:
+ return
+
+
+def delete_page_html_for_run(conn: Connection, crawl_run_id: int, *, commit: bool = True) -> None:
+ """Delete all stored HTML for a crawl run."""
+ try:
+ conn.execute("DELETE FROM crawl_page_html WHERE crawl_run_id = %s", (crawl_run_id,))
+ if commit:
+ conn.commit()
+ except Exception:
+ pass
diff --git a/src/website_profiling/db/storage.py b/src/website_profiling/db/storage.py
index 443b7bf2..0999017a 100644
--- a/src/website_profiling/db/storage.py
+++ b/src/website_profiling/db/storage.py
@@ -29,9 +29,16 @@
read_nodes,
write_crawl,
write_crawl_batch,
+ merge_crawl_result_fields_batch,
write_edges,
write_nodes,
)
+from .html_store import (
+ delete_page_html_for_run,
+ read_page_html,
+ read_page_html_for_run,
+ write_page_html_batch,
+)
from .historical import backup_db_if_exists, ensure_crawl_tables_cleared, read_historical_data, restore_historical_data
from .lighthouse_store import (
read_latest_lighthouse_run_json,
@@ -59,6 +66,7 @@
"close_db_pool",
"create_crawl_run",
"create_session",
+ "delete_page_html_for_run",
"delete_session",
"db_session",
"ensure_crawl_tables_cleared",
@@ -70,7 +78,10 @@
"get_latest_crawl_run_id",
"init_schema",
"list_sessions",
+ "merge_crawl_result_fields_batch",
"read_crawl",
+ "read_page_html",
+ "read_page_html_for_run",
"read_edges",
"read_historical_data",
"read_latest_lighthouse_run_json",
@@ -90,6 +101,7 @@
"update_session_title",
"write_crawl",
"write_crawl_batch",
+ "write_page_html_batch",
"write_edges",
"write_lh_audits_from_run",
"write_lighthouse_page_summary",
diff --git a/src/website_profiling/integrations/google/auth.py b/src/website_profiling/integrations/google/auth.py
index 6364f077..6eda7087 100644
--- a/src/website_profiling/integrations/google/auth.py
+++ b/src/website_profiling/integrations/google/auth.py
@@ -36,7 +36,7 @@ def _app_client_credentials() -> tuple[str, str]:
return app_client_credentials()
-def _property_refresh_token(property_id: int) -> tuple[str, str | None]:
+def _property_google_auth(property_id: int) -> tuple[str, str | None, str]:
from ...db import db_session
from ...db.property_store import get_property_by_id
@@ -44,21 +44,16 @@ def _property_refresh_token(property_id: int) -> tuple[str, str | None]:
prop = get_property_by_id(conn, property_id)
if not prop:
raise RuntimeError(f"Property id {property_id} not found.")
- domain = prop.get("canonical_domain") or "this site"
token = (prop.get("google_refresh_token") or "").strip()
- if not token:
- raise RuntimeError(
- f"Google not connected for {domain}. "
- "Set Site URL, open Integrations, and click Connect with Google for this site."
- )
- return token, prop.get("google_auth_mode")
+ domain = prop.get("canonical_domain") or "this site"
+ return token, prop.get("google_auth_mode"), domain
def build_credentials(property_id: int | None = None):
"""
Load Google OAuth2 credentials.
property_id is required for OAuth user tokens.
- Service account uses google_app_settings.service_account_json when property_id is None.
+ Service account uses google_app_settings.service_account_json (app-wide).
"""
try:
from google.oauth2.credentials import Credentials
@@ -66,11 +61,17 @@ def build_credentials(property_id: int | None = None):
except ImportError as e:
raise ImportError(f"{INSTALL_HINT}\n({e})") from e
+ from ...db.google_app_store import has_service_account, build_service_account_credentials
+
if property_id is not None:
- refresh_token, prop_auth_mode = _property_refresh_token(property_id)
- if prop_auth_mode == "service_account":
+ refresh_token, prop_auth_mode, domain = _property_google_auth(property_id)
+ if prop_auth_mode == "service_account" or (not refresh_token and has_service_account()):
+ return build_service_account_credentials()
+ if not refresh_token:
raise RuntimeError(
- "Per-property service account is not implemented yet. Use OAuth Connect."
+ f"Google not connected for {domain}. "
+ "Set Site URL, open Integrations, and click Connect with Google for this site, "
+ "or upload an app-wide service account JSON in Integrations."
)
client_id, client_secret = _app_client_credentials()
creds = Credentials(
@@ -83,8 +84,6 @@ def build_credentials(property_id: int | None = None):
creds.refresh(Request())
return creds
- from ...db.google_app_store import has_service_account, build_service_account_credentials
-
if has_service_account():
return build_service_account_credentials()
diff --git a/src/website_profiling/integrations/google/keyword_store.py b/src/website_profiling/integrations/google/keyword_store.py
index c1d9b71b..6a6b5f4a 100644
--- a/src/website_profiling/integrations/google/keyword_store.py
+++ b/src/website_profiling/integrations/google/keyword_store.py
@@ -97,6 +97,35 @@ def append_keyword_history(
conn.commit()
+def read_keyword_snapshots_for_property(
+ conn: Connection,
+ property_id: int | None,
+ *,
+ limit: int = 2,
+) -> list[dict[str, Any]]:
+ """Return the most recent keyword_data snapshots for rank delta tools."""
+ if property_id is None:
+ return []
+ try:
+ cur = conn.execute(
+ """
+ SELECT fetched_at, data FROM keyword_data
+ WHERE property_id = %s
+ ORDER BY id DESC
+ LIMIT %s
+ """,
+ (property_id, max(1, int(limit))),
+ )
+ out: list[dict[str, Any]] = []
+ for row in cur.fetchall():
+ data = _parse_row_json(row)
+ if isinstance(data, dict):
+ out.append({"fetched_at": row["fetched_at"], **data})
+ return out
+ except Exception:
+ return []
+
+
def read_keyword_history(
conn: Connection,
keyword: str,
diff --git a/src/website_profiling/integrations/google/store.py b/src/website_profiling/integrations/google/store.py
index 0a4da7ae..ba5fdece 100644
--- a/src/website_profiling/integrations/google/store.py
+++ b/src/website_profiling/integrations/google/store.py
@@ -63,3 +63,103 @@ def read_latest_google_data(
def _to_payload_shape(data: dict[str, Any]) -> dict[str, Any]:
"""Strip gsc_full/ga4_full keys from the payload."""
return {k: v for k, v in data.items() if k not in ("gsc_full", "ga4_full")}
+
+
+def read_google_data_full(
+ conn: Connection,
+ property_id: int | None = None,
+) -> Optional[dict[str, Any]]:
+ """Return latest google_data row including gsc_full and ga4_full blobs."""
+ try:
+ if property_id is not None:
+ cur = conn.execute(
+ """
+ SELECT data FROM google_data
+ WHERE property_id = %s
+ ORDER BY id DESC LIMIT 1
+ """,
+ (property_id,),
+ )
+ else:
+ cur = conn.execute("SELECT data FROM google_data ORDER BY id DESC LIMIT 1")
+ row = cur.fetchone()
+ if row is None:
+ return None
+ data = _parse_row_json(row)
+ return data if isinstance(data, dict) else None
+ except Exception:
+ return None
+
+
+def read_prior_google_snapshot(
+ conn: Connection,
+ property_id: int | None = None,
+ *,
+ skip: int = 1,
+) -> Optional[dict[str, Any]]:
+ """Return the Nth-most-recent google_data row (skip=1 → prior snapshot)."""
+ try:
+ offset = max(0, int(skip))
+ if property_id is not None:
+ cur = conn.execute(
+ """
+ SELECT data FROM google_data
+ WHERE property_id = %s
+ ORDER BY id DESC
+ OFFSET %s LIMIT 1
+ """,
+ (property_id, offset),
+ )
+ else:
+ cur = conn.execute(
+ """
+ SELECT data FROM google_data
+ ORDER BY id DESC
+ OFFSET %s LIMIT 1
+ """,
+ (offset,),
+ )
+ row = cur.fetchone()
+ if row is None:
+ return None
+ data = _parse_row_json(row)
+ return data if isinstance(data, dict) else None
+ except Exception:
+ return None
+
+
+def gsc_row_deltas(
+ current_rows: list[dict[str, Any]],
+ prior_rows: list[dict[str, Any]],
+ *,
+ key_field: str,
+) -> list[dict[str, Any]]:
+ """Compute click/impression/position deltas keyed by page or query field."""
+ prior_by_key: dict[str, dict[str, Any]] = {}
+ for row in prior_rows:
+ if not isinstance(row, dict):
+ continue
+ key = str(row.get(key_field) or "").strip().lower()
+ if key:
+ prior_by_key[key] = row
+
+ deltas: list[dict[str, Any]] = []
+ for row in current_rows:
+ if not isinstance(row, dict):
+ continue
+ key = str(row.get(key_field) or "").strip().lower()
+ if not key:
+ continue
+ prior = prior_by_key.get(key) or {}
+ cur_clicks = float(row.get("clicks") or 0)
+ pri_clicks = float(prior.get("clicks") or 0)
+ cur_impr = float(row.get("impressions") or 0)
+ pri_impr = float(prior.get("impressions") or 0)
+ cur_pos = float(row.get("position") or row.get("avg_position") or 0)
+ pri_pos = float(prior.get("position") or prior.get("avg_position") or 0)
+ out = dict(row)
+ out["clicks_delta"] = cur_clicks - pri_clicks
+ out["impressions_delta"] = cur_impr - pri_impr
+ out["position_delta"] = cur_pos - pri_pos if pri_pos else None
+ deltas.append(out)
+ return deltas
diff --git a/src/website_profiling/lighthouse/config.py b/src/website_profiling/lighthouse/config.py
new file mode 100644
index 00000000..1a6c7b0b
--- /dev/null
+++ b/src/website_profiling/lighthouse/config.py
@@ -0,0 +1,166 @@
+"""Lighthouse CLI configuration and command helpers."""
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import threading
+from pathlib import Path
+
+# Lighthouse "good" thresholds for human summary
+LCP_GOOD_MS = 2500
+CLS_GOOD = 0.1
+TBT_GOOD_MS = 200
+FCP_GOOD_MS = 1800
+
+_LIGHTHOUSE_INSTALL_MSG = (
+ "Lighthouse not found. Install Node/npm (https://nodejs.org), then run: npm install -g lighthouse. "
+ "Chrome or Chromium is also required for headless mode."
+)
+
+_NPX_LIGHTHOUSE_LOCK = threading.Lock()
+_LIGHTHOUSE_FLOW_MODES = frozenset({"snapshot", "timespan"})
+
+def _repo_root() -> str:
+ explicit = (os.environ.get("WEBSITE_PROFILING_ROOT") or "").strip()
+ if explicit:
+ return explicit
+ return str(Path(__file__).resolve().parents[3])
+
+
+def _lighthouse_flow_script() -> str:
+ return os.path.join(_repo_root(), "scripts", "lighthouse_user_flow.mjs")
+
+
+def _normalize_lighthouse_mode(mode: str | None) -> str:
+ m = (mode or "navigation").strip().lower() or "navigation"
+ if m not in ("navigation", "snapshot", "timespan"):
+ raise RuntimeError(
+ f"Invalid lighthouse_mode {m!r}; use navigation, snapshot, or timespan."
+ )
+ return m
+
+
+def _node_cmd() -> str:
+ node = shutil.which("node")
+ if node is None:
+ raise RuntimeError(
+ "Node.js not found. Install Node.js (https://nodejs.org) for Lighthouse user flows."
+ )
+ return node
+
+
+def _build_report_html_content(summary: dict[str, Any]) -> str:
+ """Build report.html content (for DB or file). Returns HTML string."""
+ import html as html_module
+ mm = summary.get("median_metrics") or {}
+ cs = summary.get("category_scores") or {}
+ failures = summary.get("top_failures") or []
+ raw_reports = summary.get("raw_reports") or []
+ url = html_module.escape(summary.get("url", ""))
+ path_summary = "summary.json"
+ path_human = "human_summary.txt"
+ path_diag = "diagnostics.json"
+ raw_dir = "raw_runs"
+ rows_fail = "".join(
+ f"