From 956416b9afc8288b3478070fab678301d2a75b2c Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Fri, 12 Jun 2026 23:36:53 +0530 Subject: [PATCH 1/4] Fising tools and improving tools calling --- .github/workflows/ci.yml | 3 +- AGENT.md | 2 +- README.md | 6 +- docs/MCP.md | 43 +- scripts/local-test.ps1 | 5 + scripts/local-test.sh | 2 + src/website_profiling/config.py | 7 + .../integrations/google/gsc_inspection.py | 95 +++ .../integrations/google/rich_results.py | 56 +- src/website_profiling/llm/agent.py | 12 +- src/website_profiling/mcp/server.py | 26 +- .../reporting/compare_payload.py | 60 ++ .../tools/audit_tools/compare_slices.py | 24 + .../tools/audit_tools/crawl_lists.py | 268 +++++++- .../tools/audit_tools/crawl_metrics.py | 81 +++ .../tools/audit_tools/geo_tools.py | 349 ++++++++++ .../tools/audit_tools/google.py | 52 ++ .../tools/audit_tools/integration_tools.py | 143 ++++ .../tools/audit_tools/keywords.py | 8 + .../tools/audit_tools/llm_tools.py | 205 ++++++ .../tools/audit_tools/payload_extras.py | 142 ++++ .../tools/audit_tools/registry.py | 87 +++ .../tools/audit_tools/report.py | 19 +- .../tools/audit_tools/tool_catalog.py | 54 ++ tests/test_audit_tools_expanded.py | 21 +- tests/test_audit_tools_expansion.py | 203 ++++++ tests/test_audit_tools_expansion_coverage.py | 633 ++++++++++++++++++ tests/test_compare_payload.py | 45 ++ tests/test_config_parsing_unit.py | 7 +- tests/test_mcp_registry.py | 2 +- tests/test_mcp_server_helpers.py | 6 +- 31 files changed, 2579 insertions(+), 87 deletions(-) create mode 100644 src/website_profiling/integrations/google/gsc_inspection.py create mode 100644 src/website_profiling/tools/audit_tools/crawl_metrics.py create mode 100644 src/website_profiling/tools/audit_tools/geo_tools.py create mode 100644 src/website_profiling/tools/audit_tools/integration_tools.py create mode 100644 src/website_profiling/tools/audit_tools/payload_extras.py create mode 100644 tests/test_audit_tools_expansion.py create mode 100644 tests/test_audit_tools_expansion_coverage.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a2e1d1..588d4e98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,8 @@ jobs: pytest tests/test_alert_checker.py tests/test_schedule_runner.py tests/test_export_audit.py \ tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_audit_tools_expanded.py \ tests/test_audit_tools_coverage.py tests/test_audit_tools_dispatch_coverage.py \ - tests/test_audit_tools_links_extras.py tests/test_export_custom_coverage.py \ + tests/test_audit_tools_links_extras.py tests/test_audit_tools_expansion.py \ + tests/test_audit_tools_expansion_coverage.py tests/test_export_custom_coverage.py \ tests/test_export_artifacts_coverage.py tests/test_export_compare_coverage.py \ tests/test_export_tools_coverage.py tests/test_image_tools.py tests/test_export_custom.py \ tests/test_export_artifacts.py tests/test_export_compare.py tests/test_export_workbook.py \ diff --git a/AGENT.md b/AGENT.md index a93961df..8993821e 100644 --- a/AGENT.md +++ b/AGENT.md @@ -25,7 +25,7 @@ - **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline. - **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch. - **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/chat` POST (SSE agent); `/api/chat/sessions` GET/POST; `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run -- **MCP:** `python -m website_profiling.mcp` (stdio, **121 read-only audit tools** + MCP resources). See `docs/MCP.md`. Requires `pip install -r requirements-mcp.txt`. +- **MCP:** `python -m website_profiling.mcp` (stdio, **221 read-only audit tools** + MCP resources). See `docs/MCP.md`. Requires `pip install -r requirements-mcp.txt`. - **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages` tables, migration `012_chat_sessions`). - **Job store:** in-memory on `globalThis` in `web/src/server/pipelineJobs.ts` — job status/log is lost on server restart (single-process dev/Docker only). - **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`** diff --git a/README.md b/README.md index 8c7f815e..4c723c7c 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Repository: [codefrydev/WebsiteProfiling](https://github.com/codefrydev/WebsiteP -Also included: **AI chat** over audit data (optional), **121 MCP tools**, keyword explorer, backlinks, compare runs, and portfolio management for agencies. +Also included: **AI chat** over audit data (optional), **221 MCP tools**, keyword explorer, backlinks, compare runs, and portfolio management for agencies.

Site Audit preview @@ -91,7 +91,7 @@ WebsiteProfiling/ │ ├── integrations/ # Google Search Console, GA4, Bing, CrUX │ ├── llm/ # AI enrich + chat agent │ ├── tools/ # Exports, audit query tools, MCP helpers -│ ├── mcp/ # MCP server (121 read-only tools) +│ ├── mcp/ # MCP server (221 read-only tools) │ ├── db/ # PostgreSQL storage layer │ ├── commands/ # CLI subcommands │ ├── cli.py # Pipeline entrypoint @@ -182,7 +182,7 @@ Google Search Console / Analytics: connect via **Integrations** (gear icon) in t | **Ollama** | Local daemon at `http://127.0.0.1:11434`. Chat UI lists installed models plus the live Ollama cloud catalog (billing: free local, account free tier, Pro). Native tool calling when supported; otherwise ReAct fallback. Pick the model in-chat without leaving the page. | | **OpenAI** / **Anthropic** | API key in AI settings; native tool calling with streaming. | -The agent uses the same **121 read-only audit tools** as the MCP server (`docs/MCP.md`). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`). +The agent uses the same **221 read-only audit tools** as the MCP server (`docs/MCP.md`). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`). Production: `docker-compose.prod.yml` (set `POSTGRES_PASSWORD`, `AUTH_SECRET`). diff --git a/docs/MCP.md b/docs/MCP.md index d3ba6f94..4b0424bf 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -43,7 +43,7 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings): | `audit://glossary` | Excerpt from `docs/GLOSSARY.md` | | `audit://tools` | Tool catalog grouped by SEO domain | -## Tools (176 read-only + export) +## Tools (221 read-only + export) ### Export and deliverables @@ -63,7 +63,7 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu ### Issues and workflow -`list_issues`, `search_issues`, `list_issues_by_category`, `get_category_issues`, `list_issue_workflow`, `list_issues_with_ai_fixes`, `list_seo_onpage_issues` +`list_issues`, `search_issues`, `list_top_impact_issues`, `prioritize_fix_roadmap`, `list_issues_by_category`, `get_category_issues`, `list_issue_workflow`, `list_issues_with_ai_fixes`, `generate_issue_fix`, `summarize_category_for_client`, `list_seo_onpage_issues` ### On-page SEO @@ -71,7 +71,15 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu ### Crawl and pages -`search_pages`, `search_pages_advanced`, `get_page_details`, `get_page_analysis`, `get_internal_links`, `list_redirects`, `list_broken_links`, `list_status_4xx_pages`, `list_status_5xx_pages`, `get_status_code_breakdown`, `get_response_time_stats`, `get_depth_distribution`, `get_crawl_segments`, `get_browser_diagnostics_summary`, `list_pages_with_console_errors`, `list_pages_by_fetch_method`, `get_crawl_links_table`, `get_graph_edges_sample`, `list_long_redirect_chains`, `list_robots_blocked_urls`, `get_top_pages_by_pagerank` +`search_pages`, `search_pages_advanced`, `get_page_details`, `get_page_analysis`, `get_internal_links`, `list_redirects`, `list_broken_links`, `list_status_4xx_pages`, `list_status_5xx_pages`, `list_pages_soft_404`, `list_dead_end_pages`, `list_duplicate_title_groups`, `list_heavy_pages_by_bytes`, `list_pages_poor_cache_headers`, `list_pages_low_content_ratio`, `get_heading_outline_for_url`, `get_status_code_breakdown`, `get_response_time_stats`, `get_depth_distribution`, `get_crawl_segments`, `get_browser_diagnostics_summary`, `list_pages_with_console_errors`, `list_pages_by_fetch_method`, `get_crawl_links_table`, `get_graph_edges_sample`, `list_long_redirect_chains`, `list_robots_blocked_urls`, `get_top_pages_by_pagerank`, `get_pagination_audit_summary`, `get_js_rendering_delta` + +### Accessibility and assets + +`list_pages_with_axe_violations`, `get_axe_audit_summary`, `list_pages_with_mixed_content`, `get_asset_weight_summary`, `get_readability_summary` + +### Rich results and portfolio extras + +`get_rich_results_summary`, `list_rich_results_failures`, `get_competitor_keyword_gap`, `get_portfolio_benchmark`, `get_site_anchor_text_summary` ### Schema and technical @@ -91,11 +99,11 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu ### Keywords -`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`, `expand_keywords`, `generate_content_brief` +`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `get_serp_feature_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`, `list_keywords_ctr_opportunity`, `expand_keywords`, `generate_content_brief` -### Google +### Google and CTR -`get_google_summary`, `get_google_integration_status`, `get_gsc_top_queries`, `get_gsc_top_pages`, `get_ga4_summary`, `get_ga4_page_metrics`, `get_gsc_page_query_slice` +`get_google_summary`, `get_google_integration_status`, `get_gsc_top_queries`, `get_gsc_top_pages`, `get_gsc_ctr_opportunity_pages`, `get_ga4_summary`, `get_ga4_page_metrics`, `get_gsc_page_query_slice`, `get_gsc_url_inspection`, `get_gsc_index_coverage`, `analyze_serp_snippet_for_url` ### Backlinks @@ -107,21 +115,27 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu ### Drift, health, and compare -`get_health_history`, `get_category_health_history`, `compare_reports`, `compare_issue_deltas`, `compare_category_deltas`, `compare_seo_health_deltas`, `compare_lighthouse_deltas`, `compare_url_set_diff`, `compare_redirect_deltas`, `compare_link_metric_deltas`, `compare_security_deltas`, `compare_duplicate_deltas`, `compare_tech_deltas`, `compare_content_metrics`, `compare_google_metrics`, `compare_priority_counts`, `compare_health_score_delta` +`get_health_history`, `get_category_health_history`, `compare_reports`, `compare_issue_deltas`, `compare_category_deltas`, `compare_seo_health_deltas`, `compare_lighthouse_deltas`, `compare_url_set_diff`, `compare_redirect_deltas`, `compare_link_metric_deltas`, `compare_security_deltas`, `compare_duplicate_deltas`, `compare_tech_deltas`, `compare_content_metrics`, `compare_google_metrics`, `compare_priority_counts`, `compare_health_score_delta`, `compare_indexation_deltas`, `compare_orphan_deltas` + +### GEO / AEO + +`get_geo_readiness_score`, `get_aeo_content_signals_for_url`, `get_llms_txt_status`, `draft_llms_txt`, `get_faq_schema_coverage`, `list_pages_missing_faq_schema`, `get_eeat_signals_summary`, `get_internal_link_suggestions`, `check_ai_citation_presence` + +### Integrations + +`get_bing_index_status` (requires `bing_webmaster_api_key` in audit settings) ### Ops and logs `get_integration_alerts`, `get_property_ops`, `list_crawl_runs`, `list_log_uploads`, `get_latest_log_analysis`, `get_log_top_paths`, `list_log_only_paths`, `list_crawl_only_paths`, `get_log_googlebot_stats`, `get_log_analysis_by_id`, `get_page_coach` -## Future pipeline items (not yet exposed as tools) - -These require additional third-party integrations or product scope beyond current crawl data: +## Future pipeline items - Full backlink index and anchor-text analytics (beyond GSC Links import) - SERP rank tracking beyond GSC position snapshots -- Standalone Google Rich Results Test API (current `validate_rich_results` uses crawl heuristics + GSC URL Inspection) +- Live AI citation checks across ChatGPT/Perplexity (current `check_ai_citation_presence` uses on-site heuristics) -Already available: `validate_rich_results`, `export_sitemap_xml`, workbook export, axe audits via `enable_axe` on browser crawls. +Already available: `validate_rich_results`, `get_gsc_url_inspection`, `export_sitemap_xml`, workbook export, axe audits via `enable_axe` on browser crawls. ## Example prompts @@ -138,6 +152,11 @@ Already available: `validate_rich_results`, `export_sitemap_xml`, workbook expor - "Compare report 38 to the current audit and give me a CSV diff" - "Build a client report with executive summary, category scores, and top critical issues as PDF" - "Which images are largest and unoptimized?" +- "What should we fix first on high-traffic pages?" (use `list_top_impact_issues` or `prioritize_fix_roadmap`) +- "What's our GEO readiness score?" +- "Inspect GSC indexing for https://example.com/page" +- "Which pages are soft 404s or dead ends?" +- "Suggest internal links for our top blog post" - "List pages with images missing alt or lazy loading" ## In-app chat diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1 index 48b1b3be..b201d11b 100644 --- a/scripts/local-test.ps1 +++ b/scripts/local-test.ps1 @@ -259,6 +259,9 @@ function Invoke-PytestTools { tests/test_audit_tools_expanded.py ` tests/test_audit_tools_coverage.py ` tests/test_audit_tools_dispatch_coverage.py ` + tests/test_audit_tools_links_extras.py ` + tests/test_audit_tools_expansion.py ` + tests/test_audit_tools_expansion_coverage.py ` tests/test_export_custom_coverage.py ` tests/test_export_artifacts_coverage.py ` tests/test_export_compare_coverage.py ` @@ -267,6 +270,8 @@ function Invoke-PytestTools { tests/test_export_custom.py ` tests/test_export_artifacts.py ` tests/test_export_compare.py ` + tests/test_export_workbook.py ` + tests/test_export_sitemap.py ` tests/test_mcp_registry.py ` tests/test_mcp_resources.py ` tests/test_tools_branch_coverage.py ` diff --git a/scripts/local-test.sh b/scripts/local-test.sh index a24022a8..56a0e627 100755 --- a/scripts/local-test.sh +++ b/scripts/local-test.sh @@ -148,6 +148,8 @@ run_pytest_tools() { tests/test_audit_tools_coverage.py \ tests/test_audit_tools_dispatch_coverage.py \ tests/test_audit_tools_links_extras.py \ + tests/test_audit_tools_expansion.py \ + tests/test_audit_tools_expansion_coverage.py \ tests/test_export_custom_coverage.py \ tests/test_export_artifacts_coverage.py \ tests/test_export_compare_coverage.py \ diff --git a/src/website_profiling/config.py b/src/website_profiling/config.py index 10c09f99..9308b661 100644 --- a/src/website_profiling/config.py +++ b/src/website_profiling/config.py @@ -29,6 +29,13 @@ def load_config(path: str) -> dict[str, str]: return result +def get_str(cfg: dict, key: str, default: str = "") -> str: + raw = cfg.get(key) + if raw is None: + return default + return str(raw) + + def get_bool(cfg: dict, key: str, default: bool = False) -> bool: return str(cfg.get(key, default)).lower() in ("true", "1", "yes") diff --git a/src/website_profiling/integrations/google/gsc_inspection.py b/src/website_profiling/integrations/google/gsc_inspection.py new file mode 100644 index 00000000..97bb88b0 --- /dev/null +++ b/src/website_profiling/integrations/google/gsc_inspection.py @@ -0,0 +1,95 @@ +"""Google Search Console URL Inspection API helpers.""" +from __future__ import annotations + +from typing import Any + + +def inspect_url(creds: Any, site_url: str, url: str) -> dict[str, Any]: + """Run GSC URL Inspection for one URL. Returns indexing + rich results slice.""" + from .gsc import _build_service, _call_with_retry, list_gsc_sites, resolve_gsc_site_url + + sites = list_gsc_sites(creds) + resolved, err = resolve_gsc_site_url(site_url, sites) + if not resolved: + return { + "ok": False, + "url": url, + "error": err or "GSC site URL not accessible.", + "provenance": "Search Console", + } + service = _build_service(creds) + body = {"inspectionUrl": url, "siteUrl": resolved} + resp = _call_with_retry( + lambda: service.urlInspection().index().inspect(body=body).execute() + ) + inspection = resp.get("inspectionResult") or {} + index_status = inspection.get("indexStatusResult") or {} + rich = inspection.get("richResultsResult") or {} + verdict = str(rich.get("verdict") or "UNKNOWN") + detected = rich.get("detectedItems") or [] + types: list[str] = [] + for item in detected: + if isinstance(item, dict) and item.get("richResultType"): + types.append(str(item["richResultType"])) + return { + "ok": True, + "url": url, + "site_url": resolved, + "indexing": { + "verdict": index_status.get("verdict"), + "coverage_state": index_status.get("coverageState"), + "robots_txt_state": index_status.get("robotsTxtState"), + "indexing_state": index_status.get("indexingState"), + "last_crawl_time": index_status.get("lastCrawlTime"), + "page_fetch_state": index_status.get("pageFetchState"), + }, + "rich_results": { + "verdict": verdict, + "schema_types": types[:10], + "issues": [ + str(i.get("issueMessage") or i.get("severity") or i) + for i in (rich.get("issues") or [])[:5] + if isinstance(i, dict) or i + ], + }, + "provenance": "Search Console", + } + + +def inspect_url_rich_results_row(creds: Any, site_url: str, url: str) -> dict[str, Any]: + """Compatibility wrapper matching rich_results.validate_urls row shape.""" + result = inspect_url(creds, site_url, url) + if not result.get("ok"): + return { + "url": url, + "status": "error", + "provenance": "Google Search Console", + "source": "gsc", + "message": result.get("error") or "Inspection failed", + } + rich = result.get("rich_results") or {} + verdict = str(rich.get("verdict") or "UNKNOWN") + types = rich.get("schema_types") or [] + status = "pass" + if verdict in ("FAIL", "ERROR"): + status = "fail" + elif verdict in ("PARTIAL", "WARN", "WARNING"): + status = "warning" + elif verdict in ("NEUTRAL", "UNKNOWN", "NOT_APPLICABLE"): + status = "info" + message = f"Rich Results verdict: {verdict}" + if types: + message += f" ({', '.join(types[:5])})" + row: dict[str, Any] = { + "url": url, + "status": status, + "provenance": "Google Search Console", + "source": "gsc", + "message": message, + "verdict": verdict, + } + if types: + row["schema_types"] = types + if rich.get("issues"): + row["issues"] = rich["issues"] + return row diff --git a/src/website_profiling/integrations/google/rich_results.py b/src/website_profiling/integrations/google/rich_results.py index 28e559e3..149a5875 100644 --- a/src/website_profiling/integrations/google/rich_results.py +++ b/src/website_profiling/integrations/google/rich_results.py @@ -70,61 +70,9 @@ def _local_row(url: str, link: dict[str, Any] | None) -> dict[str, Any]: def _inspect_via_gsc(creds: Any, site_url: str, url: str) -> dict[str, Any]: - from .gsc import _build_service, _call_with_retry, resolve_gsc_site_url, list_gsc_sites + from .gsc_inspection import inspect_url_rich_results_row - sites = list_gsc_sites(creds) - resolved, err = resolve_gsc_site_url(site_url, sites) - if not resolved: - return { - "url": url, - "status": "error", - "provenance": "Google Search Console", - "source": "gsc", - "message": err or "GSC site URL not accessible.", - } - service = _build_service(creds) - body = {"inspectionUrl": url, "siteUrl": resolved} - resp = _call_with_retry( - lambda: service.urlInspection().index().inspect(body=body).execute() - ) - inspection = resp.get("inspectionResult") or {} - rich = inspection.get("richResultsResult") or {} - verdict = str(rich.get("verdict") or "UNKNOWN") - detected = rich.get("detectedItems") or [] - types: list[str] = [] - for item in detected: - if isinstance(item, dict): - rt = item.get("richResultType") - if rt: - types.append(str(rt)) - status = "pass" - if verdict in ("FAIL", "ERROR"): - status = "fail" - elif verdict in ("PARTIAL", "WARN", "WARNING"): - status = "warning" - elif verdict in ("NEUTRAL", "UNKNOWN", "NOT_APPLICABLE"): - status = "info" - message = f"Rich Results verdict: {verdict}" - if types: - message += f" ({', '.join(types[:5])})" - row: dict[str, Any] = { - "url": url, - "status": status, - "provenance": "Google Search Console", - "source": "gsc", - "message": message, - "verdict": verdict, - } - if types: - row["schema_types"] = types[:10] - issues = rich.get("issues") or [] - if issues: - row["issues"] = [ - str(i.get("issueMessage") or i.get("severity") or i) - for i in issues[:5] - if isinstance(i, dict) or i - ] - return row + return inspect_url_rich_results_row(creds, site_url, url) def _inspect_via_rich_results_api(api_key: str, url: str) -> dict[str, Any]: diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py index 687f5d09..21868726 100644 --- a/src/website_profiling/llm/agent.py +++ b/src/website_profiling/llm/agent.py @@ -17,18 +17,20 @@ Tool domains (prefer specific tools over generic list_issues): - Portfolio/report: get_report_summary, get_category_scores, list_audit_categories, get_executive_summary, get_audit_recommendations, list_report_history, get_portfolio_summary -- Issues: list_issues, search_issues, get_critical_issues, list_issues_by_category, get_category_issues, list_issues_with_ai_fixes, list_issue_workflow +- Issues: list_issues, search_issues, list_top_impact_issues, prioritize_fix_roadmap, get_critical_issues, list_issues_by_category, get_category_issues, list_issues_with_ai_fixes, generate_issue_fix, list_issue_workflow - On-page: list_pages_missing_title, list_pages_noindex, list_seo_onpage_issues, list_content_url_issues, list_pages_missing_canonical, list_canonical_mismatch, list_pages_with_missing_alt, list_pages_missing_viewport -- Crawl/pages: search_pages, search_pages_advanced, get_page_details, get_page_analysis, list_status_4xx_pages, get_status_code_breakdown, get_depth_distribution, list_long_redirect_chains, list_robots_blocked_urls, get_top_pages_by_pagerank +- Crawl/pages: search_pages, search_pages_advanced, get_page_details, get_page_analysis, list_status_4xx_pages, list_pages_soft_404, list_dead_end_pages, list_duplicate_title_groups, list_heavy_pages_by_bytes, get_asset_weight_summary, get_readability_summary, get_status_code_breakdown, get_depth_distribution, list_long_redirect_chains, list_robots_blocked_urls, get_top_pages_by_pagerank - Schema/technical: get_schema_coverage, get_seo_health, get_security_findings, get_security_findings_summary, get_tech_stack_summary, list_pages_by_technology - Indexation: get_indexation_coverage, list_indexation_gaps, get_indexation_url_join -- Keywords: get_keyword_summary, get_striking_distance_keywords, list_keywords_by_position, get_keyword_serp_overlay, expand_keywords, generate_content_brief -- Google: get_google_summary, get_gsc_top_queries, get_gsc_top_pages, get_google_integration_status, get_gsc_page_query_slice, get_ga4_page_metrics +- Keywords: get_keyword_summary, get_striking_distance_keywords, list_keywords_ctr_opportunity, list_keywords_by_position, get_keyword_serp_overlay, get_serp_feature_overlay, expand_keywords, generate_content_brief +- Google: get_google_summary, get_gsc_top_queries, get_gsc_top_pages, get_gsc_ctr_opportunity_pages, get_google_integration_status, get_gsc_page_query_slice, get_gsc_url_inspection, get_gsc_index_coverage, get_ga4_page_metrics, analyze_serp_snippet_for_url - Links/backlinks: get_gsc_sample_links, get_backlinks_velocity, get_third_party_links_overlay, list_broken_link_sources, get_page_coach - Performance: get_lighthouse_summary, list_slow_pages, get_crux_summary, get_lighthouse_human_summary, list_lighthouse_poor_accessibility_pages, list_lighthouse_cwv_failures - Content/charts: get_issue_priority_breakdown, get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_outlink_distribution, get_content_analytics, get_top_crawled_pages, get_duplicate_cluster - Ops/logs: get_property_ops, list_crawl_runs, get_latest_log_analysis, get_log_top_paths, list_log_only_paths, list_crawl_only_paths, get_log_googlebot_stats -- Drift: compare_reports, compare_category_deltas, compare_issue_deltas, compare_url_set_diff, compare_google_metrics, compare_security_deltas, compare_health_score_delta, get_health_history, get_category_health_history +- Drift: compare_reports, compare_category_deltas, compare_issue_deltas, compare_indexation_deltas, compare_orphan_deltas, compare_url_set_diff, compare_google_metrics, compare_security_deltas, compare_health_score_delta, get_health_history, get_category_health_history +- GEO/AEO: get_geo_readiness_score, get_aeo_content_signals_for_url, get_llms_txt_status, draft_llms_txt, get_faq_schema_coverage, get_eeat_signals_summary, get_internal_link_suggestions, check_ai_citation_presence +- Accessibility/assets: list_pages_with_axe_violations, get_axe_audit_summary, list_pages_with_mixed_content, list_pages_poor_cache_headers, get_rich_results_summary, list_rich_results_failures - Export/deliverables: export_audit_report, export_compare_csv, export_list_as_csv, compose_custom_report, export_custom_report, list_export_formats - Images: get_image_audit_summary, list_pages_with_missing_alt, list_pages_without_lazy_images, list_pages_with_images_missing_dimensions, list_site_image_urls, list_lighthouse_image_opportunities, list_largest_images, list_unoptimized_images, list_images_needing_attention diff --git a/src/website_profiling/mcp/server.py b/src/website_profiling/mcp/server.py index ca7808a3..ded725f3 100644 --- a/src/website_profiling/mcp/server.py +++ b/src/website_profiling/mcp/server.py @@ -82,6 +82,11 @@ def _tools_catalog_json() -> str: "ops": [], "export": [], "images": [], + "geo": [], + "accessibility": [], + "assets": [], + "ctr": [], + "integrations": [], } for tool in TOOL_DEFINITIONS: name = tool["name"] @@ -93,10 +98,29 @@ def _tools_catalog_json() -> str: "get_ads_txt_status", "get_security_txt_status", "get_contact_intelligence", + "get_rich_results_summary", + "list_rich_results_failures", + "get_competitor_keyword_gap", + "get_pagination_audit_summary", ): domains["portfolio"].append(name) - elif "issue" in name or "category" in name or "workflow" in name: + elif name in ( + "list_top_impact_issues", + "prioritize_fix_roadmap", + "generate_issue_fix", + "summarize_category_for_client", + ) or "issue" in name or "category" in name or "workflow" in name: domains["issues"].append(name) + elif name.startswith(("get_geo_", "get_aeo_", "get_llms_", "get_eeat_", "get_faq_", "list_pages_missing_faq", "draft_llms", "check_ai_citation")): + domains["geo"].append(name) + elif "axe" in name or "mixed_content" in name or name == "get_heading_outline_for_url": + domains["accessibility"].append(name) + elif name in ("get_asset_weight_summary", "get_readability_summary", "list_heavy_pages_by_bytes", "list_pages_poor_cache_headers", "list_pages_low_content_ratio"): + domains["assets"].append(name) + elif "ctr" in name or name in ("list_keywords_ctr_opportunity", "analyze_serp_snippet_for_url"): + domains["ctr"].append(name) + elif name in ("get_gsc_url_inspection", "get_gsc_index_coverage", "get_bing_index_status", "get_serp_feature_overlay"): + domains["integrations"].append(name) elif name.startswith(("list_pages_", "list_canonical", "list_long_", "list_robots_", "get_top_pages_by", "search_pages", "get_page_", "list_redirects", "list_broken", "list_status_", "get_status_code", "get_response_time", "get_depth", "get_crawl_", "get_browser", "list_pages_with", "list_pages_by")): domains["crawl"].append(name) elif "schema" in name or name == "get_seo_health": diff --git a/src/website_profiling/reporting/compare_payload.py b/src/website_profiling/reporting/compare_payload.py index 3ef511a8..fc187e6c 100644 --- a/src/website_profiling/reporting/compare_payload.py +++ b/src/website_profiling/reporting/compare_payload.py @@ -529,6 +529,66 @@ def _url_map(payload: dict[str, Any]) -> dict[str, str]: } +def build_indexation_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> dict[str, Any]: + """Indexation coverage count and gap list changes between reports.""" + cur_cov = current.get("indexation_coverage") if isinstance(current.get("indexation_coverage"), dict) else {} + base_cov = baseline.get("indexation_coverage") if isinstance(baseline.get("indexation_coverage"), dict) else {} + cur_counts = cur_cov.get("counts") if isinstance(cur_cov.get("counts"), dict) else {} + base_counts = base_cov.get("counts") if isinstance(base_cov.get("counts"), dict) else {} + count_deltas: list[dict[str, Any]] = [] + for key in sorted(set(cur_counts) | set(base_counts)): + cur_v = cur_counts.get(key) + base_v = base_counts.get(key) + try: + delta = int(cur_v or 0) - int(base_v or 0) + except (TypeError, ValueError): + delta = None + count_deltas.append({"metric": key, "current": cur_v, "baseline": base_v, "delta": delta}) + gap_types = ("sitemap_only", "crawled_not_in_sitemap", "gsc_not_crawled") + gap_deltas: dict[str, Any] = {} + cur_lists = cur_cov.get("lists") if isinstance(cur_cov.get("lists"), dict) else {} + base_lists = base_cov.get("lists") if isinstance(base_cov.get("lists"), dict) else {} + + def _norm_set(items: list[Any]) -> set[str]: + return {norm_report_url(str(u)) for u in items if u} + + for gap in gap_types: + cur_set = _norm_set(cur_lists.get(gap) or []) + base_set = _norm_set(base_lists.get(gap) or []) + added = sorted(cur_set - base_set) + removed = sorted(base_set - cur_set) + gap_deltas[gap] = { + "added_count": len(added), + "removed_count": len(removed), + "added": added[:50], + "removed": removed[:50], + } + return {"count_deltas": count_deltas, "gap_deltas": gap_deltas} + + +def build_orphan_deltas(current: dict[str, Any], baseline: dict[str, Any]) -> dict[str, Any]: + """Orphan URL set changes between reports.""" + def _orphan_set(payload: dict[str, Any]) -> set[str]: + urls = payload.get("orphan_urls") or [] + if not isinstance(urls, list): + return set() + return {norm_report_url(str(u)) for u in urls if u} + + cur_set = _orphan_set(current) + base_set = _orphan_set(baseline) + added = sorted(cur_set - base_set) + removed = sorted(base_set - cur_set) + return { + "current_count": len(cur_set), + "baseline_count": len(base_set), + "delta": len(cur_set) - len(base_set), + "added": added[:100], + "removed": removed[:100], + "added_count": len(added), + "removed_count": len(removed), + } + + def build_full_compare( current: dict[str, Any], baseline: dict[str, Any], diff --git a/src/website_profiling/tools/audit_tools/compare_slices.py b/src/website_profiling/tools/audit_tools/compare_slices.py index a7b2636c..f51bacea 100644 --- a/src/website_profiling/tools/audit_tools/compare_slices.py +++ b/src/website_profiling/tools/audit_tools/compare_slices.py @@ -10,9 +10,11 @@ build_content_metrics, build_duplicate_deltas, build_google_metrics, + build_indexation_deltas, build_issue_deltas, build_lighthouse_url_deltas, build_link_metric_deltas, + build_orphan_deltas, build_priority_counts, build_redirect_deltas, build_security_deltas, @@ -230,3 +232,25 @@ def compare_health_score_delta(conn: Connection, ctx: AuditToolContext, args: di "delta": (cur_health - base_health) if cur_health is not None and base_health is not None else None, }, } + + +def compare_indexation_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + **build_indexation_deltas(current, baseline), + } + + +def compare_orphan_deltas(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return err + assert current is not None and baseline is not None + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + **build_orphan_deltas(current, baseline), + } diff --git a/src/website_profiling/tools/audit_tools/crawl_lists.py b/src/website_profiling/tools/audit_tools/crawl_lists.py index ed7a02fd..d14c2a0f 100644 --- a/src/website_profiling/tools/audit_tools/crawl_lists.py +++ b/src/website_profiling/tools/audit_tools/crawl_lists.py @@ -7,7 +7,7 @@ from psycopg import Connection from ...reporting.categories import REDIRECT_CHAIN_LONG -from ._slice import cap_list, parse_limit +from ._slice import _parse_page_analysis, cap_list, parse_limit from .context import AuditToolContext _REDIRECT_CHAIN_MIN = REDIRECT_CHAIN_LONG @@ -279,3 +279,269 @@ def get_top_pages_by_pagerank(conn: Connection, ctx: AuditToolContext, args: dic ranked.sort(key=lambda x: float(x.get("pagerank") or 0), reverse=True) sliced = cap_list(ranked, limit, max_cap=50) return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +_SOFT_404_MARKERS = ("not found", "404", "page not found", "doesn't exist", "does not exist") + + +def list_pages_soft_404(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _soft(r: dict[str, Any]) -> bool: + title = str(r.get("title") or "").lower() + return any(m in title for m in _SOFT_404_MARKERS) + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_soft, + projection=lambda r: { + "url": str(r.get("url") or ""), + "title": str(r.get("title") or ""), + "status": str(r.get("status") or ""), + }, + ) + + +def list_pages_with_axe_violations(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _has_axe(r: dict[str, Any]) -> bool: + pa = _parse_page_analysis(r) + axe = pa.get("axe_violations") + return isinstance(axe, list) and len(axe) > 0 + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_has_axe, + projection=lambda r: { + "url": str(r.get("url") or ""), + "violation_count": len(_parse_page_analysis(r).get("axe_violations") or []), + "title": str(r.get("title") or ""), + }, + ) + + +def get_axe_audit_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages_with_violations": 0, "violations_by_rule": [], "total_violations": 0} + rule_counts: dict[str, int] = {} + pages = 0 + total = 0 + for _, row in df.iterrows(): + pa = _parse_page_analysis(row.to_dict()) + axe = pa.get("axe_violations") + if not isinstance(axe, list) or not axe: + continue + pages += 1 + for v in axe: + if not isinstance(v, dict): + continue + rule = str(v.get("id") or v.get("description") or "unknown") + rule_counts[rule] = rule_counts.get(rule, 0) + 1 + total += 1 + ranked = sorted( + [{"rule_id": k, "count": v} for k, v in rule_counts.items()], + key=lambda x: -x["count"], + )[:30] + return { + "pages_with_violations": pages, + "total_violations": total, + "violations_by_rule": ranked, + "provenance": "Crawl", + } + + +def list_pages_with_mixed_content(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _mixed(r: dict[str, Any]) -> bool: + try: + return int(r.get("mixed_content_count") or 0) > 0 + except (TypeError, ValueError): + return False + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_mixed, + projection=lambda r: { + "url": str(r.get("url") or ""), + "mixed_content_count": int(r.get("mixed_content_count") or 0), + "title": str(r.get("title") or ""), + }, + ) + + +def list_dead_end_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + orphan_set = set() + if payload: + for u in payload.get("orphan_urls") or []: + if u: + orphan_set.add(_norm_url(str(u))) + inlink_by_url: dict[str, int] = {} + if payload: + for rec in payload.get("top_pages") or payload.get("links") or []: + if isinstance(rec, dict) and rec.get("url"): + inlink_by_url[_norm_url(str(rec["url"]))] = int(rec.get("inlinks") or 0) + + def _dead_end(r: dict[str, Any]) -> bool: + url = _norm_url(str(r.get("url") or "")) + if not url or url in orphan_set: + return False + try: + outlinks = int(r.get("outlinks") or 0) + except (TypeError, ValueError): + outlinks = 0 + if outlinks > 0: + return False + inlinks = inlink_by_url.get(url, 0) + return inlinks > 0 + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_dead_end, + projection=lambda r: { + "url": str(r.get("url") or ""), + "outlinks": int(r.get("outlinks") or 0), + "title": str(r.get("title") or ""), + }, + ) + + +def list_duplicate_title_groups(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"groups": [], "total": 0, "truncated": False} + work = _success_df(df) + buckets: dict[str, list[str]] = {} + for _, row in work.iterrows(): + title = str(row.get("title") or "").strip() + meta = str(row.get("meta_description") or "").strip() + if not title: + continue + key = f"{title}|{meta}" + url = str(row.get("url") or "") + if url: + buckets.setdefault(key, []).append(url) + groups = [ + {"title": k.split("|", 1)[0], "meta_description": k.split("|", 1)[1] if "|" in k else "", "urls": urls, "count": len(urls)} + for k, urls in buckets.items() + if len(urls) > 1 + ] + groups.sort(key=lambda g: -g["count"]) + limit = parse_limit(args.get("limit"), 20, 50) + sliced = cap_list(groups, limit, max_cap=50) + return {"groups": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Crawl"} + + +def list_heavy_pages_by_bytes(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False} + pages: list[dict[str, Any]] = [] + for _, row in _success_df(df).iterrows(): + rec = row.to_dict() + try: + js = int(rec.get("total_js_bytes") or 0) + css = int(rec.get("total_css_bytes") or 0) + scripts = int(rec.get("script_count") or 0) + except (TypeError, ValueError): + continue + total_bytes = js + css + if total_bytes <= 0 and scripts <= 0: + continue + pages.append({ + "url": str(rec.get("url") or ""), + "total_js_bytes": js, + "total_css_bytes": css, + "script_count": scripts, + "total_asset_bytes": total_bytes, + "title": str(rec.get("title") or ""), + }) + pages.sort(key=lambda p: -int(p.get("total_asset_bytes") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Crawl"} + + +def list_pages_poor_cache_headers(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _poor_cache(r: dict[str, Any]) -> bool: + cache = str(r.get("cache_control") or "").strip().lower() + etag = str(r.get("etag") or "").strip() + if not cache and not etag: + return True + if cache in ("no-cache", "no-store") and not etag: + return True + return False + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_poor_cache, + projection=lambda r: { + "url": str(r.get("url") or ""), + "cache_control": str(r.get("cache_control") or ""), + "etag": str(r.get("etag") or ""), + "title": str(r.get("title") or ""), + }, + ) + + +def list_pages_low_content_ratio(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + try: + threshold = float(args.get("max_content_html_ratio", 15)) + except (TypeError, ValueError): + threshold = 15.0 + + def _low_ratio(r: dict[str, Any]) -> bool: + try: + ratio = float(r.get("content_html_ratio") or 0) + except (TypeError, ValueError): + return False + return 0 < ratio < threshold + + return _filter_crawl_pages( + conn, + ctx, + args, + predicate=_low_ratio, + projection=lambda r: { + "url": str(r.get("url") or ""), + "content_html_ratio": r.get("content_html_ratio"), + "word_count": r.get("word_count"), + "title": str(r.get("title") or ""), + }, + ) + + +def get_heading_outline_for_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required"} + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"error": "no crawl data", "url": url} + needle = url.rstrip("/").lower() + for _, row in df.iterrows(): + row_url = str(row.get("url") or "").rstrip("/").lower() + if row_url != needle: + continue + rec = row.to_dict() + pa = _parse_page_analysis(rec) + return { + "url": str(rec.get("url") or ""), + "heading_sequence": str(rec.get("heading_sequence") or ""), + "heading_text": str(rec.get("heading_text") or ""), + "headings": pa.get("headings") if isinstance(pa.get("headings"), list) else [], + "provenance": "Crawl", + } + return {"error": "url not found in crawl", "url": url} diff --git a/src/website_profiling/tools/audit_tools/crawl_metrics.py b/src/website_profiling/tools/audit_tools/crawl_metrics.py new file mode 100644 index 00000000..50ee55db --- /dev/null +++ b/src/website_profiling/tools/audit_tools/crawl_metrics.py @@ -0,0 +1,81 @@ +"""Crawl aggregate metrics: asset weight and readability.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from .context import AuditToolContext + + +def _percentile(values: list[float], pct: float) -> float | None: + if not values: + return None + ordered = sorted(values) + idx = int(round((pct / 100.0) * (len(ordered) - 1))) + idx = max(0, min(idx, len(ordered) - 1)) + return round(ordered[idx], 1) + + +def get_asset_weight_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"missing": True, "note": "no crawl data"} + js_vals: list[float] = [] + css_vals: list[float] = [] + script_vals: list[float] = [] + for _, row in df.iterrows(): + if not str(row.get("status") or "").startswith("2"): + continue + for col, bucket in (("total_js_bytes", js_vals), ("total_css_bytes", css_vals), ("script_count", script_vals)): + if col not in df.columns: + continue + try: + v = float(row.get(col) or 0) + except (TypeError, ValueError): + continue + if v > 0: + bucket.append(v) + return { + "js_bytes": {"p50": _percentile(js_vals, 50), "p95": _percentile(js_vals, 95), "count": len(js_vals)}, + "css_bytes": {"p50": _percentile(css_vals, 50), "p95": _percentile(css_vals, 95), "count": len(css_vals)}, + "script_count": {"p50": _percentile(script_vals, 50), "p95": _percentile(script_vals, 95), "count": len(script_vals)}, + "provenance": "Crawl", + } + + +def get_readability_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty or "reading_level" not in df.columns: + return {"missing": True, "note": "reading_level not in crawl data"} + levels: list[float] = [] + buckets: dict[str, int] = {"0-6": 0, "7-9": 0, "10-12": 0, "13+": 0} + for _, row in df.iterrows(): + if not str(row.get("status") or "").startswith("2"): + continue + try: + lvl = float(row.get("reading_level") or 0) + except (TypeError, ValueError): + continue + if lvl <= 0: + continue + levels.append(lvl) + if lvl <= 6: + buckets["0-6"] += 1 + elif lvl <= 9: + buckets["7-9"] += 1 + elif lvl <= 12: + buckets["10-12"] += 1 + else: + buckets["13+"] += 1 + mean = round(sum(levels) / len(levels), 1) if levels else None + median = _percentile(levels, 50) if levels else None + return { + "mean_reading_level": mean, + "median_reading_level": median, + "histogram": buckets, + "pages_with_reading_level": len(levels), + "provenance": "Crawl", + } diff --git a/src/website_profiling/tools/audit_tools/geo_tools.py b/src/website_profiling/tools/audit_tools/geo_tools.py new file mode 100644 index 00000000..2c4a729e --- /dev/null +++ b/src/website_profiling/tools/audit_tools/geo_tools.py @@ -0,0 +1,349 @@ +"""GEO/AEO readiness tools: llms.txt, FAQ schema, citation signals, internal link suggestions.""" +from __future__ import annotations + +import math +import re +from collections import Counter +from typing import Any +from urllib.parse import urljoin, urlparse + +import requests +from psycopg import Connection + +from ._slice import _parse_page_analysis, _row_schema_types_list, cap_list, parse_limit +from .context import AuditToolContext + +_FAQ_TYPES = frozenset({"faqpage", "qapage", "question"}) +_QA_URL_HINTS = ("/faq", "/faqs", "/help", "/support", "/questions") + + +def _fetch_llms_txt(domain: str) -> dict[str, Any]: + if not domain: + return {"found": False, "error": "domain unknown"} + base = f"https://{domain.lstrip('https://').lstrip('http://').split('/')[0]}" + paths = ("/llms.txt", "/.well-known/llms.txt") + for path in paths: + url = urljoin(base + "/", path.lstrip("/")) + try: + resp = requests.get(url, timeout=8, headers={"User-Agent": "SiteAudit/1.0"}) + if resp.status_code == 200 and resp.text.strip(): + return { + "found": True, + "url": url, + "status_code": resp.status_code, + "size_bytes": len(resp.content), + "preview": resp.text.strip()[:500], + } + except requests.RequestException: + continue + return {"found": False, "checked_urls": [urljoin(base, p) for p in paths]} + + +def get_llms_txt_status(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + domain = scoped.resolve_property_domain(conn) + result = _fetch_llms_txt(domain) + result["domain"] = domain + result["provenance"] = "Crawl" + return result + + +def _has_faq_schema(row: dict[str, Any]) -> bool: + types = [t.lower() for t in _row_schema_types_list(row)] + return any(t in _FAQ_TYPES or "faq" in t for t in types) + + +def get_faq_schema_coverage(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages_with_faq_schema": 0, "total_2xx": 0, "coverage_pct": 0} + total = 0 + with_faq = 0 + for _, row in df.iterrows(): + if not str(row.get("status") or "").startswith("2"): + continue + total += 1 + if _has_faq_schema(row.to_dict()): + with_faq += 1 + pct = round(with_faq / total * 100, 1) if total else 0 + return { + "pages_with_faq_schema": with_faq, + "total_2xx": total, + "coverage_pct": pct, + "provenance": "Crawl", + } + + +def list_pages_missing_faq_schema(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False} + pages: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + url = str(rec.get("url") or "").lower() + heading = str(rec.get("heading_text") or rec.get("h1") or "").lower() + looks_qa = any(h in url for h in _QA_URL_HINTS) or "faq" in heading or "?" in heading + if not looks_qa or _has_faq_schema(rec): + continue + pages.append({"url": str(rec.get("url") or ""), "title": str(rec.get("title") or ""), "reason": "qa_heuristic_no_faq_schema"}) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Estimated"} + + +def get_geo_readiness_score(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + df = scoped.load_crawl_df(conn) + components: dict[str, float] = {} + total_2xx = 0 + schema_pages = 0 + good_word_count = 0 + good_headings = 0 + if df is not None and not df.empty: + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + total_2xx += 1 + if _row_schema_types_list(rec) or str(rec.get("has_schema") or "").lower() in ("true", "1", "yes"): + schema_pages += 1 + try: + wc = int(rec.get("word_count") or 0) + except (TypeError, ValueError): + wc = 0 + if wc >= 300: + good_word_count += 1 + seq = str(rec.get("heading_sequence") or "") + if seq and "h1" in seq.lower() and "h2" in seq.lower(): + good_headings += 1 + if total_2xx: + components["schema_coverage"] = round(schema_pages / total_2xx * 100, 1) + components["substantive_content"] = round(good_word_count / total_2xx * 100, 1) + components["heading_structure"] = round(good_headings / total_2xx * 100, 1) + else: + components["schema_coverage"] = 0 + components["substantive_content"] = 0 + components["heading_structure"] = 0 + faq = get_faq_schema_coverage(conn, scoped, args) + components["faq_schema_coverage"] = float(faq.get("coverage_pct") or 0) + ner = payload.get("ner_site_summary") if isinstance(payload.get("ner_site_summary"), dict) else {} + entities = ner.get("entities") or ner.get("top_entities") or [] + entity_count = len(entities) if isinstance(entities, list) else 0 + components["entity_richness"] = min(100.0, entity_count * 5.0) + llms = _fetch_llms_txt(scoped.resolve_property_domain(conn)) + components["llms_txt_present"] = 100.0 if llms.get("found") else 0.0 + score = round( + components["schema_coverage"] * 0.2 + + components["substantive_content"] * 0.2 + + components["heading_structure"] * 0.15 + + components["faq_schema_coverage"] * 0.15 + + components["entity_richness"] * 0.15 + + components["llms_txt_present"] * 0.15, + 1, + ) + return { + "geo_readiness_score": score, + "components": components, + "provenance": "Estimated", + } + + +def get_aeo_content_signals_for_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required"} + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"error": "no crawl data", "url": url} + needle = url.rstrip("/").lower() + for _, row in df.iterrows(): + if str(row.get("url") or "").rstrip("/").lower() != needle: + continue + rec = row.to_dict() + excerpt = str(rec.get("content_excerpt") or "") + words = excerpt.split() + lead = " ".join(words[:80]) + has_list = bool(re.search(r"^\s*[-*•]\s", excerpt, re.M)) or "

  • " in str(rec.get("html") or "").lower() + has_definition = bool(re.search(r"\b(is|are|means|refers to)\b", lead[:400], re.I)) + try: + wc = int(rec.get("word_count") or 0) + except (TypeError, ValueError): + wc = 0 + entities = rec.get("top_keywords") + if isinstance(entities, str): + entities = [entities] + entity_mentions = len(entities) if isinstance(entities, list) else 0 + quotability = 0 + if wc >= 200: + quotability += 25 + if has_list: + quotability += 20 + if has_definition: + quotability += 25 + if _has_faq_schema(rec): + quotability += 30 + return { + "url": str(rec.get("url") or ""), + "word_count": wc, + "lead_excerpt": lead[:300], + "has_lists": has_list, + "has_definition_pattern": has_definition, + "entity_keyword_count": entity_mentions, + "quotability_score": min(100, quotability), + "provenance": "Estimated", + } + return {"error": "url not found in crawl", "url": url} + + +def get_eeat_signals_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"missing": True} + author_pages = 0 + org_schema = 0 + about_contact = 0 + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + types = [t.lower() for t in _row_schema_types_list(rec)] + if any(t in ("person", "author") for t in types): + author_pages += 1 + if any(t in ("organization", "localbusiness", "corporation") for t in types): + org_schema += 1 + path = urlparse(str(rec.get("url") or "")).path.lower() + if any(p in path for p in ("/about", "/contact", "/team", "/author")): + about_contact += 1 + return { + "pages_with_author_schema": author_pages, + "pages_with_organization_schema": org_schema, + "about_contact_pages": about_contact, + "provenance": "Crawl", + } + + +def get_js_rendering_delta(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty or "fetch_method" not in df.columns: + return {"deltas": [], "total": 0, "note": "fetch_method not in crawl — use javascript or auto render mode"} + by_url: dict[str, dict[str, dict[str, Any]]] = {} + for _, row in df.iterrows(): + url = str(row.get("url") or "").rstrip("/").lower() + method = str(row.get("fetch_method") or "static").lower() + if not url: + continue + try: + word_count = int(row.get("word_count") or 0) + except (TypeError, ValueError): + word_count = 0 + by_url.setdefault(url, {})[method] = { + "title": str(row.get("title") or ""), + "word_count": word_count, + "h1": str(row.get("h1") or ""), + } + deltas: list[dict[str, Any]] = [] + for url, methods in by_url.items(): + static = methods.get("static") + rendered = methods.get("rendered") or methods.get("javascript") + if not static or not rendered: + continue + title_diff = static.get("title") != rendered.get("title") + wc_diff = abs(int(static.get("word_count") or 0) - int(rendered.get("word_count") or 0)) + h1_diff = static.get("h1") != rendered.get("h1") + if title_diff or wc_diff > 50 or h1_diff: + deltas.append({ + "url": url, + "static": static, + "rendered": rendered, + "title_differs": title_diff, + "word_count_delta": wc_diff, + "h1_differs": h1_diff, + }) + deltas.sort(key=lambda d: -int(d.get("word_count_delta") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(deltas, limit, max_cap=50) + return {"deltas": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Crawl"} + + +def _tokenize(text: str) -> list[str]: + return [w.lower() for w in re.findall(r"[a-z0-9]{3,}", text)] + + +def get_internal_link_suggestions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """TF-IDF related pages with suggested anchor hints for a source URL.""" + scoped = ctx.with_args(args) + source_url = str(args.get("url") or "").strip() + if not source_url: + return {"error": "url is required"} + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"error": "no crawl data", "suggestions": []} + docs: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + url = str(rec.get("url") or "") + text = " ".join([ + str(rec.get("title") or ""), + str(rec.get("h1") or ""), + str(rec.get("content_excerpt") or ""), + ]) + tokens = _tokenize(text) + if not tokens: + continue + docs.append({"url": url, "tokens": tokens, "title": str(rec.get("title") or "")}) + if len(docs) < 2: + return {"url": source_url, "suggestions": [], "note": "insufficient crawl pages"} + source_doc = next((d for d in docs if d["url"].rstrip("/").lower() == source_url.rstrip("/").lower()), None) + if not source_doc: + return {"error": "source url not in crawl", "url": source_url} + df_counts = len(docs) + idf: dict[str, float] = {} + doc_freq: Counter[str] = Counter() + for d in docs: + for t in set(d["tokens"]): + doc_freq[t] += 1 + for t, c in doc_freq.items(): + idf[t] = math.log((1 + df_counts) / (1 + c)) + 1 + source_tf = Counter(source_doc["tokens"]) + source_vec = {t: (source_tf[t] / len(source_doc["tokens"])) * idf.get(t, 1) for t in source_tf} + source_norm = math.sqrt(sum(v * v for v in source_vec.values())) or 1 + scored: list[dict[str, Any]] = [] + for d in docs: + if d["url"].rstrip("/").lower() == source_url.rstrip("/").lower(): + continue + target_tf = Counter(d["tokens"]) + target_vec = {t: (target_tf[t] / len(d["tokens"])) * idf.get(t, 1) for t in target_tf} + dot = sum(source_vec.get(t, 0) * target_vec.get(t, 0) for t in set(source_vec) | set(target_vec)) + target_norm = math.sqrt(sum(v * v for v in target_vec.values())) or 1 + sim = dot / (source_norm * target_norm) + if sim <= 0.05: + continue + shared = sorted(set(source_doc["tokens"]) & set(d["tokens"]), key=lambda t: -idf.get(t, 0))[:3] + anchor_hint = d["title"] or (shared[0] if shared else "related page") + scored.append({ + "target_url": d["url"], + "similarity": round(sim, 4), + "suggested_anchor": anchor_hint[:80], + "shared_terms": shared, + }) + scored.sort(key=lambda x: -float(x.get("similarity") or 0)) + limit = parse_limit(args.get("limit"), 5, 10) + sliced = cap_list(scored, limit, max_cap=10) + return { + "url": source_url, + "suggestions": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "provenance": "Estimated", + } diff --git a/src/website_profiling/tools/audit_tools/google.py b/src/website_profiling/tools/audit_tools/google.py index f97abbe0..14184fec 100644 --- a/src/website_profiling/tools/audit_tools/google.py +++ b/src/website_profiling/tools/audit_tools/google.py @@ -135,3 +135,55 @@ def _ga4_path_key(raw: str) -> str: if ga4_slice: return {"path": path, "metrics": ga4_slice, "fetched_at": data.get("fetched_at")} return {"error": "path not found in GA4 top pages", "path": path, "missing": True} + + +def get_gsc_ctr_opportunity_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Pages with high impressions but CTR below industry curve at their position.""" + from ...integrations.google.keyword_enrich import ctr_as_fraction, industry_ctr + + scoped = ctx.with_args(args) + data = scoped.load_google(conn) + if not data: + return {"error": "no google data found", "pages": [], "total": 0, "truncated": False} + gsc = data.get("gsc") if isinstance(data.get("gsc"), dict) else {} + pages = gsc.get("pages") or gsc.get("top_pages") or [] + if not isinstance(pages, list): + pages = [] + try: + min_impressions = int(args.get("min_impressions", 100)) + except (TypeError, ValueError): + min_impressions = 100 + opportunities: list[dict[str, Any]] = [] + for row in pages: + if not isinstance(row, dict): + continue + impressions = int(row.get("impressions") or 0) + if impressions < min_impressions: + continue + try: + pos = float(row.get("position") or 0) + except (TypeError, ValueError): + pos = 0 + if pos <= 0: + continue + ctr_frac = ctr_as_fraction(row.get("ctr")) + expected = industry_ctr(pos) + if ctr_frac > 0 and ctr_frac < expected * 0.7: + opportunities.append({ + "page": row.get("page") or row.get("url"), + "clicks": row.get("clicks"), + "impressions": impressions, + "ctr": row.get("ctr"), + "position": pos, + "expected_ctr_fraction": round(expected, 4), + "opportunity": "improve CTR (title/description)", + }) + opportunities.sort(key=lambda p: -int(p.get("impressions") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(opportunities, limit, max_cap=50) + return { + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "provenance": "Search Console", + } diff --git a/src/website_profiling/tools/audit_tools/integration_tools.py b/src/website_profiling/tools/audit_tools/integration_tools.py new file mode 100644 index 00000000..ab52f4cb --- /dev/null +++ b/src/website_profiling/tools/audit_tools/integration_tools.py @@ -0,0 +1,143 @@ +"""Third-party integration audit tools: GSC inspection, Bing, SERP, AI citation.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...db.property_store import get_property_by_id +from ...integrations.google.auth import build_credentials +from ...integrations.google.gsc_inspection import inspect_url +from ._slice import cap_list, parse_limit +from .context import AuditToolContext + + +def _property_google_config(conn: Connection, property_id: int | None) -> tuple[dict[str, Any] | None, Any, str]: + if property_id is None: + return None, None, "property_id is required" + prop = get_property_by_id(conn, property_id) + if not prop: + return None, None, "property not found" + if not prop.get("google_refresh_token"): + return None, None, "Google not connected — configure OAuth in Integrations" + try: + creds = build_credentials(property_id=property_id) + except Exception as e: + return None, None, f"Google credentials error: {e}" + if not creds: + return None, None, "Google not connected — configure OAuth in Integrations" + gsc_site = str(prop.get("gsc_site_url") or prop.get("canonical_domain") or "").strip() + if not gsc_site: + return None, None, "GSC site URL not configured for property" + return prop, creds, gsc_site + + +def get_gsc_url_inspection(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required", "missing": True} + prop, creds, gsc_site = _property_google_config(conn, scoped.property_id) + if prop is None or creds is None: + return {"error": gsc_site, "missing": True} + return inspect_url(creds, gsc_site, url) + + +def get_gsc_index_coverage(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Estimated indexation coverage from crawl + sitemap + GSC URL sets.""" + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "missing": True} + cov = payload.get("indexation_coverage") + if not isinstance(cov, dict): + return {"error": "indexation_coverage not in report", "missing": True} + counts = cov.get("counts") if isinstance(cov.get("counts"), dict) else {} + lists_total = cov.get("lists_total") if isinstance(cov.get("lists_total"), dict) else {} + return { + "counts": counts, + "gap_totals": lists_total, + "note": "Estimated from crawl, sitemap, and GSC URL join — not live Inspection API", + "provenance": "Estimated", + } + + +def get_bing_index_status(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required", "missing": True} + if scoped.property_id is None: + return {"error": "property_id is required", "missing": True} + prop = get_property_by_id(conn, scoped.property_id) + if not prop: + return {"error": "property not found", "missing": True} + from ...db.config_store import read_pipeline_config + + known, _ = read_pipeline_config(conn) + api_key = str(known.get("bing_webmaster_api_key") or "").strip() + site_url = str(prop.get("gsc_site_url") or prop.get("canonical_domain") or "").strip() + if not api_key: + return {"error": "bing_webmaster_api_key not configured in audit settings", "missing": True} + from ...integrations.bing.webmaster import _bing_json_get + + raw = _bing_json_get("GetUrlInfo", api_key, siteUrl=site_url, url=url) + if raw.get("error"): + return {"error": str(raw.get("error")), "missing": True, "provenance": "Bing Webmaster"} + data = raw.get("d") if isinstance(raw.get("d"), dict) else raw + return {"url": url, "bing": data, "provenance": "Bing Webmaster"} + + +def get_serp_feature_overlay(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required", "keywords": [], "missing": True} + data = scoped.load_keywords(conn) + if not data: + return {"error": "no keyword data found", "keywords": [], "missing": True} + rows = data.get("rows") or [] + with_features = [ + r for r in rows + if isinstance(r, dict) and ( + r.get("serp_features") is not None + or r.get("serp_estimated_competition") is not None + ) + ] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(with_features, limit, max_cap=50) + return { + "keywords": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "serp_overlay_count": data.get("serp_overlay_count"), + "provenance": "Estimated", + } + + +def check_ai_citation_presence(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Heuristic brand/domain citation readiness from on-site signals (no live LLM API).""" + scoped = ctx.with_args(args) + query = str(args.get("query") or args.get("brand") or "").strip() + domain = scoped.resolve_property_domain(conn) + if not query and not domain: + return {"error": "query or brand is required", "missing": True} + payload = scoped.load_payload(conn) + brand_hits = 0 + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + for iss in cat.get("issues") or []: + if isinstance(iss, dict) and query.lower() in str(iss.get("message") or "").lower(): + brand_hits += 1 + ner = payload.get("ner_site_summary") if isinstance(payload.get("ner_site_summary"), dict) else {} + entities = ner.get("entities") or [] + entity_match = query.lower() in [str(e).lower() for e in entities] if query else False + schema_cov = payload.get("schema_coverage") if isinstance(payload.get("schema_coverage"), dict) else {} + return { + "query": query or domain, + "domain": domain, + "entity_in_ner_summary": entity_match, + "schema_pages": schema_cov.get("pages_with_schema"), + "citation_readiness_note": "Live AI citation check requires external API — this is an on-site signal estimate", + "provenance": "Estimated", + } diff --git a/src/website_profiling/tools/audit_tools/keywords.py b/src/website_profiling/tools/audit_tools/keywords.py index b59f8fe1..79b9a8ae 100644 --- a/src/website_profiling/tools/audit_tools/keywords.py +++ b/src/website_profiling/tools/audit_tools/keywords.py @@ -201,6 +201,14 @@ def list_keywords_by_action(conn: Connection, ctx: AuditToolContext, args: dict[ ) +def list_keywords_ctr_opportunity(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Keywords flagged for CTR improvement (title/meta snippet optimization).""" + return _filter_keyword_rows( + conn, ctx, args, + lambda r: "improve ctr" in str(r.get("recommended_action") or "").lower(), + ) + + def list_keywords_by_position(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: min_pos = args.get("min_position") max_pos = args.get("max_position") diff --git a/src/website_profiling/tools/audit_tools/llm_tools.py b/src/website_profiling/tools/audit_tools/llm_tools.py index a635eaab..ae966dd8 100644 --- a/src/website_profiling/tools/audit_tools/llm_tools.py +++ b/src/website_profiling/tools/audit_tools/llm_tools.py @@ -128,3 +128,208 @@ def expand_keywords(conn: Connection, ctx: AuditToolContext, args: dict[str, Any "expansions": expanded, "seed_count": len(seeds), } + + +def _llm_disabled_response() -> dict[str, Any]: + from ...llm_config import load_llm_config_from_db, llm_is_enabled + + cfg = load_llm_config_from_db() + if not llm_is_enabled(cfg): + return {"error": "AI insights are disabled — enable LLM in audit settings", "missing": True} + return {} + + +def generate_issue_fix(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + from ...llm.issue_fixes import generate_issue_fix_suggestion + from ...llm_config import load_llm_config_from_db + + err = _llm_disabled_response() + if err: + return err + message = str(args.get("message") or "").strip() + if not message: + return {"error": "message is required (issue message to fix)"} + refresh = str(args.get("refresh") or "").lower() in ("true", "1", "yes") + issue = { + "message": message, + "url": args.get("url"), + "priority": args.get("priority"), + "category": args.get("category_id") or args.get("category"), + "recommendation": args.get("recommendation"), + } + result = generate_issue_fix_suggestion(issue, cfg=load_llm_config_from_db(), refresh=refresh) + result["provenance"] = "AI insights" + return result + + +def summarize_category_for_client(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + from .issues import get_category_issues + + category_id = str(args.get("category_id") or "").strip() + if not category_id: + return {"error": "category_id is required"} + data = get_category_issues(conn, ctx, {**args, "category_id": category_id}) + if data.get("error"): + return data + issues = data.get("issues") or [] + top = issues[:5] + bullets = [ + f"[{i.get('priority')}] {i.get('message')}" + (f" ({i.get('url')})" if i.get("url") else "") + for i in top + if isinstance(i, dict) + ] + summary = { + "category_id": category_id, + "category_name": data.get("name"), + "score": data.get("score"), + "issue_count": len(issues), + "headline": f"{data.get('name') or category_id}: {len(issues)} issue(s), score {data.get('score')}", + "top_issues": bullets, + } + err = _llm_disabled_response() + if not err: + from ...llm.base import get_llm_client, parse_json_response + from ...llm_config import load_llm_config_from_db + + cfg = load_llm_config_from_db() + try: + client = get_llm_client(cfg) + user = ( + "Write a 2-3 sentence client-friendly summary of this audit category. " + f"Return JSON with key summary. Data: {json.dumps(summary, default=str)[:3000]}" + ) + raw = client.complete_json("You are a technical SEO consultant writing for clients.", user) + if isinstance(raw, dict) and raw.get("summary"): + summary["narrative"] = raw["summary"] + else: + summary["narrative"] = str(raw.get("summary") or parse_json_response(str(raw)).get("summary") or "") + except Exception as e: + summary["narrative_error"] = str(e) + summary["provenance"] = "AI insights" if summary.get("narrative") else "Crawl" + return summary + + +def prioritize_fix_roadmap(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + from ...reporting.issue_impact import sort_issues_by_impact + from .report import _iter_category_issues + + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "roadmap": []} + issues = sort_issues_by_impact(_iter_category_issues(payload)) + try: + top_n = int(args.get("limit", 15)) + except (TypeError, ValueError): + top_n = 15 + top_n = max(1, min(top_n, 30)) + roadmap = [ + { + "rank": i + 1, + "priority": iss.get("priority"), + "impact_score": iss.get("impact_score"), + "message": iss.get("message"), + "url": iss.get("url"), + "category": iss.get("category"), + "gsc_clicks": iss.get("gsc_clicks"), + "ga4_sessions": iss.get("ga4_sessions"), + } + for i, iss in enumerate(issues[:top_n]) + ] + return {"roadmap": roadmap, "total_issues": len(issues), "provenance": "Crawl"} + + +def analyze_serp_snippet_for_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + from ...integrations.google.page_lookup import slice_from_google_row + + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required"} + data = scoped.load_google(conn) + gsc_slice = slice_from_google_row(data, url) if data else {} + scoped_df = scoped.load_crawl_df(conn) + page_row: dict[str, Any] = {} + if scoped_df is not None and not scoped_df.empty: + needle = url.rstrip("/").lower() + for _, row in scoped_df.iterrows(): + if str(row.get("url") or "").rstrip("/").lower() == needle: + page_row = row.to_dict() + break + base = { + "url": url, + "current_title": page_row.get("title"), + "current_meta_description": page_row.get("meta_description"), + "gsc_queries": (gsc_slice.get("gsc") or {}).get("queries") if isinstance(gsc_slice, dict) else None, + "gsc_metrics": (gsc_slice.get("gsc") or {}).get("page_metrics") if isinstance(gsc_slice, dict) else None, + } + err = _llm_disabled_response() + if err: + base["note"] = err.get("error") + base["provenance"] = "Crawl" + return base + from ...llm.base import get_llm_client + from ...llm_config import load_llm_config_from_db + + cfg = load_llm_config_from_db() + client = get_llm_client(cfg) + if not client: + base["provenance"] = "Crawl" + return base + prompt = ( + "Suggest improved title and meta description for better CTR. " + f"Context: {json.dumps(base, default=str)[:2500]}" + ) + try: + suggestions = client.complete_json( + "You are an SEO copywriter. Return JSON with title, meta_description, rationale.", + prompt, + ) + base["suggestions"] = suggestions if isinstance(suggestions, dict) else {} + base["provenance"] = "AI insights" + except Exception as e: + base["error"] = str(e) + base["provenance"] = "Crawl" + return base + + +def draft_llms_txt(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found"} + site_name = str(payload.get("site_name") or scoped.resolve_property_domain(conn) or "Site") + top_pages = (payload.get("top_pages") or payload.get("links") or [])[:10] + page_urls = [str(p.get("url")) for p in top_pages if isinstance(p, dict) and p.get("url")] + schema_cov = payload.get("schema_coverage") if isinstance(payload.get("schema_coverage"), dict) else {} + draft_lines = [ + f"# {site_name}", + "", + "> LLM-oriented site index (draft — review before publishing)", + "", + "## Key pages", + *[f"- {u}" for u in page_urls], + "", + f"## Schema coverage: {schema_cov.get('pages_with_schema', 'n/a')} pages with structured data", + ] + err = _llm_disabled_response() + if not err: + from ...llm.base import get_llm_client + from ...llm_config import load_llm_config_from_db + + try: + client = get_llm_client(load_llm_config_from_db()) + raw = client.complete_json( + "You write concise llms.txt files per emerging conventions. Return JSON with key content.", + "Polish this llms.txt draft:\n" + "\n".join(draft_lines), + ) + content = raw.get("content") if isinstance(raw, dict) else None + if content and str(content).strip(): + draft_lines = str(content).strip().splitlines() + except Exception: + pass + return { + "site_name": site_name, + "llms_txt_draft": "\n".join(draft_lines), + "provenance": "AI insights", + } diff --git a/src/website_profiling/tools/audit_tools/payload_extras.py b/src/website_profiling/tools/audit_tools/payload_extras.py new file mode 100644 index 00000000..0b91d641 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/payload_extras.py @@ -0,0 +1,142 @@ +"""Report payload slices: rich results, portfolio benchmark, competitor gaps, anchors.""" +from __future__ import annotations + +from collections import Counter +from typing import Any + +from psycopg import Connection + +from ._slice import _parse_page_analysis, cap_list, parse_limit, payload_dict_slice +from .context import AuditToolContext + + +def get_rich_results_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "missing": True} + meta = payload.get("rich_results_meta") + if not isinstance(meta, dict): + return {"missing": True, "meta": None, "note": "rich_results_meta not in report — enable rich results validation on build"} + return {"meta": meta, "missing": False, "provenance": "Crawl / GSC / API"} + + +def list_rich_results_failures(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "failures": [], "total": 0, "truncated": False} + rows = payload.get("rich_results_validation") or [] + if not isinstance(rows, list): + rows = [] + failures = [ + r for r in rows + if isinstance(r, dict) and str(r.get("status") or "").lower() not in ("pass", "ok") + ] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(failures, limit, max_cap=50) + return { + "failures": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "provenance": "Crawl / GSC / API", + } + + +def get_competitor_keyword_gap(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "rows": [], "total": 0, "truncated": False} + rows = payload.get("competitor_keyword_gap") or [] + if not isinstance(rows, list): + rows = [] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(rows, limit, max_cap=50) + return { + "rows": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "provenance": "Estimated", + } + + +def get_portfolio_benchmark(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "missing": True} + result = payload_dict_slice(payload, "portfolio_benchmark") + if result.get("missing"): + return {"missing": True, "benchmark": None, "note": "portfolio_benchmark not in report"} + return {"benchmark": result.get("data"), "missing": False, "provenance": "Crawl"} + + +def get_site_anchor_text_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "anchors": [], "total": 0, "truncated": False} + matrix = payload.get("inlink_anchor_matrix") or [] + if not isinstance(matrix, list) or not matrix: + return { + "anchors": [], + "total": 0, + "truncated": False, + "missing": True, + "note": "inlink_anchor_matrix not in report — rebuild with link_edges", + } + counter: Counter[str] = Counter() + for row in matrix: + if not isinstance(row, dict): + continue + anchor = str(row.get("anchor_text") or "").strip() or "(empty)" + try: + count = int(row.get("inlink_count") or 0) + except (TypeError, ValueError): + count = 0 + counter[anchor] += count + ranked = [{"anchor_text": a, "inlink_count": c} for a, c in counter.most_common()] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(ranked, limit, max_cap=50) + return { + "anchors": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "provenance": "Crawl", + } + + +def get_pagination_audit_summary(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"orphan_prev_count": 0, "amp_mismatch_count": 0, "pages_with_rel_next": 0, "pages_with_rel_prev": 0} + orphan_prev = 0 + amp_mismatch = 0 + rel_next = 0 + rel_prev = 0 + for _, row in df.iterrows(): + if not str(row.get("status") or "").startswith("2"): + continue + pa = _parse_page_analysis(row.to_dict()) + pag = pa.get("pagination") if isinstance(pa.get("pagination"), dict) else {} + has_next = bool(pag.get("rel_next")) + has_prev = bool(pag.get("rel_prev")) + if has_next: + rel_next += 1 + if has_prev: + rel_prev += 1 + if has_prev and not has_next: + orphan_prev += 1 + amphtml = pag.get("amphtml") + canon = str(row.get("canonical_url") or "").strip() + if amphtml and canon and amphtml != canon: + amp_mismatch += 1 + return { + "orphan_prev_count": orphan_prev, + "amp_mismatch_count": amp_mismatch, + "pages_with_rel_next": rel_next, + "pages_with_rel_prev": rel_prev, + "provenance": "Crawl", + } diff --git a/src/website_profiling/tools/audit_tools/registry.py b/src/website_profiling/tools/audit_tools/registry.py index 07a6ce7c..faa8da99 100644 --- a/src/website_profiling/tools/audit_tools/registry.py +++ b/src/website_profiling/tools/audit_tools/registry.py @@ -32,9 +32,11 @@ compare_duplicate_deltas, compare_google_metrics, compare_health_score_delta, + compare_indexation_deltas, compare_issue_deltas, compare_lighthouse_deltas, compare_link_metric_deltas, + compare_orphan_deltas, compare_priority_counts, compare_redirect_deltas, compare_security_deltas, @@ -42,6 +44,7 @@ compare_tech_deltas, compare_url_set_diff, ) +from .crawl_metrics import get_asset_weight_summary, get_readability_summary from .content import ( get_content_analytics, get_content_duplicates, @@ -53,16 +56,36 @@ ) from .context import AuditToolContext from .crawl_lists import ( + get_axe_audit_summary, + get_heading_outline_for_url, get_top_pages_by_pagerank, list_canonical_mismatch, + list_dead_end_pages, + list_duplicate_title_groups, + list_heavy_pages_by_bytes, list_long_redirect_chains, + list_pages_low_content_ratio, list_pages_missing_canonical, list_pages_missing_og_image, list_pages_missing_viewport, + list_pages_poor_cache_headers, list_pages_skipped_headings, + list_pages_soft_404, + list_pages_with_axe_violations, list_pages_with_missing_alt, + list_pages_with_mixed_content, list_robots_blocked_urls, ) +from .geo_tools import ( + get_aeo_content_signals_for_url, + get_eeat_signals_summary, + get_faq_schema_coverage, + get_geo_readiness_score, + get_internal_link_suggestions, + get_js_rendering_delta, + get_llms_txt_status, + list_pages_missing_faq_schema, +) from .crawl import ( get_browser_diagnostics_summary, get_crawl_links_table, @@ -88,10 +111,18 @@ get_ga4_page_metrics, get_ga4_summary, get_google_summary, + get_gsc_ctr_opportunity_pages, get_gsc_page_query_slice, get_gsc_top_pages, get_gsc_top_queries, ) +from .integration_tools import ( + check_ai_citation_presence, + get_bing_index_status, + get_gsc_index_coverage, + get_gsc_url_inspection, + get_serp_feature_overlay, +) from .health import get_category_health_history, get_health_history, list_report_history from .indexation_tools import get_indexation_coverage, get_indexation_url_join, list_indexation_gaps from .international import get_hreflang_summary, get_language_summary @@ -107,6 +138,7 @@ list_keywords_by_action, list_keywords_by_impressions, list_keywords_by_position, + list_keywords_ctr_opportunity, search_keywords, ) from .lighthouse import ( @@ -141,10 +173,23 @@ list_unoptimized_images, ) from .llm_tools import ( + analyze_serp_snippet_for_url, + draft_llms_txt, expand_keywords, generate_content_brief, + generate_issue_fix, get_page_coach, get_portfolio_summary, + prioritize_fix_roadmap, + summarize_category_for_client, +) +from .payload_extras import ( + get_competitor_keyword_gap, + get_pagination_audit_summary, + get_portfolio_benchmark, + get_rich_results_summary, + get_site_anchor_text_summary, + list_rich_results_failures, ) from .onpage import ( list_content_url_issues, @@ -196,6 +241,7 @@ get_report_summary, get_site_level, list_issues, + list_top_impact_issues, search_issues, ) from .report_extras import ( @@ -399,6 +445,47 @@ "list_largest_images": list_largest_images, "list_unoptimized_images": list_unoptimized_images, "list_images_needing_attention": list_images_needing_attention, + "list_top_impact_issues": list_top_impact_issues, + "get_rich_results_summary": get_rich_results_summary, + "list_rich_results_failures": list_rich_results_failures, + "get_competitor_keyword_gap": get_competitor_keyword_gap, + "get_portfolio_benchmark": get_portfolio_benchmark, + "get_site_anchor_text_summary": get_site_anchor_text_summary, + "get_pagination_audit_summary": get_pagination_audit_summary, + "list_pages_soft_404": list_pages_soft_404, + "list_pages_with_axe_violations": list_pages_with_axe_violations, + "get_axe_audit_summary": get_axe_audit_summary, + "list_pages_with_mixed_content": list_pages_with_mixed_content, + "list_dead_end_pages": list_dead_end_pages, + "list_duplicate_title_groups": list_duplicate_title_groups, + "list_heavy_pages_by_bytes": list_heavy_pages_by_bytes, + "list_pages_poor_cache_headers": list_pages_poor_cache_headers, + "list_pages_low_content_ratio": list_pages_low_content_ratio, + "get_heading_outline_for_url": get_heading_outline_for_url, + "get_asset_weight_summary": get_asset_weight_summary, + "get_readability_summary": get_readability_summary, + "list_keywords_ctr_opportunity": list_keywords_ctr_opportunity, + "get_gsc_ctr_opportunity_pages": get_gsc_ctr_opportunity_pages, + "compare_indexation_deltas": compare_indexation_deltas, + "compare_orphan_deltas": compare_orphan_deltas, + "get_llms_txt_status": get_llms_txt_status, + "get_faq_schema_coverage": get_faq_schema_coverage, + "list_pages_missing_faq_schema": list_pages_missing_faq_schema, + "get_geo_readiness_score": get_geo_readiness_score, + "get_aeo_content_signals_for_url": get_aeo_content_signals_for_url, + "get_eeat_signals_summary": get_eeat_signals_summary, + "get_js_rendering_delta": get_js_rendering_delta, + "get_internal_link_suggestions": get_internal_link_suggestions, + "generate_issue_fix": generate_issue_fix, + "summarize_category_for_client": summarize_category_for_client, + "prioritize_fix_roadmap": prioritize_fix_roadmap, + "analyze_serp_snippet_for_url": analyze_serp_snippet_for_url, + "draft_llms_txt": draft_llms_txt, + "get_gsc_url_inspection": get_gsc_url_inspection, + "get_gsc_index_coverage": get_gsc_index_coverage, + "get_bing_index_status": get_bing_index_status, + "get_serp_feature_overlay": get_serp_feature_overlay, + "check_ai_citation_presence": check_ai_citation_presence, } diff --git a/src/website_profiling/tools/audit_tools/report.py b/src/website_profiling/tools/audit_tools/report.py index 20d908f2..d5c25b62 100644 --- a/src/website_profiling/tools/audit_tools/report.py +++ b/src/website_profiling/tools/audit_tools/report.py @@ -34,14 +34,18 @@ def _iter_category_issues(payload: dict[str, Any]) -> list[dict[str, Any]]: if not isinstance(issue, dict): continue rec = str(issue.get("llm_recommendation") or issue.get("recommendation") or "") - rows.append({ + row: dict[str, Any] = { "category_id": cat_id, "category": cat_name, "priority": str(issue.get("priority") or "Medium"), "message": str(issue.get("message") or ""), "url": str(issue.get("url") or ""), "recommendation": rec, - }) + } + for key in ("impact_score", "gsc_clicks", "gsc_impressions", "ga4_sessions"): + if issue.get(key) is not None: + row[key] = issue.get(key) + rows.append(row) rows.sort(key=lambda x: _PRIORITY_ORDER.get(x.get("priority", "Low"), 99)) return rows @@ -173,6 +177,12 @@ def search_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) or message_contains in str(i.get("recommendation") or "").lower() ] + sort_mode = str(args.get("sort") or "").strip().lower() + if sort_mode == "impact": + from ...reporting.issue_impact import sort_issues_by_impact + + issues = sort_issues_by_impact(issues) + total = len(issues) truncated = total > limit return { @@ -182,6 +192,11 @@ def search_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) } +def list_top_impact_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Issues ranked by traffic-weighted impact_score (GSC clicks + GA4 sessions + priority).""" + return list_issues(conn, ctx, {**args, "sort": "impact"}) + + def get_critical_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: """All Critical-priority audit issues (chat table visualization).""" return list_issues(conn, ctx, {**args, "priority": "Critical"}) diff --git a/src/website_profiling/tools/audit_tools/tool_catalog.py b/src/website_profiling/tools/audit_tools/tool_catalog.py index 52b1abec..f3f106f8 100644 --- a/src/website_profiling/tools/audit_tools/tool_catalog.py +++ b/src/website_profiling/tools/audit_tools/tool_catalog.py @@ -324,4 +324,58 @@ def _tool(name: str, description: str, properties: dict[str, Any], required: lis "Ranked images/pages with composite attention reasons (size, format, alt/lazy/dimension issues).", {"property_id": _PID, "report_id": _RID, "min_size_kb": {"type": "integer"}, "limit": {"type": "integer", "maximum": 100}}, ), + # Impact prioritization + _tool( + "list_top_impact_issues", + "Audit issues ranked by traffic-weighted impact_score (GSC clicks + GA4 sessions + priority).", + {"property_id": _PID, "report_id": _RID, "priority": {"type": "string"}, "category_id": {"type": "string"}, "limit": _LIMIT}, + ), + # Payload extras + _tool("get_rich_results_summary", "Rich Results validation meta counts from report build.", {"property_id": _PID, "report_id": _RID}), + _tool("list_rich_results_failures", "URLs failing rich-results validation (status != pass).", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_competitor_keyword_gap", "Competitor keyword gap rows from competitor_keyword_gap_json config.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_portfolio_benchmark", "Property health vs portfolio median benchmark.", {"property_id": _PID, "report_id": _RID}), + _tool("get_site_anchor_text_summary", "Top sitewide inlink anchor texts from inlink_anchor_matrix.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_pagination_audit_summary", "rel=prev/next and AMP pairing issue counts from crawl.", {"property_id": _PID, "report_id": _RID}), + # Crawl extras + _tool("list_pages_soft_404", "2xx pages whose title suggests not-found (soft 404).", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_with_axe_violations", "Pages with axe-core accessibility violations (enable_axe crawl).", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_axe_audit_summary", "Site-wide axe violation counts by rule id.", {"property_id": _PID, "report_id": _RID}), + _tool("list_pages_with_mixed_content", "HTTPS pages with mixed HTTP content references.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_dead_end_pages", "Crawlable pages with inlinks but zero outlinks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_duplicate_title_groups", "Groups of pages sharing the same title and meta description.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_heavy_pages_by_bytes", "Pages ranked by total JS + CSS bytes.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_poor_cache_headers", "Pages missing or weak cache-control / etag headers.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_low_content_ratio", "Pages with low content_html_ratio (bloated HTML).", {"property_id": _PID, "report_id": _RID, "max_content_html_ratio": {"type": "number"}, "limit": _LIMIT}), + _tool("get_heading_outline_for_url", "Heading sequence and outline for one URL.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]), + # Crawl metrics + _tool("get_asset_weight_summary", "JS/CSS bytes and script_count percentiles from crawl.", {"property_id": _PID, "report_id": _RID}), + _tool("get_readability_summary", "Reading level histogram and mean/median from crawl.", {"property_id": _PID, "report_id": _RID}), + # CTR opportunities + _tool("list_keywords_ctr_opportunity", "Keywords flagged for CTR improvement (title/meta).", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]), + _tool("get_gsc_ctr_opportunity_pages", "GSC pages with high impressions and below-curve CTR.", {"property_id": _PID, "min_impressions": {"type": "integer"}, "limit": _LIMIT}), + # Compare extras + _tool("compare_indexation_deltas", "Indexation coverage count and gap list changes vs baseline.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]), + _tool("compare_orphan_deltas", "Orphan URL set changes vs baseline report.", {"baseline_report_id": _RID, "report_id": _RID}, ["baseline_report_id"]), + # GEO / AEO + _tool("get_llms_txt_status", "Check for /llms.txt and /.well-known/llms.txt on the property domain.", {"property_id": _PID, "report_id": _RID}), + _tool("get_faq_schema_coverage", "FAQPage/QAPage schema coverage across crawled pages.", {"property_id": _PID, "report_id": _RID}), + _tool("list_pages_missing_faq_schema", "Q&A-style URLs missing FAQ schema markup.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_geo_readiness_score", "Composite 0-100 GEO readiness score from crawl signals.", {"property_id": _PID, "report_id": _RID}), + _tool("get_aeo_content_signals_for_url", "Per-URL answer-engine quotability signals.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]), + _tool("get_eeat_signals_summary", "Author/Organization schema and about/contact page counts.", {"property_id": _PID, "report_id": _RID}), + _tool("get_js_rendering_delta", "Static vs rendered title/word-count differences.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_internal_link_suggestions", "TF-IDF related pages and anchor hints for a source URL.", {"url": _URL, "property_id": _PID, "report_id": _RID, "limit": {"type": "integer", "maximum": 10}}, ["url"]), + # LLM generators + _tool("generate_issue_fix", "LLM fix suggestion for one audit issue message.", {"property_id": _PID, "message": {"type": "string"}, "url": _URL, "priority": {"type": "string"}, "category_id": {"type": "string"}, "refresh": {"type": "boolean"}}, ["message"]), + _tool("summarize_category_for_client", "Client-friendly category summary with optional LLM narrative.", {"category_id": {"type": "string"}, "property_id": _PID, "report_id": _RID}, ["category_id"]), + _tool("prioritize_fix_roadmap", "Top N issues ranked by impact_score for a fix roadmap.", {"property_id": _PID, "report_id": _RID, "limit": {"type": "integer", "maximum": 30}}), + _tool("analyze_serp_snippet_for_url", "GSC query context plus LLM title/meta CTR suggestions.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]), + _tool("draft_llms_txt", "Draft llms.txt content from top pages and schema coverage.", {"property_id": _PID, "report_id": _RID}), + # Integrations + _tool("get_gsc_url_inspection", "Live GSC URL Inspection (indexing + rich results). Requires Google OAuth.", {"url": _URL, "property_id": _PID}, ["url", "property_id"]), + _tool("get_gsc_index_coverage", "Estimated indexation coverage from crawl + sitemap + GSC join.", {"property_id": _PID, "report_id": _RID}), + _tool("get_bing_index_status", "Bing Webmaster URL info (requires bing_webmaster_api_key).", {"url": _URL, "property_id": _PID}, ["url", "property_id"]), + _tool("get_serp_feature_overlay", "Keywords with SERP feature / competition overlay data.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]), + _tool("check_ai_citation_presence", "On-site citation readiness estimate for brand/query (no live LLM API).", {"property_id": _PID, "query": {"type": "string"}, "brand": {"type": "string"}}), ] diff --git a/tests/test_audit_tools_expanded.py b/tests/test_audit_tools_expanded.py index 59fcad20..a0ffdcc7 100644 --- a/tests/test_audit_tools_expanded.py +++ b/tests/test_audit_tools_expanded.py @@ -72,6 +72,14 @@ def _full_payload() -> dict: "security_findings": [{"url": "https://ex.com", "severity": "high", "finding_type": "hsts", "message": "Missing HSTS"}], "tech_stack_summary": {"technologies": [{"name": "WordPress", "count": 10}]}, "competitor_link_gap": {"gaps": [{"domain": "rival.com"}]}, + "competitor_keyword_gap": [{"keyword": "widgets", "competitor": "rival.com"}], + "portfolio_benchmark": {"median_health_score": 75, "property_health_score": 80}, + "rich_results_meta": {"checked": 5, "gsc_count": 2, "heuristic_count": 3}, + "rich_results_validation": [ + {"url": "https://ex.com/", "status": "pass"}, + {"url": "https://ex.com/bad", "status": "fail", "message": "Invalid schema"}, + ], + "inlink_anchor_matrix": [{"target_url": "https://ex.com/", "anchor_text": "home", "inlink_count": 3}], "bing_backlinks": {"ok": True, "total": 100}, "crux_summary": {"ok": True, "lcp_p75": 2.1}, "gsc_links": {"imported_at": "2026-06-01", "top_linking_sites": []}, @@ -134,7 +142,16 @@ def _full_payload() -> dict: "name": "Technical SEO", "score": 80, "issues": [ - {"priority": "Critical", "message": "Missing title", "url": "https://ex.com/a", "recommendation": "Add title"}, + { + "priority": "Critical", + "message": "Missing title", + "url": "https://ex.com/a", + "recommendation": "Add title", + "impact_score": 1205.0, + "gsc_clicks": 20, + "gsc_impressions": 500, + "ga4_sessions": 1, + }, {"priority": "High", "message": "Slow page", "url": "https://ex.com/blog/slow", "recommendation": "Optimize"}, ], }, @@ -161,7 +178,7 @@ def conn() -> MagicMock: def test_handler_schema_parity() -> None: names = {t["name"] for t in TOOL_DEFINITIONS} assert names == tool_handler_names() - assert len(TOOL_DEFINITIONS) == 180 + assert len(TOOL_DEFINITIONS) == 221 def test_slice_helpers() -> None: diff --git a/tests/test_audit_tools_expansion.py b/tests/test_audit_tools_expansion.py new file mode 100644 index 00000000..70a64773 --- /dev/null +++ b/tests/test_audit_tools_expansion.py @@ -0,0 +1,203 @@ +"""Tests for expanded audit tools (impact, payload extras, GEO, compare deltas).""" +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from website_profiling.tools.audit_tools import AuditToolContext, dispatch_tool +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx + + +@pytest.fixture +def ctx() -> AuditToolContext: + return AuditToolContext(property_id=1, report_id=1) + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +def _payload() -> dict: + return { + "categories": [ + { + "id": "technical_seo", + "name": "Technical", + "score": 80, + "issues": [ + { + "priority": "Critical", + "message": "Missing title", + "url": "https://ex.com/a", + "impact_score": 1205.0, + "gsc_clicks": 20, + }, + {"priority": "Low", "message": "Minor", "url": "", "impact_score": 1.0}, + ], + } + ], + "indexation_coverage": { + "counts": {"crawled": 10, "sitemap": 12}, + "lists": {"sitemap_only": ["https://ex.com/x"], "crawled_not_in_sitemap": [], "gsc_not_crawled": []}, + "lists_total": {"sitemap_only": 1, "crawled_not_in_sitemap": 0, "gsc_not_crawled": 0}, + }, + "orphan_urls": ["https://ex.com/orphan"], + "rich_results_meta": {"checked": 2}, + "rich_results_validation": [{"url": "https://ex.com/bad", "status": "fail"}], + "competitor_keyword_gap": [{"keyword": "widgets"}], + "portfolio_benchmark": {"median_health_score": 70}, + "inlink_anchor_matrix": [{"target_url": "https://ex.com/", "anchor_text": "home", "inlink_count": 2}], + "top_pages": [{"url": "https://ex.com/", "inlinks": 3, "outlinks": 0}], + "links": [{"url": "https://ex.com/", "inlinks": 3, "outlinks": 0}], + "ner_site_summary": {"entities": ["Acme"]}, + "schema_coverage": {"pages_with_schema": 5}, + "site_name": "Example", + } + + +def _crawl_df() -> pd.DataFrame: + return pd.DataFrame([ + { + "url": "https://ex.com/", + "status": "200", + "title": "Home", + "meta_description": "Home desc", + "h1": "Home", + "outlinks": 0, + "mixed_content_count": 1, + "total_js_bytes": 50000, + "total_css_bytes": 10000, + "script_count": 5, + "content_html_ratio": 10, + "reading_level": 8.5, + "cache_control": "", + "etag": "", + "heading_sequence": "h1,h2", + "heading_text": "Home\nSub", + "content_excerpt": "Widgets are devices used for many purposes in industry and home.", + "word_count": 400, + "has_schema": "true", + "page_analysis": json.dumps({"json_ld_types": ["Organization"], "axe_violations": [{"id": "label"}]}), + "fetch_method": "static", + }, + { + "url": "https://ex.com/404-page", + "status": "200", + "title": "Page not found", + "meta_description": "x", + "h1": "Oops", + "outlinks": 2, + "mixed_content_count": 0, + "page_analysis": "{}", + }, + { + "url": "https://ex.com/faq", + "status": "200", + "title": "FAQ", + "meta_description": "FAQ", + "h1": "FAQ?", + "outlinks": 1, + "has_schema": "false", + "page_analysis": "{}", + }, + ]) + + +def test_list_top_impact_issues_includes_traffic_fields(conn: MagicMock, ctx: AuditToolContext) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()): + out = dispatch_tool("list_top_impact_issues", {"limit": 5}, context=ctx, conn=conn) + assert out["issues"][0].get("impact_score") == 1205.0 + assert out["issues"][0].get("gsc_clicks") == 20 + + +def test_payload_extras_tools(conn: MagicMock, ctx: AuditToolContext) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()), patch.object(Ctx, "load_crawl_df", return_value=_crawl_df()): + assert dispatch_tool("get_rich_results_summary", {}, context=ctx, conn=conn)["meta"]["checked"] == 2 + failures = dispatch_tool("list_rich_results_failures", {}, context=ctx, conn=conn) + assert failures["total"] == 1 + assert dispatch_tool("get_competitor_keyword_gap", {}, context=ctx, conn=conn)["total"] == 1 + assert dispatch_tool("get_portfolio_benchmark", {}, context=ctx, conn=conn)["benchmark"]["median_health_score"] == 70 + anchors = dispatch_tool("get_site_anchor_text_summary", {}, context=ctx, conn=conn) + assert anchors["anchors"][0]["anchor_text"] == "home" + + +def test_crawl_extras_tools(conn: MagicMock, ctx: AuditToolContext) -> None: + payload = _payload() + with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=_crawl_df()): + soft = dispatch_tool("list_pages_soft_404", {}, context=ctx, conn=conn) + assert soft["total"] >= 1 + axe = dispatch_tool("get_axe_audit_summary", {}, context=ctx, conn=conn) + assert axe["pages_with_violations"] >= 1 + mixed = dispatch_tool("list_pages_with_mixed_content", {}, context=ctx, conn=conn) + assert mixed["total"] >= 1 + dupes = dispatch_tool("list_duplicate_title_groups", {}, context=ctx, conn=conn) + assert "groups" in dupes + outline = dispatch_tool("get_heading_outline_for_url", {"url": "https://ex.com/"}, context=ctx, conn=conn) + assert outline.get("heading_sequence") + + +def test_compare_indexation_and_orphan_deltas(conn: MagicMock, ctx: AuditToolContext) -> None: + current = _payload() + baseline = {**_payload(), "indexation_coverage": {"counts": {"crawled": 8}, "lists": {}, "lists_total": {}}, "orphan_urls": []} + with patch("website_profiling.tools.audit_tools.compare_slices.load_compare_pair", return_value=(current, baseline, 2, 1, None)): + idx = dispatch_tool("compare_indexation_deltas", {"baseline_report_id": 1}, context=ctx, conn=conn) + assert idx["count_deltas"] + orphan = dispatch_tool("compare_orphan_deltas", {"baseline_report_id": 1}, context=ctx, conn=conn) + assert orphan["added_count"] >= 1 + + +def test_geo_tools_mocked(conn: MagicMock, ctx: AuditToolContext) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()), patch.object(Ctx, "load_crawl_df", return_value=_crawl_df()), patch( + "website_profiling.tools.audit_tools.geo_tools._fetch_llms_txt", + return_value={"found": False}, + ): + geo = dispatch_tool("get_geo_readiness_score", {}, context=ctx, conn=conn) + assert 0 <= geo["geo_readiness_score"] <= 100 + faq = dispatch_tool("get_faq_schema_coverage", {}, context=ctx, conn=conn) + assert "coverage_pct" in faq + suggestions = dispatch_tool( + "get_internal_link_suggestions", + {"url": "https://ex.com/"}, + context=ctx, + conn=conn, + ) + assert "suggestions" in suggestions + + +def test_prioritize_fix_roadmap(conn: MagicMock, ctx: AuditToolContext) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()): + out = dispatch_tool("prioritize_fix_roadmap", {"limit": 5}, context=ctx, conn=conn) + assert out["roadmap"][0]["rank"] == 1 + assert out["roadmap"][0]["impact_score"] == 1205.0 + + +def test_integration_tools_missing_config(conn: MagicMock, ctx: AuditToolContext) -> None: + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value={"canonical_domain": "ex.com"}): + gsc = dispatch_tool("get_gsc_url_inspection", {"url": "https://ex.com/"}, context=ctx, conn=conn) + assert gsc["missing"] is True + bing = dispatch_tool("get_bing_index_status", {"url": "https://ex.com/"}, context=ctx, conn=conn) + assert bing["missing"] is True + + +def test_gsc_index_coverage_from_payload(conn: MagicMock, ctx: AuditToolContext) -> None: + with patch.object(Ctx, "load_payload", return_value=_payload()): + out = dispatch_tool("get_gsc_index_coverage", {}, context=ctx, conn=conn) + assert out["counts"]["crawled"] == 10 + assert out["provenance"] == "Estimated" + + +def test_gsc_url_inspection_mocked(conn: MagicMock, ctx: AuditToolContext) -> None: + prop = {"google_refresh_token": "tok", "gsc_site_url": "https://ex.com/"} + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value=prop), patch( + "website_profiling.tools.audit_tools.integration_tools.build_credentials", + return_value=object(), + ), patch( + "website_profiling.tools.audit_tools.integration_tools.inspect_url", + return_value={"verdict": "PASS", "provenance": "GSC"}, + ): + out = dispatch_tool("get_gsc_url_inspection", {"url": "https://ex.com/page"}, context=ctx, conn=conn) + assert out["verdict"] == "PASS" diff --git a/tests/test_audit_tools_expansion_coverage.py b/tests/test_audit_tools_expansion_coverage.py new file mode 100644 index 00000000..10bf15fc --- /dev/null +++ b/tests/test_audit_tools_expansion_coverage.py @@ -0,0 +1,633 @@ +"""Line-coverage tests for audit tools expansion modules.""" +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest +import requests + +from website_profiling.tools.audit_tools import dispatch_tool +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx +from website_profiling.tools.audit_tools import crawl_lists as cl_mod +from website_profiling.tools.audit_tools import crawl_metrics as cm_mod +from website_profiling.tools.audit_tools import geo_tools as geo_mod +from website_profiling.tools.audit_tools import google as google_mod +from website_profiling.tools.audit_tools import integration_tools as int_mod +from website_profiling.tools.audit_tools import keywords as kw_mod +from website_profiling.tools.audit_tools import llm_tools as llm_mod +from website_profiling.tools.audit_tools import payload_extras as pe_mod +from website_profiling.tools.audit_tools import compare_slices as cmp_mod +from website_profiling.tools.audit_tools import report as report_mod + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +@pytest.fixture +def ctx() -> Ctx: + return Ctx(property_id=1, report_id=1) + + +def _crawl_df() -> pd.DataFrame: + return pd.DataFrame([ + { + "url": "https://ex.com/", + "status": "200", + "title": "Home", + "meta_description": "Home desc", + "h1": "Home", + "outlinks": 0, + "mixed_content_count": 1, + "total_js_bytes": 50000, + "total_css_bytes": 10000, + "script_count": 5, + "content_html_ratio": 10, + "reading_level": 8.5, + "cache_control": "", + "etag": "", + "heading_sequence": "h1,h2", + "heading_text": "Home\nSub", + "content_excerpt": "Widgets are devices used for many purposes.", + "word_count": 400, + "has_schema": "true", + "page_analysis": json.dumps({ + "json_ld_types": ["Organization", "FAQPage"], + "axe_violations": [{"id": "label"}, "skip"], + "pagination": {"rel_next": True, "rel_prev": True, "amphtml": "https://ex.com/amp"}, + "headings": [{"level": "h1", "text": "Home"}], + }), + "fetch_method": "static", + "canonical_url": "https://ex.com/other", + }, + { + "url": "https://ex.com/", + "status": "200", + "title": "Home rendered", + "meta_description": "Home desc", + "h1": "Home R", + "outlinks": 2, + "word_count": 500, + "fetch_method": "rendered", + "page_analysis": json.dumps({"json_ld_types": ["Person"]}), + }, + { + "url": "https://ex.com/about", + "status": "200", + "title": "About", + "meta_description": "About", + "h1": "About us", + "outlinks": 1, + "page_analysis": json.dumps({"json_ld_types": ["Organization"]}), + }, + { + "url": "https://ex.com/faq", + "status": "200", + "title": "FAQ", + "meta_description": "FAQ", + "h1": "FAQ?", + "outlinks": 1, + "has_schema": "false", + "page_analysis": "{}", + }, + { + "url": "https://ex.com/dup", + "status": "200", + "title": "Dup", + "meta_description": "same", + "outlinks": 1, + "page_analysis": "{}", + }, + { + "url": "https://ex.com/dup2", + "status": "200", + "title": "Dup", + "meta_description": "same", + "outlinks": 1, + "page_analysis": "{}", + }, + { + "url": "https://ex.com/bad-ratio", + "status": "200", + "title": "Bad", + "content_html_ratio": "bad", + "page_analysis": "{}", + }, + { + "url": "https://ex.com/cache", + "status": "200", + "title": "Cache", + "cache_control": "no-cache", + "etag": "", + "page_analysis": "{}", + }, + ]) + + +def test_payload_extras_edge_paths(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_payload", return_value=None): + assert pe_mod.get_rich_results_summary(conn, ctx, {})["missing"] is True + assert pe_mod.list_rich_results_failures(conn, ctx, {})["error"] + assert pe_mod.get_competitor_keyword_gap(conn, ctx, {})["error"] + assert pe_mod.get_portfolio_benchmark(conn, ctx, {})["missing"] is True + assert pe_mod.get_site_anchor_text_summary(conn, ctx, {})["error"] + + with patch.object(Ctx, "load_payload", return_value={"rich_results_meta": "bad"}): + assert pe_mod.get_rich_results_summary(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value={"rich_results_validation": "bad"}): + assert pe_mod.list_rich_results_failures(conn, ctx, {})["total"] == 0 + + with patch.object(Ctx, "load_payload", return_value={"competitor_keyword_gap": "bad"}): + assert pe_mod.get_competitor_keyword_gap(conn, ctx, {})["total"] == 0 + + with patch.object(Ctx, "load_payload", return_value={"inlink_anchor_matrix": []}): + assert pe_mod.get_site_anchor_text_summary(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value={"inlink_anchor_matrix": [{"anchor_text": "x", "inlink_count": "bad"}]}): + out = pe_mod.get_site_anchor_text_summary(conn, ctx, {}) + assert out["anchors"][0]["inlink_count"] == 0 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert pe_mod.get_pagination_audit_summary(conn, ctx, {})["orphan_prev_count"] == 0 + + pag_df = pd.DataFrame([ + {"url": "https://ex.com/p1", "status": "200", "canonical_url": "https://ex.com/c", "page_analysis": json.dumps({"pagination": {"rel_prev": True, "amphtml": "https://ex.com/amp"}})}, + {"url": "https://ex.com/p2", "status": "404", "page_analysis": "{}"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=pag_df): + pag = pe_mod.get_pagination_audit_summary(conn, ctx, {}) + assert pag["orphan_prev_count"] >= 1 + assert pag["amp_mismatch_count"] >= 1 + + with patch.object(Ctx, "load_payload", return_value={"portfolio_benchmark": {"x": 1}}): + assert pe_mod.get_portfolio_benchmark(conn, ctx, {})["missing"] is False + + +def test_crawl_metrics_and_lists(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert cm_mod.get_asset_weight_summary(conn, ctx, {})["missing"] is True + assert cm_mod.get_readability_summary(conn, ctx, {})["missing"] is True + + bad_df = pd.DataFrame([ + {"url": "https://ex.com/", "status": "200", "total_js_bytes": "bad", "reading_level": "bad"}, + {"url": "https://ex.com/2", "status": "200", "reading_level": 0}, + {"url": "https://ex.com/3", "status": "200", "reading_level": 14}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=bad_df): + assert cm_mod.get_asset_weight_summary(conn, ctx, {})["js_bytes"]["count"] == 0 + assert cm_mod.get_readability_summary(conn, ctx, {})["pages_with_reading_level"] == 1 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame([{"url": "x", "status": "200"}])): + assert cl_mod.get_axe_audit_summary(conn, ctx, {})["pages_with_violations"] == 0 + assert cl_mod.list_duplicate_title_groups(conn, ctx, {})["total"] == 0 + assert cl_mod.list_heavy_pages_by_bytes(conn, ctx, {})["total"] == 0 + assert cl_mod.get_heading_outline_for_url(conn, ctx, {"url": "https://ex.com"})["error"] + + df = _crawl_df() + with patch.object(Ctx, "load_crawl_df", return_value=df): + assets = cm_mod.get_asset_weight_summary(conn, ctx, {}) + assert assets["js_bytes"]["count"] >= 1 + read = cm_mod.get_readability_summary(conn, ctx, {}) + assert read["pages_with_reading_level"] >= 1 + assert cl_mod.list_pages_with_axe_violations(conn, ctx, {})["total"] >= 1 + assert cl_mod.get_axe_audit_summary(conn, ctx, {})["total_violations"] >= 1 + assert cl_mod.list_pages_with_mixed_content(conn, ctx, {})["total"] >= 1 + assert cl_mod.list_duplicate_title_groups(conn, ctx, {})["total"] >= 1 + assert cl_mod.list_heavy_pages_by_bytes(conn, ctx, {})["total"] >= 1 + assert cl_mod.list_pages_poor_cache_headers(conn, ctx, {})["total"] >= 1 + assert cl_mod.list_pages_low_content_ratio(conn, ctx, {"max_content_html_ratio": "bad"})["total"] >= 1 + assert cl_mod.get_heading_outline_for_url(conn, ctx, {})["error"] == "url is required" + assert cl_mod.get_heading_outline_for_url(conn, ctx, {"url": "https://ex.com/missing"})["error"] + + payload = { + "orphan_urls": ["https://ex.com/orphan"], + "top_pages": [{"url": "https://ex.com/", "inlinks": 3}], + } + with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df): + dead = cl_mod.list_dead_end_pages(conn, ctx, {}) + assert dead["total"] >= 1 + outline = cl_mod.get_heading_outline_for_url(conn, ctx, {"url": "https://ex.com/"}) + assert outline.get("heading_sequence") + + +def test_geo_tools_paths(conn: MagicMock, ctx: Ctx) -> None: + assert geo_mod._fetch_llms_txt("")["found"] is False + + mock_resp = MagicMock(status_code=200, text="llms content", content=b"llms content") + with patch("website_profiling.tools.audit_tools.geo_tools.requests.get", return_value=mock_resp): + found = geo_mod._fetch_llms_txt("ex.com") + assert found["found"] is True + + with patch("website_profiling.tools.audit_tools.geo_tools.requests.get", side_effect=requests.RequestException("fail")): + assert geo_mod._fetch_llms_txt("ex.com")["found"] is False + + with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch.object( + Ctx, "load_crawl_df", return_value=None, + ): + assert geo_mod.get_faq_schema_coverage(conn, ctx, {})["total_2xx"] == 0 + assert geo_mod.list_pages_missing_faq_schema(conn, ctx, {})["total"] == 0 + assert geo_mod.get_eeat_signals_summary(conn, ctx, {})["missing"] is True + assert geo_mod.get_js_rendering_delta(conn, ctx, {})["total"] == 0 + + payload = {"ner_site_summary": {"entities": ["Acme", "Widgets"]}} + with patch.object(Ctx, "load_payload", return_value=payload), patch.object( + Ctx, "load_crawl_df", return_value=_crawl_df(), + ), patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch( + "website_profiling.tools.audit_tools.geo_tools._fetch_llms_txt", + return_value={"found": True}, + ): + geo = geo_mod.get_geo_readiness_score(conn, ctx, {}) + assert 0 <= geo["geo_readiness_score"] <= 100 + aeo = geo_mod.get_aeo_content_signals_for_url(conn, ctx, {"url": "https://ex.com/"}) + assert aeo.get("quotability_score") is not None + assert geo_mod.get_aeo_content_signals_for_url(conn, ctx, {})["error"] + eeat = geo_mod.get_eeat_signals_summary(conn, ctx, {}) + assert eeat["pages_with_organization_schema"] >= 1 + js = geo_mod.get_js_rendering_delta(conn, ctx, {}) + assert js["total"] >= 1 + missing = geo_mod.list_pages_missing_faq_schema(conn, ctx, {}) + assert missing["total"] >= 1 + links = geo_mod.get_internal_link_suggestions(conn, ctx, {"url": "https://ex.com/"}) + assert links.get("suggestions") is not None + assert geo_mod.get_internal_link_suggestions(conn, ctx, {})["error"] + assert geo_mod.get_internal_link_suggestions(conn, ctx, {"url": "https://ex.com/nope"})["error"] + + with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch( + "website_profiling.tools.audit_tools.geo_tools._fetch_llms_txt", return_value={"found": False}, + ): + assert geo_mod.get_llms_txt_status(conn, ctx, {})["domain"] == "ex.com" + + sparse = pd.DataFrame([{"url": "https://ex.com/only", "status": "200", "title": "Only", "content_excerpt": "one page only here"}]) + with patch.object(Ctx, "load_crawl_df", return_value=sparse): + assert geo_mod.get_internal_link_suggestions(conn, ctx, {"url": "https://ex.com/only"})["note"] + + aeo_df = pd.DataFrame([{ + "url": "https://ex.com/aeo", + "status": "200", + "content_excerpt": "- bullet one\nWidgets are tools that means something.", + "html": "
  • item
    • ", + "word_count": 250, + "top_keywords": "widgets", + "page_analysis": json.dumps({"json_ld_types": ["FAQPage"]}), + }]) + with patch.object(Ctx, "load_crawl_df", return_value=aeo_df): + aeo = geo_mod.get_aeo_content_signals_for_url(conn, ctx, {"url": "https://ex.com/aeo"}) + assert aeo["has_lists"] is True + assert geo_mod.get_aeo_content_signals_for_url(conn, ctx, {"url": "https://ex.com/missing"})["error"] + + empty_geo = pd.DataFrame([{"url": "https://ex.com/e", "status": "404", "page_analysis": "{}"}]) + with patch.object(Ctx, "load_payload", return_value={}), patch.object(Ctx, "load_crawl_df", return_value=empty_geo), patch.object( + Ctx, "resolve_property_domain", return_value="ex.com", + ), patch("website_profiling.tools.audit_tools.geo_tools._fetch_llms_txt", return_value={"found": False}): + geo_empty = geo_mod.get_geo_readiness_score(conn, ctx, {}) + assert geo_empty["components"]["schema_coverage"] == 0 + + +def test_google_ctr_and_keywords(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_google", return_value=None): + assert google_mod.get_gsc_ctr_opportunity_pages(conn, ctx, {})["error"] + + gsc_data = { + "gsc": { + "pages": [ + {"page": "https://ex.com/a", "impressions": 500, "position": 5, "ctr": "0.5%"}, + "skip", + {"page": "https://ex.com/b", "impressions": 50, "position": 10, "ctr": "5%"}, + {"page": "https://ex.com/c", "impressions": 200, "position": "bad"}, + ], + }, + } + with patch.object(Ctx, "load_google", return_value=gsc_data): + out = google_mod.get_gsc_ctr_opportunity_pages(conn, ctx, {"min_impressions": "bad"}) + assert "pages" in out + assert out["provenance"] == "Search Console" + + with patch.object(Ctx, "load_keywords", return_value={"rows": [{"recommended_action": "Improve CTR now", "keyword": "widgets"}]}): + ctr_kw = kw_mod.list_keywords_ctr_opportunity(conn, ctx, {}) + assert ctr_kw["total"] >= 1 + + with patch.object(Ctx, "load_google", return_value={"gsc": {"pages": "bad"}}): + assert google_mod.get_gsc_ctr_opportunity_pages(conn, ctx, {})["total"] == 0 + + high_ctr = {"gsc": {"pages": [{"page": "https://ex.com/good", "impressions": 1000, "position": 3, "ctr": "15%"}]}} + with patch.object(Ctx, "load_google", return_value=high_ctr): + assert google_mod.get_gsc_ctr_opportunity_pages(conn, ctx, {})["total"] == 0 + + +def test_integration_tools_paths(conn: MagicMock, ctx: Ctx) -> None: + assert int_mod.get_gsc_url_inspection(conn, Ctx(property_id=None), {"url": "https://ex.com"})["missing"] + assert int_mod.get_gsc_url_inspection(conn, ctx, {})["missing"] + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value=None): + assert int_mod.get_gsc_url_inspection(conn, ctx, {"url": "https://ex.com"})["missing"] + + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value={"canonical_domain": "ex.com"}): + assert int_mod.get_bing_index_status(conn, ctx, {})["missing"] + with patch("website_profiling.db.config_store.read_pipeline_config", return_value=({"bing_webmaster_api_key": "key"}, {})): + with patch("website_profiling.integrations.bing.webmaster._bing_json_get", return_value={"d": {"indexed": True}}): + bing = int_mod.get_bing_index_status(conn, ctx, {"url": "https://ex.com/page"}) + assert bing["provenance"] == "Bing Webmaster" + with patch("website_profiling.integrations.bing.webmaster._bing_json_get", return_value={"error": "fail"}): + assert int_mod.get_bing_index_status(conn, ctx, {"url": "https://ex.com/page"})["missing"] + + with patch.object(Ctx, "load_payload", return_value=None): + assert int_mod.get_gsc_index_coverage(conn, ctx, {})["missing"] + with patch.object(Ctx, "load_payload", return_value={"indexation_coverage": "bad"}): + assert int_mod.get_gsc_index_coverage(conn, ctx, {})["missing"] + + with patch.object(Ctx, "load_keywords", return_value={"rows": [{"serp_features": ["faq"]}], "serp_overlay_count": 1}): + serp = int_mod.get_serp_feature_overlay(conn, ctx, {}) + assert serp["total"] == 1 + + with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch.object( + Ctx, "load_payload", + return_value={"categories": [{"issues": [{"message": "Acme brand issue"}]}], "ner_site_summary": {"entities": ["Acme"]}, "schema_coverage": {"pages_with_schema": 3}}, + ): + cite = int_mod.check_ai_citation_presence(conn, ctx, {"query": "Acme"}) + assert cite["entity_in_ner_summary"] is True + + prop = {"google_refresh_token": "t", "gsc_site_url": "https://ex.com/"} + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value=prop), patch( + "website_profiling.tools.audit_tools.integration_tools.build_credentials", side_effect=RuntimeError("creds"), + ): + out = int_mod.get_gsc_url_inspection(conn, ctx, {"url": "https://ex.com"}) + assert "credentials error" in out["error"] + + with patch.object(Ctx, "load_keywords", return_value=None): + assert int_mod.get_serp_feature_overlay(conn, ctx, {})["missing"] is True + with patch.object(Ctx, "resolve_property_domain", return_value=""): + assert int_mod.check_ai_citation_presence(conn, Ctx(property_id=1), {})["missing"] is True + + +def test_llm_tools_paths(conn: MagicMock, ctx: Ctx) -> None: + with patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}), patch( + "website_profiling.llm_config.llm_is_enabled", return_value=False, + ): + assert llm_mod._llm_disabled_response()["missing"] is True + assert llm_mod.generate_issue_fix(conn, ctx, {})["missing"] is True + + with patch("website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}): + assert llm_mod.generate_issue_fix(conn, ctx, {"message": ""})["error"] + + with patch("website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}), patch( + "website_profiling.llm.issue_fixes.generate_issue_fix_suggestion", + return_value={"fix": "x"}, + ), patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}): + out = llm_mod.generate_issue_fix(conn, ctx, {"message": "Fix title"}) + assert out["provenance"] == "AI insights" + + cat_data = {"name": "Tech", "score": 80, "issues": [{"priority": "High", "message": "Slow", "url": "https://ex.com"}]} + assert llm_mod.summarize_category_for_client(conn, ctx, {})["error"] == "category_id is required" + + with patch("website_profiling.tools.audit_tools.issues.get_category_issues", return_value=cat_data), patch( + "website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={"missing": True}, + ): + summary = llm_mod.summarize_category_for_client(conn, ctx, {"category_id": "tech"}) + assert summary["provenance"] == "Crawl" + + with patch("website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}), patch( + "website_profiling.llm.base.get_llm_client", + return_value=MagicMock(complete_json=MagicMock(return_value={"summary": "Client text"})), + ), patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}), patch( + "website_profiling.tools.audit_tools.issues.get_category_issues", return_value=cat_data, + ): + narrative = llm_mod.summarize_category_for_client(conn, ctx, {"category_id": "tech"}) + assert narrative["narrative"] == "Client text" + + with patch.object(Ctx, "load_payload", return_value=None): + assert llm_mod.prioritize_fix_roadmap(conn, ctx, {})["error"] + with patch.object(Ctx, "load_payload", return_value={"categories": []}), patch( + "website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={"error": "off"}, + ), patch.object(Ctx, "load_google", return_value=None), patch.object(Ctx, "load_crawl_df", return_value=None): + snippet = llm_mod.analyze_serp_snippet_for_url(conn, ctx, {"url": "https://ex.com"}) + assert snippet["provenance"] == "Crawl" + + with patch.object(Ctx, "load_payload", return_value={"site_name": "Ex", "top_pages": [{"url": "https://ex.com"}]}), patch( + "website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}, + ), patch( + "website_profiling.llm.base.get_llm_client", + return_value=MagicMock(complete_json=MagicMock(return_value={"content": "# Ex\n\nPolished"})), + ), patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}): + draft = llm_mod.draft_llms_txt(conn, ctx, {}) + assert "Polished" in draft["llms_txt_draft"] + + with patch("website_profiling.tools.audit_tools.issues.get_category_issues", return_value={"error": "no cat"}): + assert llm_mod.summarize_category_for_client(conn, ctx, {"category_id": "x"})["error"] == "no cat" + + with patch.object(Ctx, "load_payload", return_value={"categories": []}), patch( + "website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}, + ), patch("website_profiling.llm.base.get_llm_client", return_value=None), patch.object( + Ctx, "load_google", return_value={"gsc": {}}, + ), patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame([{"url": "https://ex.com", "title": "T", "meta_description": "D"}])): + snippet = llm_mod.analyze_serp_snippet_for_url(conn, ctx, {"url": "https://ex.com"}) + assert snippet["provenance"] == "Crawl" + + with patch.object(Ctx, "load_payload", return_value=None): + assert llm_mod.draft_llms_txt(conn, ctx, {})["error"] + + client = MagicMock(complete_json=MagicMock(side_effect=RuntimeError("llm fail"))) + with patch.object(Ctx, "load_payload", return_value={"site_name": "Ex"}), patch( + "website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}, + ), patch("website_profiling.llm.base.get_llm_client", return_value=client), patch( + "website_profiling.llm_config.load_llm_config_from_db", return_value={}, + ): + err_snippet = llm_mod.analyze_serp_snippet_for_url(conn, ctx, {"url": "https://ex.com"}) + assert "error" in err_snippet + + +def test_report_search_impact_sort(conn: MagicMock, ctx: Ctx) -> None: + payload = { + "categories": [{ + "id": "x", + "issues": [ + {"priority": "Low", "message": "a", "impact_score": 1}, + {"priority": "High", "message": "b", "impact_score": 99, "gsc_clicks": 5}, + ], + }], + } + with patch.object(Ctx, "load_payload", return_value=payload): + out = report_mod.search_issues(conn, ctx, {"sort": "impact", "limit": 10}) + assert out["issues"][0]["impact_score"] == 99 + top = report_mod.list_top_impact_issues(conn, ctx, {"limit": 1}) + assert top["issues"][0]["gsc_clicks"] == 5 + + +def test_expansion_coverage_gaps(conn: MagicMock, ctx: Ctx) -> None: + empty = pd.DataFrame() + with patch.object(Ctx, "load_crawl_df", return_value=empty): + assert cl_mod.get_axe_audit_summary(conn, ctx, {})["pages_with_violations"] == 0 + assert cl_mod.list_duplicate_title_groups(conn, ctx, {})["total"] == 0 + assert cl_mod.list_heavy_pages_by_bytes(conn, ctx, {})["total"] == 0 + assert cl_mod.get_heading_outline_for_url(conn, ctx, {"url": "https://ex.com"})["error"] == "no crawl data" + + orphan_payload = {"orphan_urls": ["https://ex.com/orphaned"], "top_pages": [{"url": "https://ex.com/orphaned", "inlinks": 2}]} + orphan_df = pd.DataFrame([{"url": "https://ex.com/orphaned", "status": "200", "outlinks": 0, "title": "Orphan"}]) + with patch.object(Ctx, "load_payload", return_value=orphan_payload), patch.object(Ctx, "load_crawl_df", return_value=orphan_df): + assert cl_mod.list_dead_end_pages(conn, ctx, {})["total"] == 0 + + no_assets = pd.DataFrame([{"url": "https://ex.com/light", "status": "200", "total_js_bytes": 0, "total_css_bytes": 0, "script_count": 0}]) + with patch.object(Ctx, "load_crawl_df", return_value=no_assets): + assert cl_mod.list_heavy_pages_by_bytes(conn, ctx, {})["total"] == 0 + + read_df = pd.DataFrame([ + {"url": "https://ex.com/a", "status": "200", "reading_level": 5}, + {"url": "https://ex.com/b", "status": "200", "reading_level": 8}, + {"url": "https://ex.com/c", "status": "200", "reading_level": 11}, + {"url": "https://ex.com/d", "status": "200", "reading_level": 15}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=read_df): + hist = cm_mod.get_readability_summary(conn, ctx, {})["histogram"] + assert hist["0-6"] == 1 and hist["7-9"] == 1 and hist["10-12"] == 1 and hist["13+"] == 1 + + assets_df = pd.DataFrame([{"url": "https://ex.com/", "status": "404", "total_js_bytes": 1000}]) + with patch.object(Ctx, "load_crawl_df", return_value=assets_df): + assert cm_mod.get_asset_weight_summary(conn, ctx, {})["js_bytes"]["count"] == 0 + + with patch.object(Ctx, "load_payload", return_value={"portfolio_benchmark": None}): + assert pe_mod.get_portfolio_benchmark(conn, ctx, {})["missing"] is True + with patch.object(Ctx, "load_payload", return_value={"inlink_anchor_matrix": ["bad", {"anchor_text": "", "inlink_count": 1}]}): + assert pe_mod.get_site_anchor_text_summary(conn, ctx, {})["anchors"][0]["anchor_text"] == "(empty)" + + pag_df = pd.DataFrame([{"url": "https://ex.com/p", "status": "200", "page_analysis": json.dumps({"pagination": {"rel_next": True}})}]) + with patch.object(Ctx, "load_crawl_df", return_value=pag_df): + assert pe_mod.get_pagination_audit_summary(conn, ctx, {})["pages_with_rel_next"] == 1 + + low_ctr = {"gsc": {"pages": [{"page": "https://ex.com/low", "impressions": 500, "position": 5, "ctr": 0.001}]}} + with patch.object(Ctx, "load_google", return_value=low_ctr): + assert google_mod.get_gsc_ctr_opportunity_pages(conn, ctx, {})["total"] == 1 + + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value={"google_refresh_token": "t"}), patch( + "website_profiling.tools.audit_tools.integration_tools.build_credentials", return_value=None, + ): + assert int_mod.get_gsc_url_inspection(conn, ctx, {"url": "https://ex.com"})["missing"] + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value={"google_refresh_token": "t", "canonical_domain": ""}), patch( + "website_profiling.tools.audit_tools.integration_tools.build_credentials", return_value=object(), + ): + assert "GSC site URL" in int_mod.get_gsc_url_inspection(conn, ctx, {"url": "https://ex.com"})["error"] + assert int_mod.get_bing_index_status(conn, Ctx(property_id=None), {"url": "https://ex.com"})["missing"] + with patch("website_profiling.tools.audit_tools.integration_tools.get_property_by_id", return_value=None): + assert int_mod.get_bing_index_status(conn, ctx, {"url": "https://ex.com"})["missing"] + + with patch.object(Ctx, "load_keywords", return_value={"rows": []}): + assert int_mod.get_serp_feature_overlay(conn, ctx, {})["total"] == 0 + + with patch.object(Ctx, "load_payload", return_value={"categories": ["skip", {"issues": ["skip"]}]}), patch.object( + Ctx, "resolve_property_domain", return_value="ex.com", + ): + cite = int_mod.check_ai_citation_presence(conn, ctx, {}) + assert cite["query"] == "ex.com" + + skip_df = pd.DataFrame([{"url": "https://ex.com/s", "status": "404", "page_analysis": "{}"}]) + with patch.object(Ctx, "load_crawl_df", return_value=skip_df): + assert geo_mod.list_pages_missing_faq_schema(conn, ctx, {})["total"] == 0 + assert geo_mod.get_eeat_signals_summary(conn, ctx, {})["about_contact_pages"] == 0 + + js_df = pd.DataFrame([ + {"url": "https://ex.com/js", "status": "200", "fetch_method": "static", "title": "A", "word_count": 10, "h1": "A"}, + {"url": "https://ex.com/js", "status": "200", "fetch_method": "rendered", "title": "B", "word_count": 100, "h1": "B"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=js_df): + deltas = geo_mod.get_js_rendering_delta(conn, ctx, {}) + assert deltas["total"] == 1 + + link_df = pd.DataFrame([ + {"url": "https://ex.com/a", "status": "200", "title": "Alpha widgets", "content_excerpt": "widgets alpha beta", "h1": "Alpha"}, + {"url": "https://ex.com/b", "status": "200", "title": "Beta widgets", "content_excerpt": "widgets beta gamma", "h1": "Beta"}, + {"url": "https://ex.com/c", "status": "200", "title": "Gamma", "content_excerpt": "gamma delta", "h1": "Gamma"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=link_df): + links = geo_mod.get_internal_link_suggestions(conn, ctx, {"url": "https://ex.com/a"}) + assert links["suggestions"] + + with patch.object(Ctx, "load_payload", return_value={"categories": []}), patch( + "website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}, + ): + assert llm_mod.prioritize_fix_roadmap(conn, ctx, {"limit": "bad"})["roadmap"] == [] + + with patch("website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}), patch( + "website_profiling.llm.base.get_llm_client", return_value=MagicMock(complete_json=MagicMock(return_value={})), + ), patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}), patch( + "website_profiling.tools.audit_tools.issues.get_category_issues", return_value={"name": "T", "score": 1, "issues": []}, + ), patch("website_profiling.llm.base.parse_json_response", return_value={"summary": "parsed"}): + summary = llm_mod.summarize_category_for_client(conn, ctx, {"category_id": "t"}) + assert summary.get("narrative") == "parsed" + + with patch.object(Ctx, "load_payload", return_value={"site_name": "Ex"}), patch( + "website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}, + ), patch("website_profiling.llm.base.get_llm_client", return_value=MagicMock(complete_json=MagicMock(side_effect=RuntimeError("x")))): + draft = llm_mod.draft_llms_txt(conn, ctx, {}) + assert "Ex" in draft["llms_txt_draft"] + + assert llm_mod.analyze_serp_snippet_for_url(conn, ctx, {})["error"] == "url is required" + + read_skip = pd.DataFrame([{"url": "https://ex.com/x", "status": "404", "reading_level": 9}]) + with patch.object(Ctx, "load_crawl_df", return_value=read_skip): + assert cm_mod.get_readability_summary(conn, ctx, {})["pages_with_reading_level"] == 0 + + with patch.object(Ctx, "load_crawl_df", return_value=None): + assert geo_mod.get_aeo_content_signals_for_url(conn, ctx, {"url": "https://ex.com"})["error"] == "no crawl data" + + bad_wc = pd.DataFrame([{ + "url": "https://ex.com/badwc", + "status": "200", + "content_excerpt": "plain text", + "word_count": "many", + "page_analysis": "{}", + }]) + with patch.object(Ctx, "load_crawl_df", return_value=bad_wc): + assert geo_mod.get_aeo_content_signals_for_url(conn, ctx, {"url": "https://ex.com/badwc"})["word_count"] == 0 + + js_blank = pd.DataFrame([{"url": "", "status": "200", "fetch_method": "static", "title": "x", "word_count": 1, "h1": "x"}]) + with patch.object(Ctx, "load_crawl_df", return_value=js_blank): + assert geo_mod.get_js_rendering_delta(conn, ctx, {})["total"] == 0 + + sparse_links = pd.DataFrame([ + {"url": "https://ex.com/a", "status": "404", "title": "", "content_excerpt": "", "h1": ""}, + {"url": "https://ex.com/b", "status": "200", "title": "B", "content_excerpt": "only one valid page", "h1": "B"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=sparse_links): + assert geo_mod.get_internal_link_suggestions(conn, ctx, {"url": "https://ex.com/b"})["note"] + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert geo_mod.get_internal_link_suggestions(conn, ctx, {"url": "https://ex.com/x"})["error"] == "no crawl data" + + no_token_rows = pd.DataFrame([ + {"url": "https://ex.com/short", "status": "200", "title": "ab", "h1": "cd", "content_excerpt": "ef"}, + {"url": "https://ex.com/good", "status": "200", "title": "widgets page", "h1": "widgets", "content_excerpt": "widgets content here"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=no_token_rows): + assert geo_mod.get_internal_link_suggestions(conn, ctx, {"url": "https://ex.com/good"})["note"] + + assert int_mod.get_serp_feature_overlay(conn, Ctx(property_id=None), {})["missing"] is True + + with patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}), patch( + "website_profiling.llm_config.llm_is_enabled", return_value=True, + ): + assert llm_mod._llm_disabled_response() == {} + + with patch("website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}), patch( + "website_profiling.llm.base.get_llm_client", return_value=MagicMock(complete_json=MagicMock(side_effect=RuntimeError("boom"))), + ), patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}), patch( + "website_profiling.tools.audit_tools.issues.get_category_issues", return_value={"name": "T", "score": 1, "issues": []}, + ): + err_summary = llm_mod.summarize_category_for_client(conn, ctx, {"category_id": "t"}) + assert "narrative_error" in err_summary + + client_ok = MagicMock(complete_json=MagicMock(return_value={"title": "New", "meta_description": "Meta"})) + with patch.object(Ctx, "load_google", return_value={"gsc": {}}), patch.object( + Ctx, "load_crawl_df", return_value=pd.DataFrame([{"url": "https://ex.com", "title": "Old", "meta_description": "Old meta"}]), + ), patch("website_profiling.tools.audit_tools.llm_tools._llm_disabled_response", return_value={}), patch( + "website_profiling.llm.base.get_llm_client", return_value=client_ok, + ), patch("website_profiling.llm_config.load_llm_config_from_db", return_value={}): + serp = llm_mod.analyze_serp_snippet_for_url(conn, ctx, {"url": "https://ex.com"}) + assert serp["provenance"] == "AI insights" + + +def test_compare_slices_error_paths(conn: MagicMock, ctx: Ctx) -> None: + err = {"error": "bad baseline"} + with patch("website_profiling.tools.audit_tools.compare_slices.load_compare_pair", return_value=(None, None, None, None, err)): + assert cmp_mod.compare_indexation_deltas(conn, ctx, {}) == err + assert cmp_mod.compare_orphan_deltas(conn, ctx, {}) == err diff --git a/tests/test_compare_payload.py b/tests/test_compare_payload.py index 0d6723a9..48fb4b69 100644 --- a/tests/test_compare_payload.py +++ b/tests/test_compare_payload.py @@ -9,9 +9,11 @@ build_duplicate_deltas, build_full_compare, build_google_metrics, + build_indexation_deltas, build_issue_deltas, build_lighthouse_url_deltas, build_link_metric_deltas, + build_orphan_deltas, build_priority_counts, build_redirect_deltas, build_security_deltas, @@ -240,6 +242,49 @@ def test_category_scores_skips_invalid() -> None: assert scores[0]["delta"] == -5 +def test_indexation_and_orphan_deltas() -> None: + cur = { + "indexation_coverage": { + "counts": {"crawled": 12, "sitemap": 10}, + "lists": { + "sitemap_only": ["https://ex.com/new-gap"], + "crawled_not_in_sitemap": [], + "gsc_not_crawled": ["https://ex.com/gsc-only"], + }, + }, + "orphan_urls": ["https://ex.com/orphan-a", "https://ex.com/orphan-b"], + } + base = { + "indexation_coverage": { + "counts": {"crawled": 10, "gsc": 8}, + "lists": { + "sitemap_only": ["https://ex.com/old-gap"], + "crawled_not_in_sitemap": ["https://ex.com/crawl-gap"], + "gsc_not_crawled": [], + }, + }, + "orphan_urls": ["https://ex.com/orphan-a"], + } + idx = build_indexation_deltas(cur, base) + assert any(d["metric"] == "crawled" and d["delta"] == 2 for d in idx["count_deltas"]) + assert idx["gap_deltas"]["sitemap_only"]["added_count"] >= 1 + assert idx["gap_deltas"]["crawled_not_in_sitemap"]["removed_count"] >= 1 + + orphans = build_orphan_deltas(cur, base) + assert orphans["added_count"] == 1 + assert orphans["removed_count"] == 0 + assert orphans["delta"] == 1 + + empty = build_orphan_deltas({"orphan_urls": "not-a-list"}, {}) + assert empty["current_count"] == 0 + + bad_counts = build_indexation_deltas( + {"indexation_coverage": {"counts": {"crawled": "many"}, "lists": {}}}, + {"indexation_coverage": {"counts": {"crawled": "few"}, "lists": {}}}, + ) + assert bad_counts["count_deltas"][0]["delta"] is None + + def test_full_compare_truncation() -> None: many_issues = [ {"priority": "Low", "message": f"issue-{i}", "url": f"https://ex.com/p{i}"} diff --git a/tests/test_config_parsing_unit.py b/tests/test_config_parsing_unit.py index 22170125..679c51bc 100644 --- a/tests/test_config_parsing_unit.py +++ b/tests/test_config_parsing_unit.py @@ -24,9 +24,12 @@ def test_load_config_parses_equals_and_colon_and_ignores_comments(tmp_path) -> N def test_getters_bool_int_float_list() -> None: - from website_profiling.config import get_bool, get_float, get_int, get_list + from website_profiling.config import get_bool, get_float, get_int, get_list, get_str - cfg = {"b1": "true", "b2": "0", "i": "10", "f": "1.25", "l": " a, b , ,c "} + cfg = {"b1": "true", "b2": "0", "i": "10", "f": "1.25", "l": " a, b , ,c ", "s": "hello"} + assert get_str(cfg, "s") == "hello" + assert get_str(cfg, "missing", "default") == "default" + assert get_str(cfg, "missing") == "" assert get_bool(cfg, "b1", False) is True assert get_bool(cfg, "b2", True) is False assert get_int(cfg, "i") == 10 diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py index 1e853fdb..67d8fded 100644 --- a/tests/test_mcp_registry.py +++ b/tests/test_mcp_registry.py @@ -7,7 +7,7 @@ def test_tool_definitions_schema() -> None: - assert len(TOOL_DEFINITIONS) == 180 + assert len(TOOL_DEFINITIONS) == 221 for tool in TOOL_DEFINITIONS: assert tool.get("name") assert tool.get("description") diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py index 1a5dfaed..4e5d3749 100644 --- a/tests/test_mcp_server_helpers.py +++ b/tests/test_mcp_server_helpers.py @@ -61,8 +61,10 @@ def test_read_glossary_excerpt_missing(monkeypatch) -> None: def test_tools_catalog_json_includes_security_tools() -> None: catalog = json.loads(mcp_server._tools_catalog_json()) - assert catalog["tool_count"] >= 176 + assert catalog["tool_count"] >= 221 assert "get_security_findings" in catalog["domains"]["security"] + assert "get_geo_readiness_score" in catalog["domains"]["geo"] + assert "get_gsc_url_inspection" in catalog["domains"]["integrations"] def test_tools_catalog_json_backlinks_domain() -> None: @@ -168,7 +170,7 @@ async def __aexit__(self, *_args): assert captured["name"] == "site-audit" assert captured["ran"] is True tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type] - assert len(tools) >= 176 + assert len(tools) >= 221 resources = asyncio.run(captured["list_resources"]()) # type: ignore[arg-type] assert any(r["uri"] == "audit://property/7" for r in resources) From 66bfdee8fcc77dc7e1e649a7c7ad88e6d02a426b Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 00:02:27 +0530 Subject: [PATCH 2/4] fixing weird bug --- src/__main__.py | 2 + .../commands/pipeline_cmd.py | 111 +++++++++---- src/website_profiling/console_io.py | 67 ++++++++ src/website_profiling/lighthouse/runner.py | 106 +++++++----- src/website_profiling/mcp/__main__.py | 2 + src/website_profiling/progress.py | 5 +- .../tools/schedule_runner.py | 10 +- .../test_commands_config_stores_edge_unit.py | 3 +- tests/test_console_io.py | 156 ++++++++++++++++++ tests/test_pipeline_cmd_run_unit.py | 118 +++++++++++++ tests/test_pipeline_report_pool_unit.py | 19 ++- tests/test_progress.py | 7 +- web/src/components/AppShell.tsx | 4 +- web/src/components/PageLayout.tsx | 2 +- web/src/components/ViewTabs.tsx | 2 +- web/src/components/charts/ChartPanel.tsx | 26 +++ .../components/charts/DistributionChart.tsx | 7 +- web/src/components/charts/RankedBarChart.tsx | 9 +- web/src/components/charts/SimpleBarChart.tsx | 11 +- web/src/components/charts/index.ts | 1 + web/src/components/issues/IssueTaskBoard.tsx | 4 +- web/src/components/links/tabs/IssuesTab.tsx | 5 +- .../components/overview/OverviewChartsTab.tsx | 30 ++-- web/src/lib/pipelineDebug.test.ts | 20 +++ web/src/lib/pipelineDebug.ts | 25 ++- web/src/lib/pipelineJobErrorMessage.test.ts | 22 +++ web/src/lib/pipelineJobErrorMessage.ts | 17 ++ web/src/lib/pipelineJobEvents.ts | 11 +- web/src/server/pipelineJobs.ts | 6 +- web/src/server/pipelineSpawnEnv.ts | 2 + web/src/utils/chartJsDefaults.ts | 23 ++- web/src/views/Content.tsx | 8 +- web/src/views/ContentAnalytics.tsx | 106 ++++++------ web/src/views/Issues.tsx | 26 +-- web/src/views/Redirects.tsx | 8 +- web/src/views/Security.tsx | 18 +- web/src/views/TextContentAnalysis.tsx | 57 ++++--- 37 files changed, 828 insertions(+), 228 deletions(-) create mode 100644 src/website_profiling/console_io.py create mode 100644 tests/test_console_io.py create mode 100644 web/src/components/charts/ChartPanel.tsx create mode 100644 web/src/lib/pipelineDebug.test.ts create mode 100644 web/src/lib/pipelineJobErrorMessage.test.ts create mode 100644 web/src/lib/pipelineJobErrorMessage.ts diff --git a/src/__main__.py b/src/__main__.py index 11a86db2..531e0606 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -9,6 +9,8 @@ sys.path.insert(0, str(_root)) from website_profiling.cli import main +from website_profiling.console_io import configure_stdio if __name__ == "__main__": + configure_stdio() main() diff --git a/src/website_profiling/commands/pipeline_cmd.py b/src/website_profiling/commands/pipeline_cmd.py index 6802aa5e..6df9f40c 100644 --- a/src/website_profiling/commands/pipeline_cmd.py +++ b/src/website_profiling/commands/pipeline_cmd.py @@ -3,11 +3,15 @@ import argparse import sys +from collections.abc import Callable +from dataclasses import dataclass +from typing import Literal import pandas as pd from ..config import get_bool, get_float, get_int, get_list -from ..progress import emit_phase_done, emit_phase_start +from ..console_io import console_print +from ..progress import emit_phase_done, emit_phase_start, emit_progress from .config_resolve import ( active_property_id_from_cfg, cleanup_lighthouse_work_dir, @@ -21,10 +25,28 @@ _ALLOWED_RENDER_MODES = frozenset({"static", "javascript", "auto"}) +@dataclass +class PhaseResult: + name: str + status: Literal["ok", "failed"] + error: str | None = None + + +def run_pipeline_phase(name: str, fn: Callable[[], None]) -> PhaseResult: + """Run one pipeline phase; failures are recorded and do not abort the process.""" + try: + fn() + return PhaseResult(name, "ok") + except Exception as e: + emit_progress(name, "error", message=str(e)) + console_print(f"[{name}] failed: {e}", file=sys.stderr) + return PhaseResult(name, "failed", error=str(e)) + + def _normalize_render_mode(cfg: dict) -> str: mode = (cfg.get("crawl_render_mode") or "static").strip().lower() if mode not in _ALLOWED_RENDER_MODES: - print( + console_print( f"Warning: invalid crawl_render_mode {mode!r}; using static.", file=sys.stderr, ) @@ -109,29 +131,62 @@ def run(cfg: dict, args: argparse.Namespace) -> None: steps.append("report") if run_plot: steps.append("plot") - print(f"Site Audit: {', '.join(steps)}", flush=True) + console_print(f"Site Audit: {', '.join(steps)}", flush=True) emit_phase_done("config") + phase_results: list[PhaseResult] = [] + if run_crawl: - _run_crawl(cfg, use_database) + phase_results.append(run_pipeline_phase("crawl", lambda: _run_crawl(cfg, use_database))) if run_lighthouse_on_pages and use_database: - _run_lighthouse_on_pages(cfg, lighthouse_max_pages) + phase_results.append( + run_pipeline_phase( + "lighthouse", + lambda: _run_lighthouse_on_pages(cfg, lighthouse_max_pages), + ) + ) if run_lighthouse and not run_lighthouse_on_pages: - _run_single_lighthouse(cfg, use_database) + phase_results.append( + run_pipeline_phase( + "lighthouse", + lambda: _run_single_lighthouse(cfg, use_database), + ) + ) if run_report: - _run_report(cfg, use_database) + phase_results.append(run_pipeline_phase("report", lambda: _run_report(cfg, use_database))) if run_plot: - _run_plot(cfg, use_database) + phase_results.append(run_pipeline_phase("plot", lambda: _run_plot(cfg, use_database))) + + _finalize_pipeline_run(phase_results) + + +def _finalize_pipeline_run(phase_results: list[PhaseResult]) -> None: + """Exit non-zero only when a critical phase failed; optional phases warn instead.""" + failed = [r for r in phase_results if r.status == "failed"] + if not failed: + return + names = ", ".join(r.name for r in failed) + failed_names = {r.name for r in failed} + report_ok = any(r.name == "report" and r.status == "ok" for r in phase_results) + critical_failed = failed_names & {"crawl", "report"} + if report_ok and not critical_failed: + console_print( + f"Pipeline completed with warnings (optional phase failures: {names})", + file=sys.stderr, + ) + return + console_print(f"Pipeline finished with failures: {names}", file=sys.stderr) + sys.exit(1) def _run_crawl(cfg: dict, use_database: bool) -> None: from ..crawl.crawler import run_crawler - print("[Crawl] Starting...", flush=True) + console_print("[Crawl] Starting...", flush=True) emit_phase_start("crawl") start_url = require_start_url(cfg, for_step="crawl") max_pages = get_int(cfg, "max_pages") @@ -171,7 +226,7 @@ def _run_crawl(cfg: dict, use_database: bool) -> None: custom_extractors = parse_extractors_config(cfg.get("custom_extractors")) enable_axe = get_bool(cfg, "enable_axe", False) - print("Crawling...") + console_print("Crawling...") run_crawler( start_url=start_url, max_pages=max_pages, @@ -215,16 +270,16 @@ def _run_crawl(cfg: dict, use_database: bool) -> None: custom_extractors=custom_extractors or None, enable_axe=enable_axe, ) - print("[Crawl] Done.", flush=True) + console_print("[Crawl] Done.", flush=True) emit_phase_done("crawl") - print("Crawl results: PostgreSQL") + console_print("Crawl results: PostgreSQL") def _run_lighthouse_on_pages(cfg: dict, lighthouse_max_pages: int) -> None: from ..db import db_session, get_latest_crawl_run_id, read_crawl from ..lighthouse.runner import run_lighthouse_on_pages as do_lighthouse_on_pages - print("[Lighthouse on pages] Starting...", flush=True) + console_print("[Lighthouse on pages] Starting...", flush=True) emit_phase_start("lighthouse", message="Lighthouse on pages") with db_session() as conn: run_id = get_latest_crawl_run_id(conn) @@ -242,7 +297,7 @@ def _run_lighthouse_on_pages(cfg: dict, lighthouse_max_pages: int) -> None: if not urls_200: urls_200 = select_lighthouse_urls_from_crawl(df, lighthouse_max_pages) if not urls_200: - print("[Lighthouse on pages] No 200 OK URLs in crawl. Skip.", flush=True) + console_print("[Lighthouse on pages] No 200 OK URLs in crawl. Skip.", flush=True) else: lh_strategy = (cfg.get("lighthouse_strategy") or "mobile").lower() if lh_strategy not in ("mobile", "desktop"): @@ -263,14 +318,14 @@ def _run_lighthouse_on_pages(cfg: dict, lighthouse_max_pages: int) -> None: ) finally: cleanup_lighthouse_work_dir(lh_out) - print("[Lighthouse on pages] Done.", flush=True) + console_print("[Lighthouse on pages] Done.", flush=True) emit_phase_done("lighthouse") def _run_single_lighthouse(cfg: dict, use_database: bool) -> None: from ..lighthouse.runner import main as lighthouse_main - print("[Lighthouse] Starting...", flush=True) + console_print("[Lighthouse] Starting...", flush=True) emit_phase_start("lighthouse") lh_url = require_lighthouse_url(cfg) lh_strategy = (cfg.get("lighthouse_strategy") or "mobile").lower() @@ -293,8 +348,8 @@ def _run_single_lighthouse(cfg: dict, use_database: bool) -> None: finally: cleanup_lighthouse_work_dir(lh_out) if exit_code != 0: - sys.exit(exit_code) - print("[Lighthouse] Done.", flush=True) + raise RuntimeError(f"Lighthouse failed with exit code {exit_code}") + console_print("[Lighthouse] Done.", flush=True) emit_phase_done("lighthouse") @@ -310,7 +365,7 @@ def _run_report(cfg: dict, use_database: bool) -> None: run_security_scan_flag = get_bool(cfg, "run_security_scan", True) security_scan_active = get_bool(cfg, "security_scan_active", False) security_max_urls_probe = get_int(cfg, "security_max_urls_probe", 20) or 20 - print("[Report] Starting...", flush=True) + console_print("[Report] Starting...", flush=True) emit_phase_start("report") out = run_simple_report( max_fetch_for_edges=max_fetch, @@ -328,32 +383,32 @@ def _run_report(cfg: dict, use_database: bool) -> None: use_database=use_database, config=cfg, ) - print("[Report] Done.", flush=True) + console_print("[Report] Done.", flush=True) emit_phase_done("report") - print(f"Report written: {out}") + console_print(f"Report written: {out}") if should_enrich_keywords_after_report(cfg) and google_db_has_gsc(cfg): - print("[Keywords] Post-audit keyword research (Search Console data found)...", flush=True) + console_print("[Keywords] Post-audit keyword research (Search Console data found)...", flush=True) emit_phase_start("keywords") from ..integrations.google.keyword_enrich import run_enrichment try: run_enrichment(cfg) - print("[Keywords] Post-audit keyword research done.", flush=True) + console_print("[Keywords] Post-audit keyword research done.", flush=True) emit_phase_done("keywords") except Exception as e: - print(f"Warning: post-audit keyword research failed: {e}", file=sys.stderr) + console_print(f"Warning: post-audit keyword research failed: {e}", file=sys.stderr) emit_phase_done("keywords", message=f"keywords failed: {e}") def _run_plot(cfg: dict, use_database: bool) -> None: from ..tools.plot import run_plot as do_plot - print("[Plot] Starting...", flush=True) + console_print("[Plot] Starting...", flush=True) emit_phase_start("plot", message="Building charts and link graph") render_mode = (cfg.get("crawl_render_mode") or "").strip().lower() or None if render_mode is not None and render_mode not in _ALLOWED_RENDER_MODES: - print( + console_print( f"Warning: invalid crawl_render_mode {render_mode!r}; using crawl run default.", file=sys.stderr, ) @@ -380,9 +435,9 @@ def _run_plot(cfg: dict, use_database: bool) -> None: js_extra_wait_ms=js_extra_wait_ms, js_block_resources=js_block_resources, ) - print("[Plot] Done.", flush=True) + console_print("[Plot] Done.", flush=True) emit_phase_done("plot", message="Charts and link graph complete") - print(f"Plot data: {e}") + console_print(f"Plot data: {e}") except Exception: emit_phase_done("plot", message="Charts step failed") raise diff --git a/src/website_profiling/console_io.py b/src/website_profiling/console_io.py new file mode 100644 index 00000000..a220c07c --- /dev/null +++ b/src/website_profiling/console_io.py @@ -0,0 +1,67 @@ +"""Safe stdout/stderr for CLI and pipeline jobs (Windows cp1252-safe).""" +from __future__ import annotations + +import json +import sys +from typing import Any, TextIO + + +def configure_stdio() -> None: + """Best-effort UTF-8 stdout/stderr; never raises.""" + for stream in (sys.stdout, sys.stderr): + if stream is None: + continue + if hasattr(stream, "reconfigure"): + try: + stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + + +def _write_bytes(stream: TextIO, text: str, *, end: str = "\n") -> None: + payload = text + end + buffer = getattr(stream, "buffer", None) + if buffer is not None: + buffer.write(payload.encode("utf-8", errors="replace")) + buffer.flush() + return + enc = getattr(stream, "encoding", None) or "utf-8" + stream.write(payload.encode(enc, errors="replace").decode(enc, errors="replace")) + stream.flush() + + +def console_write(stream: TextIO, text: str, *, end: str = "\n") -> None: + """Write human-readable text; never raises UnicodeEncodeError.""" + try: + stream.write(text + end) + stream.flush() + except UnicodeEncodeError: + _write_bytes(stream, text, end=end) + except Exception: + try: + _write_bytes(stream, text, end=end) + except Exception: + pass + + +def console_print(*args: Any, file: TextIO | None = None, **kwargs: Any) -> None: + """Print human-readable text; never raises UnicodeEncodeError.""" + stream = file if file is not None else sys.stdout + end = kwargs.get("end", "\n") + sep = kwargs.get("sep", " ") + text = sep.join(str(a) for a in args) + console_write(stream, text, end=end) + + +def emit_machine_line(prefix: str, payload: dict[str, Any]) -> None: + """Emit a machine-readable stdout line (e.g. @progress JSON); never raises.""" + line = prefix + json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n" + buffer = getattr(sys.stdout, "buffer", None) + if buffer is not None: + try: + buffer.write(line.encode("utf-8", errors="replace")) + buffer.flush() + return + except Exception: + pass + console_write(sys.stdout, line.rstrip("\n"), end="\n") diff --git a/src/website_profiling/lighthouse/runner.py b/src/website_profiling/lighthouse/runner.py index 9b8517f4..c8dd016c 100644 --- a/src/website_profiling/lighthouse/runner.py +++ b/src/website_profiling/lighthouse/runner.py @@ -15,6 +15,8 @@ from datetime import datetime, timezone from typing import Any +from ..console_io import console_print + # Lighthouse "good" thresholds for human summary LCP_GOOD_MS = 2500 CLS_GOOD = 0.1 @@ -166,20 +168,16 @@ def run_lighthouse_once( if categories: cmd.append("--only-categories=" + ",".join(categories)) try: + run_kwargs = { + "capture_output": True, + "encoding": "utf-8", + "errors": "replace", + "timeout": 300, + } if _uses_npx(base): with _NPX_LIGHTHOUSE_LOCK: - return subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=300, - ) - return subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=300, - ) + return subprocess.run(cmd, **run_kwargs) + return subprocess.run(cmd, **run_kwargs) except FileNotFoundError as e: raise RuntimeError(_LIGHTHOUSE_INSTALL_MSG) from e @@ -307,11 +305,11 @@ def run_lighthouse_audit( if strategy not in ("mobile", "desktop"): strategy = "mobile" iterations = max(1, int(iterations)) - print(f"Lighthouse audit: {url} (strategy={strategy}, iterations={iterations})", flush=True) + console_print(f"Lighthouse audit: {url} (strategy={strategy}, iterations={iterations})", flush=True) os.makedirs(output_dir, exist_ok=True) raw_runs_dir = os.path.join(output_dir, "raw_runs") os.makedirs(raw_runs_dir, exist_ok=True) - print(" Output directory ready.", flush=True) + console_print(" Output directory ready.", flush=True) ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") url_slug = _url_safe(url) @@ -320,10 +318,10 @@ def run_lighthouse_audit( categories = _parse_categories(categories) if categories else None if categories: - print(f" Categories: {', '.join(categories)}", flush=True) + console_print(f" Categories: {', '.join(categories)}", flush=True) for i in range(iterations): - print(f" Lighthouse run {i + 1}/{iterations} ({strategy})...", flush=True) + console_print(f" Lighthouse run {i + 1}/{iterations} ({strategy})...", flush=True) out_name = f"lighthouse_{url_slug}_{ts}_run{i + 1}.json" out_path = os.path.join(raw_runs_dir, out_name) proc = run_lighthouse_once(url, strategy, out_path, categories=categories) @@ -340,9 +338,9 @@ def run_lighthouse_audit( except (json.JSONDecodeError, OSError) as e: raise RuntimeError(f"Failed to parse Lighthouse JSON {out_path}: {e}") from e runs.append(extract_from_lighthouse_json(data)) - print(f" Run {i + 1}/{iterations} done.", flush=True) + console_print(f" Run {i + 1}/{iterations} done.", flush=True) - print(" Computing medians and category scores...", flush=True) + console_print(" Computing medians and category scores...", flush=True) # Medians lcps = [r["lcp_ms"] for r in runs if r["lcp_ms"] is not None] clss = [r["cls"] for r in runs if r["cls"] is not None] @@ -390,7 +388,7 @@ def run_lighthouse_audit( human_summary = " ".join(parts) if parts else "No Core Web Vitals metrics extracted." # Diagnostics from first raw run (full Lighthouse JSON) - print(" Building diagnostics from audit results...", flush=True) + console_print(" Building diagnostics from audit results...", flush=True) diagnostics: list[dict[str, Any]] = [] first_raw = raw_paths[0] if raw_paths else None if first_raw and os.path.isfile(first_raw): @@ -401,7 +399,7 @@ def run_lighthouse_audit( diagnostics = parse_lighthouse_to_diagnostics(raw_data)[:15] except Exception: pass - print(f" Found {len(diagnostics)} diagnostics.", flush=True) + console_print(f" Found {len(diagnostics)} diagnostics.", flush=True) # Human summary for file: CWV verdict + Top 5 fixes + quick wins (<400 words) human_lines = [human_summary, ""] @@ -473,7 +471,7 @@ def run_lighthouse_on_pages( workers = max(1, min(int(concurrency or 2), 8)) def _audit_one(url: str) -> None: - print(f"[Lighthouse on pages] {url}", flush=True) + console_print(f"[Lighthouse on pages] {url}", flush=True) summary = run_lighthouse_audit( url=url, strategy=strategy, @@ -508,7 +506,7 @@ def _audit_one(url: str) -> None: if workers == 1: for idx, url in enumerate(urls): try: - print(f"[Lighthouse on pages] {idx + 1}/{total}: {url}", flush=True) + console_print(f"[Lighthouse on pages] {idx + 1}/{total}: {url}", flush=True) emit_progress( "lighthouse", "audit", @@ -518,7 +516,7 @@ def _audit_one(url: str) -> None: ) _audit_one(url) except Exception as e: - print(f" Skipped (error): {e}", file=sys.stderr, flush=True) + console_print(f" Skipped (error): {e}", file=sys.stderr, flush=True) else: completed = 0 with ThreadPoolExecutor(max_workers=workers) as pool: @@ -528,7 +526,7 @@ def _audit_one(url: str) -> None: try: future.result() except Exception as e: - print(f" Skipped {url} (error): {e}", file=sys.stderr, flush=True) + console_print(f" Skipped {url} (error): {e}", file=sys.stderr, flush=True) completed += 1 emit_progress( "lighthouse", @@ -538,7 +536,7 @@ def _audit_one(url: str) -> None: url=url, ) - print(f"[Lighthouse on pages] Done. Wrote {total} URL(s) to DB.", flush=True) + console_print(f"[Lighthouse on pages] Done. Wrote {total} URL(s) to DB.", flush=True) def main( @@ -564,11 +562,37 @@ def main( categories=categories, ) except RuntimeError as e: - print(str(e), file=sys.stderr) + console_print(str(e), file=sys.stderr) + return 1 + except Exception as e: + console_print(f"Lighthouse audit failed: {e}", file=sys.stderr) + return 1 + + try: + return _persist_lighthouse_summary( + summary, + url=url, + strategy=strategy, + output_dir=output_dir, + summary_path=summary_path, + use_database=use_database, + ) + except Exception as e: + console_print(f"Lighthouse persist failed: {e}", file=sys.stderr) return 1 - # Store report HTML in summary so it is saved to PostgreSQL when DATABASE_URL is set - print(" Building report HTML...", flush=True) + +def _persist_lighthouse_summary( + summary: dict[str, Any], + *, + url: str, + strategy: str, + output_dir: str, + summary_path: str | None, + use_database: bool, +) -> int: + """Write Lighthouse artifacts; printing failures do not fail a successful audit.""" + console_print(" Building report HTML...", flush=True) summary["report_html"] = _build_report_html_content(summary) if use_database: @@ -578,13 +602,13 @@ def main( write_lighthouse_summary, write_lighthouse_run, ) - print(" Saving summary to DB...", flush=True) + console_print(" Saving summary to DB...", flush=True) with db_session() as conn: write_lighthouse_summary(conn, summary) raw_reports = summary.get("raw_reports") or [] for i, raw_path in enumerate(raw_reports): if os.path.isfile(raw_path): - print(f" Saving raw run {i + 1}/{len(raw_reports)} to DB...", flush=True) + console_print(f" Saving raw run {i + 1}/{len(raw_reports)} to DB...", flush=True) try: with open(raw_path, "r", encoding="utf-8") as f: run_data = json.load(f) @@ -596,29 +620,31 @@ def main( pass except (OSError, json.JSONDecodeError): pass - print(" Lighthouse DB write complete.", flush=True) - print(summary.get("human_summary", "")) - print(f"All Lighthouse data saved to PostgreSQL (summary, diagnostics, human summary, report HTML, raw runs)") + console_print(" Lighthouse DB write complete.", flush=True) + console_print(summary.get("human_summary", "")) + console_print( + "All Lighthouse data saved to PostgreSQL (summary, diagnostics, human summary, report HTML, raw runs)" + ) else: # No DB: write all artifacts to output_dir - print(" Writing summary.json...", flush=True) + console_print(" Writing summary.json...", flush=True) summary_file = os.path.join(output_dir, "summary.json") with open(summary_file, "w", encoding="utf-8") as f: json.dump(summary, f, indent=2, default=str) - print(" Writing human_summary.txt...", flush=True) + console_print(" Writing human_summary.txt...", flush=True) human_file = os.path.join(output_dir, "human_summary.txt") with open(human_file, "w", encoding="utf-8") as f: f.write(summary.get("human_summary_full", summary.get("human_summary", ""))) - print(" Writing diagnostics.json...", flush=True) + console_print(" Writing diagnostics.json...", flush=True) diag_file = os.path.join(output_dir, "diagnostics.json") with open(diag_file, "w", encoding="utf-8") as f: json.dump(summary.get("diagnostics", []), f, indent=2, default=str) - print(" Writing lighthouse_summary.json...", flush=True) + console_print(" Writing lighthouse_summary.json...", flush=True) out_file = summary_path or os.path.join(output_dir, "lighthouse_summary.json") with open(out_file, "w", encoding="utf-8") as f: json.dump(summary, f, indent=2, default=str) - print(" Writing report.html...", flush=True) + console_print(" Writing report.html...", flush=True) _write_report_html(output_dir, summary) - print(summary.get("human_summary", "")) - print(f"Summary: {summary_file}; diagnostics: {diag_file}; human summary: {human_file}") + console_print(summary.get("human_summary", "")) + console_print(f"Summary: {summary_file}; diagnostics: {diag_file}; human summary: {human_file}") return 0 diff --git a/src/website_profiling/mcp/__main__.py b/src/website_profiling/mcp/__main__.py index f5f6e402..f1ada529 100644 --- a/src/website_profiling/mcp/__main__.py +++ b/src/website_profiling/mcp/__main__.py @@ -1,4 +1,6 @@ from .server import main +from ..console_io import configure_stdio if __name__ == "__main__": + configure_stdio() main() diff --git a/src/website_profiling/progress.py b/src/website_profiling/progress.py index 82f9e2c7..8febfb82 100644 --- a/src/website_profiling/progress.py +++ b/src/website_profiling/progress.py @@ -1,10 +1,11 @@ """Structured pipeline progress events for the web UI (@progress JSON lines).""" from __future__ import annotations -import json import time from typing import Any +from .console_io import emit_machine_line + PREFIX = "@progress " @@ -40,7 +41,7 @@ def emit_progress( payload["elapsed_ms"] = elapsed_ms if avg_ms is not None: payload["avg_ms"] = round(avg_ms, 1) - print(f"{PREFIX}{json.dumps(payload, ensure_ascii=False)}", flush=True) + emit_machine_line(PREFIX, payload) def emit_phase_start(phase: str, message: str | None = None) -> None: diff --git a/src/website_profiling/tools/schedule_runner.py b/src/website_profiling/tools/schedule_runner.py index 7a65fb03..17f4d8d9 100644 --- a/src/website_profiling/tools/schedule_runner.py +++ b/src/website_profiling/tools/schedule_runner.py @@ -43,7 +43,12 @@ def _spawn_audit_for_property(prop_id: int, conn) -> None: known = apply_crawl_preset(preset, known) write_pipeline_config(conn, known, unknown) - env = {**os.environ, "WP_PROPERTY_ID": str(prop_id)} + env = { + **os.environ, + "WP_PROPERTY_ID": str(prop_id), + "PYTHONIOENCODING": "utf-8", + "PYTHONUTF8": "1", + } subprocess.Popen([sys.executable, "-m", "src"], env=env) print(f"[Schedule] Spawned audit for property {prop_id} ({site_url or 'no site_url'})", flush=True) @@ -76,6 +81,9 @@ def run_gsc_links_staleness_alerts() -> list[dict]: def main() -> None: + from ..console_io import configure_stdio + + configure_stdio() n = run_due_scheduled_audits() stale = run_gsc_links_staleness_alerts() print(f"Started {n} scheduled audit(s).") diff --git a/tests/test_commands_config_stores_edge_unit.py b/tests/test_commands_config_stores_edge_unit.py index dbaa148f..1288af1d 100644 --- a/tests/test_commands_config_stores_edge_unit.py +++ b/tests/test_commands_config_stores_edge_unit.py @@ -1109,9 +1109,8 @@ def fake_get_int(cfg, key, default=None): "website_profiling.lighthouse.runner", types.SimpleNamespace(main=lambda **_k: 3), ) - with pytest.raises(SystemExit) as e: + with pytest.raises(RuntimeError, match="Lighthouse failed with exit code 3"): pipeline_cmd._run_single_lighthouse({"lighthouse_strategy": "tablet"}, True) - assert e.value.code == 3 monkeypatch.setitem( sys.modules, diff --git a/tests/test_console_io.py b/tests/test_console_io.py new file mode 100644 index 00000000..ce50b221 --- /dev/null +++ b/tests/test_console_io.py @@ -0,0 +1,156 @@ +"""Tests for encoding-safe console I/O.""" +from __future__ import annotations + +import io +import json +import sys + +import pytest + +from website_profiling.console_io import ( + _write_bytes, + configure_stdio, + console_print, + console_write, + emit_machine_line, +) +from website_profiling.progress import PREFIX, emit_progress + + +def test_configure_stdio_idempotent() -> None: + configure_stdio() + configure_stdio() + + +def test_console_write_survives_cp1252_stream() -> None: + buffer = io.BytesIO() + stream = io.TextIOWrapper(buffer, encoding="cp1252", errors="strict") + console_write(stream, "LCP meets good threshold (≤2500ms).") + output = buffer.getvalue().decode("utf-8", errors="replace") + assert "2500" in output + + +def test_console_print_survives_cp1252_stdout(monkeypatch: pytest.MonkeyPatch) -> None: + buffer = io.BytesIO() + stream = io.TextIOWrapper(buffer, encoding="cp1252", errors="strict") + monkeypatch.setattr(sys, "stdout", stream) + console_print("threshold ≤2500ms") + output = buffer.getvalue().decode("utf-8", errors="replace") + assert "2500" in output + + +def test_emit_machine_line_unicode_url() -> None: + buffer = io.BytesIO() + stream = io.TextIOWrapper(buffer, encoding="cp1252", errors="strict") + old_stdout = sys.stdout + sys.stdout = stream + try: + emit_machine_line(PREFIX, {"phase": "crawl", "step": "fetch", "url": "https://ex.com/café"}) + finally: + sys.stdout = old_stdout + raw = buffer.getvalue() + line = raw.decode("utf-8", errors="replace").strip() + assert line.startswith(PREFIX) + payload = json.loads(line[len(PREFIX) :]) + assert payload["url"] == "https://ex.com/café" + + +def test_emit_progress_prints_json_line(capsys) -> None: + emit_progress("crawl", "fetch", current=3, total=10, url="https://ex.com/a") + out = capsys.readouterr().out.strip() + assert out.startswith(PREFIX) + payload = json.loads(out[len(PREFIX) :]) + assert payload["phase"] == "crawl" + assert payload["step"] == "fetch" + assert payload["current"] == 3 + assert payload["total"] == 10 + assert payload["url"] == "https://ex.com/a" + assert "ts" in payload + + +def test_configure_stdio_skips_none_stream(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "stdout", None) + configure_stdio() + + +def test_configure_stdio_handles_reconfigure_failure(monkeypatch: pytest.MonkeyPatch) -> None: + class BadStream: + encoding = "utf-8" + + def reconfigure(self, **_kwargs: object) -> None: + raise OSError("nope") + + monkeypatch.setattr(sys, "stdout", BadStream()) + monkeypatch.setattr(sys, "stderr", BadStream()) + configure_stdio() + + +def test_console_write_without_buffer() -> None: + class FakeStream: + encoding = "ascii" + writes: list[str] = [] + + def write(self, text: str) -> None: + self.writes.append(text) + + def flush(self) -> None: + pass + + stream = FakeStream() + console_write(stream, "hello") + assert any("hello" in w for w in stream.writes) + + +def test_console_write_survives_broken_stream() -> None: + class BrokenStream: + encoding = "utf-8" + + def write(self, _text: str) -> None: + raise OSError("broken") + + def flush(self) -> None: + raise OSError("broken") + + console_write(BrokenStream(), "safe") + + +def test_write_bytes_without_buffer() -> None: + class FakeStream: + encoding = "ascii" + writes: list[str] = [] + + def write(self, text: str) -> None: + self.writes.append(text) + + def flush(self) -> None: + pass + + stream = FakeStream() + _write_bytes(stream, "hi") + assert any("hi" in w for w in stream.writes) + + +def test_emit_machine_line_falls_back_when_buffer_write_fails(monkeypatch: pytest.MonkeyPatch) -> None: + class BadBuffer: + def write(self, _data: bytes) -> None: + raise OSError("fail") + + def flush(self) -> None: + pass + + class Stream: + buffer = BadBuffer() + encoding = "utf-8" + writes: list[str] = [] + + def write(self, text: str) -> None: + self.writes.append(text) + + def flush(self) -> None: + pass + + stream = Stream() + monkeypatch.setattr(sys, "stdout", stream) + emit_machine_line(PREFIX, {"ok": True}) + assert stream.writes + diff --git a/tests/test_pipeline_cmd_run_unit.py b/tests/test_pipeline_cmd_run_unit.py index 042327dc..4ef58baf 100644 --- a/tests/test_pipeline_cmd_run_unit.py +++ b/tests/test_pipeline_cmd_run_unit.py @@ -1,7 +1,10 @@ import argparse +import io +import sys import types import pandas as pd +import pytest def test_pipeline_run_calls_crawl_with_minimal_config(monkeypatch) -> None: @@ -134,3 +137,118 @@ def __exit__(self, _t, _v, _tb): pipeline_cmd._run_lighthouse_on_pages({}, lighthouse_max_pages=5) assert urls_seen["urls"] == ["https://a.com"] + +def test_pipeline_continues_to_report_when_lighthouse_fails(monkeypatch) -> None: + from website_profiling.commands import pipeline_cmd + + called = {"report": 0} + + monkeypatch.setattr(pipeline_cmd, "_run_crawl", lambda *_a, **_k: None) + monkeypatch.setattr( + pipeline_cmd, + "_run_single_lighthouse", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("lighthouse boom")), + ) + monkeypatch.setattr(pipeline_cmd, "_run_lighthouse_on_pages", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_plot", lambda *_a, **_k: None) + monkeypatch.setattr( + pipeline_cmd, + "_run_report", + lambda *_a, **_k: called.__setitem__("report", called["report"] + 1), + ) + + cfg = { + "start_url": "https://site.com", + "run_crawl": "false", + "run_report": "true", + "run_plot": "false", + "run_lighthouse": "true", + "run_lighthouse_on_pages": "false", + } + args = argparse.Namespace(command=None) + + pipeline_cmd.run(cfg, args) + assert called["report"] == 1 + + +def test_pipeline_exits_when_report_fails(monkeypatch) -> None: + from website_profiling.commands import pipeline_cmd + + monkeypatch.setattr(pipeline_cmd, "_run_crawl", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_single_lighthouse", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_lighthouse_on_pages", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_plot", lambda *_a, **_k: None) + monkeypatch.setattr( + pipeline_cmd, + "_run_report", + lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("report boom")), + ) + + cfg = { + "start_url": "https://site.com", + "run_crawl": "false", + "run_report": "true", + "run_plot": "false", + "run_lighthouse": "false", + "run_lighthouse_on_pages": "false", + } + args = argparse.Namespace(command=None) + + with pytest.raises(SystemExit) as exc: + pipeline_cmd.run(cfg, args) + assert exc.value.code == 1 + + +def test_finalize_pipeline_run_warns_on_optional_failure() -> None: + from website_profiling.commands import pipeline_cmd + + pipeline_cmd._finalize_pipeline_run( + [ + pipeline_cmd.PhaseResult("lighthouse", "failed", error="boom"), + pipeline_cmd.PhaseResult("report", "ok"), + ] + ) + + +def test_finalize_pipeline_run_exits_on_critical_failure() -> None: + from website_profiling.commands import pipeline_cmd + + with pytest.raises(SystemExit) as exc: + pipeline_cmd._finalize_pipeline_run( + [ + pipeline_cmd.PhaseResult("crawl", "failed", error="boom"), + pipeline_cmd.PhaseResult("report", "ok"), + ] + ) + assert exc.value.code == 1 + + +def test_lighthouse_main_prints_unicode_summary_on_cp1252(monkeypatch, tmp_path) -> None: + from website_profiling.lighthouse import runner as lh_runner + + summary = { + "human_summary": "LCP meets good threshold (≤2500ms).", + "human_summary_full": "LCP meets good threshold (≤2500ms).", + "raw_reports": [], + "diagnostics": [], + "median_metrics": {}, + "category_scores": {}, + "top_failures": [], + } + monkeypatch.setattr(lh_runner, "run_lighthouse_audit", lambda **_k: summary) + monkeypatch.setattr(lh_runner, "_build_report_html_content", lambda _s: "") + + buffer = io.BytesIO() + stream = io.TextIOWrapper(buffer, encoding="cp1252", errors="strict") + monkeypatch.setattr(sys, "stdout", stream) + + code = lh_runner.main( + url="https://example.com", + output_dir=str(tmp_path), + use_database=False, + iterations=1, + ) + assert code == 0 + output = buffer.getvalue().decode("utf-8", errors="replace") + assert "2500" in output + diff --git a/tests/test_pipeline_report_pool_unit.py b/tests/test_pipeline_report_pool_unit.py index d92af4de..8c65ae10 100644 --- a/tests/test_pipeline_report_pool_unit.py +++ b/tests/test_pipeline_report_pool_unit.py @@ -41,7 +41,7 @@ def test_pipeline_run_single_lighthouse_when_enabled_in_config(monkeypatch): assert called["lh"] == 1 -def test_run_single_lighthouse_exits_on_nonzero(monkeypatch): +def test_run_single_lighthouse_raises_on_nonzero(monkeypatch): from website_profiling.commands import pipeline_cmd monkeypatch.setattr(pipeline_cmd, "require_lighthouse_url", lambda _cfg: "https://a.com") @@ -50,9 +50,22 @@ def test_run_single_lighthouse_exits_on_nonzero(monkeypatch): monkeypatch.setitem(__import__("sys").modules, "website_profiling.lighthouse.runner", types.SimpleNamespace(main=lambda **_k: 2)) import pytest - with pytest.raises(SystemExit) as e: + with pytest.raises(RuntimeError, match="Lighthouse failed with exit code 2"): pipeline_cmd._run_single_lighthouse({}, True) - assert e.value.code == 2 + + +def test_run_single_lighthouse_completes_on_zero(monkeypatch): + from website_profiling.commands import pipeline_cmd + + monkeypatch.setattr(pipeline_cmd, "require_lighthouse_url", lambda _cfg: "https://a.com") + monkeypatch.setattr(pipeline_cmd, "lighthouse_work_dir", lambda: "/tmp/w") + monkeypatch.setattr(pipeline_cmd, "cleanup_lighthouse_work_dir", lambda _p: None) + monkeypatch.setitem( + __import__("sys").modules, + "website_profiling.lighthouse.runner", + types.SimpleNamespace(main=lambda **_k: 0), + ) + pipeline_cmd._run_single_lighthouse({}, True) def test_run_plot_passes_render_mode_to_run_plot(monkeypatch): diff --git a/tests/test_progress.py b/tests/test_progress.py index 2a0c3515..fbc4c103 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -42,11 +42,10 @@ def test_crawl_tracker_throttles(capsys): def test_emit_phase_done(): - with patch("website_profiling.progress.print") as mock_print: + with patch("website_profiling.progress.emit_machine_line") as mock_emit: emit_phase_done("report") - args = mock_print.call_args[0][0] - assert args.startswith(PREFIX) - payload = json.loads(args[len(PREFIX) :]) + prefix, payload = mock_emit.call_args[0] + assert prefix == PREFIX assert payload["phase"] == "report" assert payload["step"] == "done" diff --git a/web/src/components/AppShell.tsx b/web/src/components/AppShell.tsx index ae8ae5b6..deede263 100644 --- a/web/src/components/AppShell.tsx +++ b/web/src/components/AppShell.tsx @@ -288,8 +288,8 @@ export default function AppShell({ initialToast={integrationsToast} /> -
    -
    {children}
    +
    +
    {children}
    diff --git a/web/src/components/PageLayout.tsx b/web/src/components/PageLayout.tsx index 1a62cd57..3f3cef15 100644 --- a/web/src/components/PageLayout.tsx +++ b/web/src/components/PageLayout.tsx @@ -29,7 +29,7 @@ export default function PageLayout({ const fullHeightAttr = variant === 'fullHeight' ? { 'data-full-height': true } : {}; return (
    {children} diff --git a/web/src/components/ViewTabs.tsx b/web/src/components/ViewTabs.tsx index 6c644f46..527cd011 100644 --- a/web/src/components/ViewTabs.tsx +++ b/web/src/components/ViewTabs.tsx @@ -29,7 +29,7 @@ export default function ViewTabs({ }: ViewTabsProps) { return (
    diff --git a/web/src/components/charts/ChartPanel.tsx b/web/src/components/charts/ChartPanel.tsx new file mode 100644 index 00000000..4070703e --- /dev/null +++ b/web/src/components/charts/ChartPanel.tsx @@ -0,0 +1,26 @@ +import type { ReactNode, CSSProperties } from 'react'; + +export interface ChartPanelProps { + children: ReactNode; + heightClass?: string; + className?: string; + style?: CSSProperties; +} + +/** Clips Chart.js canvases so long axis labels cannot widen the page. */ +export function ChartPanel({ + children, + heightClass = 'h-56', + className = '', + style, +}: ChartPanelProps) { + return ( +
    + {children} +
    + ); +} diff --git a/web/src/components/charts/DistributionChart.tsx b/web/src/components/charts/DistributionChart.tsx index a38a725c..2b0d59f7 100644 --- a/web/src/components/charts/DistributionChart.tsx +++ b/web/src/components/charts/DistributionChart.tsx @@ -6,6 +6,7 @@ import { palette } from '@/utils/chartPalette'; import { barOptionsHorizontal } from '@/utils/chartJsDefaults'; import { doughnutOptionsWithPercentTooltip } from '@/lib/chartDoughnutUtils'; import { ChartAccessibleFallback } from './ChartAccessibleFallback'; +import { ChartPanel } from './ChartPanel'; export interface StatusDistributionChartProps { distribution: StatusDistribution; @@ -45,15 +46,15 @@ export function StatusDistributionChart({ distribution, heightClass = 'h-56' }: return ( -
    + -
    +     ); } diff --git a/web/src/components/charts/RankedBarChart.tsx b/web/src/components/charts/RankedBarChart.tsx index b503eab0..7f74e388 100644 --- a/web/src/components/charts/RankedBarChart.tsx +++ b/web/src/components/charts/RankedBarChart.tsx @@ -2,8 +2,9 @@ import { Bar } from 'react-chartjs-2'; import type { ChartData, ChartOptions } from 'chart.js'; -import { barOptionsHorizontal } from '@/utils/chartJsDefaults'; +import { barOptionsHorizontal, registerChartJsBase } from '@/utils/chartJsDefaults'; import { ChartAccessibleFallback } from './ChartAccessibleFallback'; +import { ChartPanel } from './ChartPanel'; export interface RankedBarChartProps { data: ChartData<'bar'>; @@ -25,9 +26,9 @@ export function RankedBarChart({ return ( -
    - -
    + + +     ); } diff --git a/web/src/components/charts/SimpleBarChart.tsx b/web/src/components/charts/SimpleBarChart.tsx index 865b22dc..fd434a21 100644 --- a/web/src/components/charts/SimpleBarChart.tsx +++ b/web/src/components/charts/SimpleBarChart.tsx @@ -4,6 +4,7 @@ import { useMemo } from 'react'; import { Bar } from 'react-chartjs-2'; import { palette } from '@/utils/chartPalette'; import { barOptionsHorizontal, registerChartJsBase } from '@/utils/chartJsDefaults'; +import { ChartPanel } from './ChartPanel'; registerChartJsBase(); @@ -37,8 +38,12 @@ export function SimpleBarChart({ ); return ( -
    - -
    + + + ); } diff --git a/web/src/components/charts/index.ts b/web/src/components/charts/index.ts index beb35b0e..8e8f5af4 100644 --- a/web/src/components/charts/index.ts +++ b/web/src/components/charts/index.ts @@ -1,4 +1,5 @@ export { ChartAccessibleFallback } from './ChartAccessibleFallback'; +export { ChartPanel } from './ChartPanel'; export { RatioBar, CoverageBar } from './RatioBar'; export { LighthouseScoreGrid } from './LighthouseScoreGrid'; export { StatusDistributionChart } from './DistributionChart'; diff --git a/web/src/components/issues/IssueTaskBoard.tsx b/web/src/components/issues/IssueTaskBoard.tsx index edae5a9b..009ab323 100644 --- a/web/src/components/issues/IssueTaskBoard.tsx +++ b/web/src/components/issues/IssueTaskBoard.tsx @@ -118,13 +118,13 @@ export default function IssueTaskBoard({ propertyId, reportId, issues }: IssueTa return (

    {msg}

    {item.issue.url ? (
    - {item.issue.url} + {item.issue.url}
    ) : null} diff --git a/web/src/components/links/tabs/IssuesTab.tsx b/web/src/components/links/tabs/IssuesTab.tsx index 89d8e345..be24b6c5 100644 --- a/web/src/components/links/tabs/IssuesTab.tsx +++ b/web/src/components/links/tabs/IssuesTab.tsx @@ -113,10 +113,9 @@ export default function IssuesTab({ lhData, inspectorDetails, pageUrl }: IssuesT }, [allIssues, it.typeLabels]); const typeBarOpts = useMemo((): ChartOptions<'bar'> => { - const base = barOptionsHorizontal(); + const base = barOptionsHorizontal(undefined, typeChart.labels); return { ...base, - indexAxis: 'y', plugins: { ...base.plugins, tooltip: { @@ -129,7 +128,7 @@ export default function IssuesTab({ lhData, inspectorDetails, pageUrl }: IssuesT }, }, } as ChartOptions<'bar'>; - }, [it.issueTooltip]); + }, [it.issueTooltip, typeChart.labels]); return (
    diff --git a/web/src/components/overview/OverviewChartsTab.tsx b/web/src/components/overview/OverviewChartsTab.tsx index 87646aa8..176f84b4 100644 --- a/web/src/components/overview/OverviewChartsTab.tsx +++ b/web/src/components/overview/OverviewChartsTab.tsx @@ -5,6 +5,7 @@ import { Bar } from 'react-chartjs-2'; import { strings, format } from '@/lib/strings'; import { Card, StatCard } from '@/components'; import { StatusDistributionChart, LighthouseScoreGrid } from '@/components/charts'; +import { ChartPanel } from '@/components/charts'; import { barOptionsHorizontal } from '@/utils/chartJsDefaults'; import type { ReportPayload } from '@/types'; import type { OverviewChartBlock, OverviewCharts } from './types'; @@ -21,13 +22,16 @@ function OverviewBarChart({ chart: OverviewChartBlock; yTitle: string; }) { + const labels = chart.data.labels?.map(String) ?? []; const opts = chart.horizontal - ? barOptionsHorizontal() + ? barOptionsHorizontal(undefined, labels) : barOptsVertical(yTitle, chart.aria); return ( -
    - -
    + +
    + +
    +     ); } @@ -65,7 +69,7 @@ export function OverviewChartsTab({ charts, depth }: OverviewChartsTabProps) { {vo.insightsGlance}

    {vo.insightsHint}

    -
    +
    {statusDistribution && (

    {vo.statusDist}

    @@ -97,18 +101,22 @@ export function OverviewChartsTab({ charts, depth }: OverviewChartsTabProps) { avgDepth: depth.avg_depth ?? sj.emDash, })}
    -
    - -
    + +
    + +
    +     )} {titleMetaChart && (

    {vo.titleMetaHealth}

    {vo.titleMetaHint}

    -
    - -
    + +
    + +
    +     )} {socialStats && ( diff --git a/web/src/lib/pipelineDebug.test.ts b/web/src/lib/pipelineDebug.test.ts new file mode 100644 index 00000000..1526ab13 --- /dev/null +++ b/web/src/lib/pipelineDebug.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { logPipelineFailure } from '@/lib/pipelineDebug'; + +describe('logPipelineFailure', () => { + it('logs a readable string instead of an empty object', () => { + const spy = vi.spyOn(console, 'error').mockImplementation(() => {}); + logPipelineFailure('Job finished with error', { + jobId: 'abc', + error: 'Process exited with code 1', + logLength: 42, + }); + expect(spy).toHaveBeenCalledWith( + expect.stringContaining('[Site Audit run] Job finished with error'), + ); + expect(spy.mock.calls[0]?.[0]).toContain('jobId=abc'); + expect(spy.mock.calls[0]?.[0]).toContain('error=Process exited with code 1'); + spy.mockRestore(); + }); +}); diff --git a/web/src/lib/pipelineDebug.ts b/web/src/lib/pipelineDebug.ts index fbfed8b5..79607aca 100644 --- a/web/src/lib/pipelineDebug.ts +++ b/web/src/lib/pipelineDebug.ts @@ -3,7 +3,30 @@ export function logPipelineFailure( context: string, details: Record, ): void { - console.error(`[Site Audit run] ${context}`, details); + const parts: string[] = []; + for (const [key, value] of Object.entries(details)) { + if (value === undefined || value === null || value === '') continue; + if (key === 'log' && typeof value === 'string') { + const tail = value.trim().slice(-500); + if (tail) parts.push(`logTail=${tail}`); + continue; + } + if (value instanceof Error) { + parts.push(`${key}=${value.message}`); + continue; + } + if (typeof value === 'string') { + parts.push(`${key}=${value.slice(0, 500)}`); + continue; + } + try { + parts.push(`${key}=${JSON.stringify(value)}`); + } catch { + parts.push(`${key}=[unserializable]`); + } + } + const suffix = parts.length ? `: ${parts.join(' | ')}` : ''; + console.error(`[Site Audit run] ${context}${suffix}`); } export function formatPipelineJobLog(log: string | undefined, error: string | null | undefined): string { diff --git a/web/src/lib/pipelineJobErrorMessage.test.ts b/web/src/lib/pipelineJobErrorMessage.test.ts new file mode 100644 index 00000000..efb12ce7 --- /dev/null +++ b/web/src/lib/pipelineJobErrorMessage.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; + +import { buildPipelineJobErrorMessage } from '@/lib/pipelineJobErrorMessage'; + +describe('buildPipelineJobErrorMessage', () => { + it('includes a failure hint from the log tail', () => { + const log = '...\n[lighthouse] failed: Lighthouse failed with exit code 2\n'; + expect(buildPipelineJobErrorMessage(log, 1)).toContain('[lighthouse] failed'); + expect(buildPipelineJobErrorMessage(log, 1)).toContain('Process exited with code 1'); + }); + + it('falls back to log tail when no hint line exists', () => { + const log = 'line one\nline two\nfinal output'; + const message = buildPipelineJobErrorMessage(log, 3); + expect(message).toContain('Process exited with code 3'); + expect(message).toContain('final output'); + }); + + it('handles empty logs', () => { + expect(buildPipelineJobErrorMessage('', null)).toBe('Process exited with code unknown (no output captured)'); + }); +}); diff --git a/web/src/lib/pipelineJobErrorMessage.ts b/web/src/lib/pipelineJobErrorMessage.ts new file mode 100644 index 00000000..a97fbc2a --- /dev/null +++ b/web/src/lib/pipelineJobErrorMessage.ts @@ -0,0 +1,17 @@ +/** Build a user-visible error when a Python pipeline subprocess exits non-zero. */ +export function buildPipelineJobErrorMessage(log: string, exitCode: number | null): string { + const code = exitCode ?? 'unknown'; + const tail = log.trim().slice(-3000); + const lines = tail + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + const hint = [...lines].reverse().find((line) => + /Pipeline finished with failures|Pipeline completed with warnings|\] failed:|Error:|Traceback \(most recent/i.test( + line, + ), + ); + if (hint) return `Process exited with code ${code}: ${hint.slice(0, 400)}`; + if (tail) return `Process exited with code ${code}\n\n${tail.slice(-600)}`; + return `Process exited with code ${code} (no output captured)`; +} diff --git a/web/src/lib/pipelineJobEvents.ts b/web/src/lib/pipelineJobEvents.ts index 25af0273..f9fb09db 100644 --- a/web/src/lib/pipelineJobEvents.ts +++ b/web/src/lib/pipelineJobEvents.ts @@ -1,6 +1,6 @@ import { apiUrl } from '@/lib/publicBase'; import type { PipelineJob } from '@/types/api'; -import { formatPipelineJobLog, logPipelineFailure } from '@/lib/pipelineDebug'; +import { logPipelineFailure } from '@/lib/pipelineDebug'; export interface PipelineJobStartedDetail { jobId: string; @@ -83,15 +83,6 @@ export function pollPipelineJob( const error = data.error ?? null; onUpdate({ status, log, error }); if (status === 'success' || status === 'error') { - if (status === 'error') { - logPipelineFailure('Job poll terminal error', { - jobId, - status, - error, - logLength: log.length, - displayLog: formatPipelineJobLog(log, error), - }); - } finish(); } } catch (e) { diff --git a/web/src/server/pipelineJobs.ts b/web/src/server/pipelineJobs.ts index 3ecb1799..11e50839 100644 --- a/web/src/server/pipelineJobs.ts +++ b/web/src/server/pipelineJobs.ts @@ -4,6 +4,7 @@ import fs from 'fs'; import { randomUUID } from 'crypto'; import { getPipelineSpawnEnv } from '@/server/pipelineSpawnEnv'; import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython'; +import { buildPipelineJobErrorMessage } from '@/lib/pipelineJobErrorMessage'; import { appendPipelineJobLog, cancelPipelineJobInDb, @@ -224,10 +225,7 @@ export function startPipelineJob( const status = code === 0 ? 'success' : 'error'; let error: string | undefined; if (code !== 0) { - const tail = entry.log.trim().slice(-500); - error = tail - ? `Process exited with code ${code ?? 'unknown'}` - : `Process exited with code ${code ?? 'unknown'} (no output captured)`; + error = buildPipelineJobErrorMessage(entry.log, code); } markJobFinished(id, entry, status, code, error); }); diff --git a/web/src/server/pipelineSpawnEnv.ts b/web/src/server/pipelineSpawnEnv.ts index 61740acc..8422d085 100644 --- a/web/src/server/pipelineSpawnEnv.ts +++ b/web/src/server/pipelineSpawnEnv.ts @@ -17,6 +17,8 @@ export function getPipelineSpawnEnv( DATA_DIR: getDataDir(), // Required for `python -c "from website_profiling ..."` (browser-status, exports, etc.). PYTHONPATH: path.join(repoRoot, 'src'), + PYTHONIOENCODING: 'utf-8', + PYTHONUTF8: '1', }; if (propertyId != null && Number.isFinite(propertyId)) { env.WP_PROPERTY_ID = String(propertyId); diff --git a/web/src/utils/chartJsDefaults.ts b/web/src/utils/chartJsDefaults.ts index c3709e97..48e841df 100644 --- a/web/src/utils/chartJsDefaults.ts +++ b/web/src/utils/chartJsDefaults.ts @@ -49,10 +49,29 @@ export function registerChartJsBase(): void { registered = true; } +/** Truncate long axis labels so horizontal bar charts do not widen the page. */ +export function truncateChartLabel(label: string, maxLength = 28): string { + if (label.length <= maxLength) return label; + return `${label.slice(0, Math.max(1, maxLength - 1))}…`; +} + /** Horizontal bar defaults: frequency on X, labels on Y */ -export function barOptionsHorizontal(tooltipLabel?: string): ChartOptionsLoose { +export function barOptionsHorizontal( + tooltipLabel?: string, + yAxisLabels?: readonly string[], + maxLabelLength = 28, +): ChartOptionsLoose { const grid = getGridColor(); const titleColor = getChartTitleColor(); + const yScale: Record = { grid: { color: grid } }; + if (yAxisLabels?.length) { + yScale.ticks = { + callback: (_value: unknown, index: number) => { + const label = yAxisLabels[index]; + return label ? truncateChartLabel(String(label), maxLabelLength) : ''; + }, + }; + } return { indexAxis: 'y', responsive: true, @@ -68,7 +87,7 @@ export function barOptionsHorizontal(tooltipLabel?: string): ChartOptionsLoose { }, scales: { x: { grid: { color: grid }, beginAtZero: true, title: { display: true, text: 'Count', color: titleColor } }, - y: { grid: { color: grid } }, + y: yScale, }, }; } diff --git a/web/src/views/Content.tsx b/web/src/views/Content.tsx index 081e0e2b..bb462e02 100644 --- a/web/src/views/Content.tsx +++ b/web/src/views/Content.tsx @@ -70,7 +70,7 @@ export default function Content({ searchQuery = '' }: ViewProps) { }, [contentUrls, CONTENT_FILTERS]); const issueBarOpts = useMemo(() => { - const base = barOptionsHorizontal(); + const base = barOptionsHorizontal(undefined, issueBarData.labels); return { ...base, plugins: { @@ -85,7 +85,7 @@ export default function Content({ searchQuery = '' }: ViewProps) { }, }, }; - }, []); + }, [issueBarData.labels]); const getCount = (key: string) => (contentUrls[key] || []).length; const totalIssues = useMemo( @@ -184,10 +184,10 @@ export default function Content({ searchQuery = '' }: ViewProps) { )} {totalIssues > 0 && ( - +

    {vc.issuesByType}

    {vc.issuesByTypeHint}

    -
    +
    = { grid: { color: getGridColor() } }; + if (yAxisLabels?.length) { + yScale.ticks = { + callback: (_value: unknown, index: number) => { + const label = yAxisLabels[index]; + return label ? truncateChartLabel(String(label)) : ''; + }, + }; + } return anyChartOptions({ indexAxis: 'y', responsive: true, @@ -124,7 +134,7 @@ function barOptsH() { }, scales: { x: { grid: { color: getGridColor() }, beginAtZero: true, title: { display: true, text: freq } }, - y: { grid: { color: getGridColor() } }, + y: yScale, }, }); } @@ -648,7 +658,7 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) {

    {vca.languageMix}

    -
    + -
    + )} @@ -672,7 +682,7 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) {

    {vca.entityLabels}

    -
    + -
    +     )} @@ -772,9 +782,9 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { )}

    -
    + -
    +     )} {hasRtDist && ( @@ -794,7 +804,7 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { )}

    -
    + -
    + )}
    @@ -823,28 +833,28 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { {hasIssueBar && (

    URLs flagged by issue type

    -
    + ) => ` ${ctx.raw?.toLocaleString()} URLs` } }, }, }} plugins={[barValueLabelsPlugin]} /> -
    +     )} {hasSeoOptimalBar && (

    Pages in “good” ranges

    -
    + -
    +     )} {hasThinCompare && ( @@ -912,7 +922,7 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) {

    {vca.wordCountDist}

    -
    + {wcLabels.length > 0 ? ( {sj.noData}
    )} -
    +

    {vca.readingLevelDist}

    -
    + {rlLabels.length > 0 ? ( ) => ` ${ctx.raw} pages` } } } }} + options={{ ...barOptsH(rlLabels), plugins: { ...barOptsH(rlLabels).plugins, tooltip: { callbacks: { label: (ctx: TooltipItem<'bar'>) => ` ${ctx.raw} pages` } } } }} plugins={[barValueLabelsPlugin]} /> ) : (
    {sj.noData}
    )} -
    +

    {vca.contentHtmlRatio}

    -
    + {crLabels.length > 0 ? ( {sj.noData}
    )} -
    +

    {vca.topKeywords}

    -
    + {kwLabels.length > 0 ? ( ) : (
    {vca.noKeywordData}
    )} -
    +     {hasWcPercBar && (

    {vca.wordCountLadder}

    -
    + -
    +     )}
    @@ -1023,13 +1033,13 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { {/* H1 Distribution Doughnut */}

    {vca.h1Dist}

    -
    + {hasH1Data ? ( [label, h1Chart.values[i] ?? 0] as [string, string | number])} > -
    +
    {sj.noData}
    )} -
    + {/* Title Length Quality */}

    {vca.titleTagQuality}

    -
    + {hasTitleData ? ( {sj.noData}
    )} -
    +     {/* Meta Description Quality */}

    {vca.metaDescQuality}

    -
    + {hasMetaData ? ( {sj.noData}
    )} -
    +
    {(hasTitleMetaCompare || hasSeoGapCountCompare) && ( -
    +
    {hasTitleMetaCompare && (

    {vca.titleVsMetaBuckets}

    -
    + -
    +     )} {hasSeoGapCountCompare && (

    {vca.seoOptimalVsGapCounts}

    -
    + -
    +     )}
    @@ -1181,7 +1191,7 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { {hasSocialData && (

    {vca.socialOverview}

    -
    + -
    +     )} {hasSocialMissCompare && (

    {vca.missingSocialUrlCompare}

    -
    + -
    +     )} @@ -1382,7 +1392,7 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { )}

    )} -
    + -
    + {/* Word count percentile summary */} diff --git a/web/src/views/Issues.tsx b/web/src/views/Issues.tsx index f5f41835..b0bd57de 100644 --- a/web/src/views/Issues.tsx +++ b/web/src/views/Issues.tsx @@ -52,7 +52,7 @@ function IssueCard({ item, vi, emDash }: IssueCardProps) { const Icon = PRIORITY_ICONS[p]; return (
    @@ -78,7 +78,7 @@ function IssueCard({ item, vi, emDash }: IssueCardProps) {
    {vi.fixRecommendation}
    -

    +

    {iss.llm_recommendation || iss.recommendation || emDash}

    {iss.llm_recommendation && iss.recommendation && iss.llm_recommendation !== iss.recommendation ? ( @@ -246,7 +246,7 @@ export default function Issues({ searchQuery = '' }: ViewProps) { }, [resolvedCategory, priorityFilter, q]); const categoryBarOpts = useMemo(() => { - const base = barOptionsHorizontal(); + const base = barOptionsHorizontal(undefined, categoryChartLabels); return { ...base, plugins: { @@ -261,7 +261,7 @@ export default function Issues({ searchQuery = '' }: ViewProps) { }, }, }; - }, [vi]); + }, [vi, categoryChartLabels]); if (!data) return null; @@ -272,7 +272,7 @@ export default function Issues({ searchQuery = '' }: ViewProps) { })}`; return ( - + - +
    +

    {vi.issuesByCategory}

    {vi.issuesByCategoryHint}

    -
    +
    - +

    {vi.issuesByPriority}

    {vi.issuesByPriorityHint}

    -
    -
    +
    +
    +
    {priorityOrder.map((p) => { const cfg = PRIORITY_CONFIG[p]; const Icon = PRIORITY_ICONS[p]; @@ -418,7 +418,7 @@ export default function Issues({ searchQuery = '' }: ViewProps) {

    {vi.noMatches}

    ) : ( -
    +
    {categoryTabs.length > 1 ? ( { - const base = barOptionsHorizontal(); + const base = barOptionsHorizontal(undefined, statusLabels); return { ...base, plugins: { @@ -56,7 +56,7 @@ export default function Redirects({ searchQuery = '' }: ViewProps) { }, }, }; - }, []); + }, [statusLabels]); if (!data) return null; @@ -64,10 +64,10 @@ export default function Redirects({ searchQuery = '' }: ViewProps) { {redirects.length > 0 && statusLabels.length > 0 && ( - +

    {vr.chartTitle}

    {vr.chartHint}

    -
    +
    p[0]), typeValues: pairs.map((p) => p[1]) }; }, [allFindings]); + const vs = strings.views.security; + const typeBarOpts = useMemo(() => { - const base = barOptionsHorizontal(); + const base = barOptionsHorizontal(undefined, typeLabels); return { ...base, plugins: { @@ -146,16 +148,14 @@ export default function Security({ searchQuery = '' }: ViewProps) { callbacks: { label: (ctx: TooltipItem<'bar'>) => { const n = Number(ctx.raw); - const vs = strings.views.security; return ` ${format(vs.findingTooltip, { n: n.toLocaleString(), s: n !== 1 ? 's' : '' })}`; }, }, }, }, }; - }, []); + }, [vs.findingTooltip, typeLabels]); - const vs = strings.views.security; const vsp = vs.pagination; const filteredFindings = useMemo(() => { @@ -237,11 +237,11 @@ export default function Security({ searchQuery = '' }: ViewProps) { {activeTab === 'charts' && allFindings.length > 0 && ( -
    - +
    +

    {vs.findingsBySeverity}

    {vs.findingsBySeverityHint}

    -
    +
    {typeLabels.length > 0 && ( - +

    {vs.findingsByType}

    {vs.findingsByTypeHint}

    -
    +
    = { grid: { color: getGridColor() } }; + if (yAxisLabels?.length) { + yScale.ticks = { + callback: (_value: unknown, index: number) => { + const label = yAxisLabels[index]; + return label ? truncateChartLabel(String(label)) : ''; + }, + }; + } return anyChartOptions({ indexAxis: 'y', responsive: true, @@ -129,7 +140,7 @@ function barOptsH(xTitle?: string) { grace: '10%', title: { display: true, text: xTitle ?? freq }, }, - y: { grid: { color: getGridColor() } }, + y: yScale, }, }); } @@ -456,7 +467,7 @@ export default function TextContentAnalysis({ searchQuery = '' }: ViewProps) {

    {vtca.keywordFrequencyHist}

    -
    + -
    + ) : null} @@ -488,16 +499,16 @@ export default function TextContentAnalysis({ searchQuery = '' }: ViewProps) { {keywordsChartPagination.total.toLocaleString()} terms

    -
    + -
    + {keywordsChartPagination.total > 0 ? (
    @@ -548,10 +559,10 @@ export default function TextContentAnalysis({ searchQuery = '' }: ViewProps) { )} -
    +

    {vtca.wordCountDist}

    -
    + {wcLabels.length > 0 ? ( {sj.noData}
    )} -
    +

    {vtca.readingLevelDist}

    -
    + {rlLabels.length > 0 ? ( ) => ` ${ctx.raw} pages` } }, }, }} @@ -585,12 +596,12 @@ export default function TextContentAnalysis({ searchQuery = '' }: ViewProps) { ) : (
    {sj.noData}
    )} -
    +

    {vtca.contentHtmlRatio}

    -
    + {crLabels.length > 0 ? ( {sj.noData}
    )} -
    + {hasWcPercBar ? (

    {vtca.wordCountLadder}

    -
    + -
    +     ) : null}
    @@ -647,7 +658,7 @@ export default function TextContentAnalysis({ searchQuery = '' }: ViewProps) {

    {vtca.languageMix}

    -
    + -
    +     ) : null} @@ -665,15 +676,15 @@ export default function TextContentAnalysis({ searchQuery = '' }: ViewProps) {

    {vtca.entityLabels}

    -
    + -
    + ) : null} From 3ee7f2eabc824b36ce7e9a78b69999d8034c6b3c Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 00:26:51 +0530 Subject: [PATCH 3/4] updates --- .agents/skills/data-visualization/SKILL.md | 305 -- .../SKILL.md | 17 - .../python-performance-optimization/SKILL.md | 851 ----- .agents/skills/seo-audit/SKILL.md | 412 --- .agents/skills/seo-audit/evals/evals.json | 136 - .../references/ai-writing-detection.md | 200 - .../vercel-react-best-practices/AGENTS.md | 3254 ----------------- .../vercel-react-best-practices/README.md | 123 - .../vercel-react-best-practices/SKILL.md | 141 - .../rules/_sections.md | 46 - .../rules/_template.md | 28 - .../rules/advanced-event-handler-refs.md | 55 - .../rules/advanced-init-once.md | 42 - .../rules/advanced-use-latest.md | 39 - .../rules/async-api-routes.md | 38 - .../rules/async-defer-await.md | 80 - .../rules/async-dependencies.md | 51 - .../rules/async-parallel.md | 28 - .../rules/async-suspense-boundaries.md | 99 - .../rules/bundle-barrel-imports.md | 59 - .../rules/bundle-conditional.md | 31 - .../rules/bundle-defer-third-party.md | 49 - .../rules/bundle-dynamic-imports.md | 35 - .../rules/bundle-preload.md | 50 - .../rules/client-event-listeners.md | 74 - .../rules/client-localstorage-schema.md | 71 - .../rules/client-passive-event-listeners.md | 48 - .../rules/client-swr-dedup.md | 56 - .../rules/js-batch-dom-css.md | 107 - .../rules/js-cache-function-results.md | 80 - .../rules/js-cache-property-access.md | 28 - .../rules/js-cache-storage.md | 70 - .../rules/js-combine-iterations.md | 32 - .../rules/js-early-exit.md | 50 - .../rules/js-flatmap-filter.md | 60 - .../rules/js-hoist-regexp.md | 45 - .../rules/js-index-maps.md | 37 - .../rules/js-length-check-first.md | 49 - .../rules/js-min-max-loop.md | 82 - .../rules/js-set-map-lookups.md | 24 - .../rules/js-tosorted-immutable.md | 57 - .../rules/rendering-activity.md | 26 - .../rules/rendering-animate-svg-wrapper.md | 47 - .../rules/rendering-conditional-render.md | 40 - .../rules/rendering-content-visibility.md | 38 - .../rules/rendering-hoist-jsx.md | 46 - .../rules/rendering-hydration-no-flicker.md | 82 - .../rendering-hydration-suppress-warning.md | 30 - .../rules/rendering-resource-hints.md | 85 - .../rules/rendering-script-defer-async.md | 68 - .../rules/rendering-svg-precision.md | 28 - .../rules/rendering-usetransition-loading.md | 75 - .../rules/rerender-defer-reads.md | 39 - .../rules/rerender-dependencies.md | 45 - .../rules/rerender-derived-state-no-effect.md | 40 - .../rules/rerender-derived-state.md | 29 - .../rules/rerender-functional-setstate.md | 74 - .../rules/rerender-lazy-state-init.md | 58 - .../rules/rerender-memo-with-default-value.md | 38 - .../rules/rerender-memo.md | 44 - .../rules/rerender-move-effect-to-event.md | 45 - .../rules/rerender-no-inline-components.md | 82 - .../rerender-simple-expression-in-memo.md | 35 - .../rules/rerender-transitions.md | 40 - .../rerender-use-ref-transient-values.md | 73 - .../rules/server-after-nonblocking.md | 73 - .../rules/server-auth-actions.md | 96 - .../rules/server-cache-lru.md | 41 - .../rules/server-cache-react.md | 76 - .../rules/server-dedup-props.md | 65 - .../rules/server-hoist-static-io.md | 142 - .../rules/server-parallel-fetching.md | 83 - .../rules/server-serialization.md | 38 - .gitignore | 2 + scripts/lighthouse_user_flow.mjs | 114 + skills-lock.json | 30 - .../commands/config_resolve.py | 44 +- .../commands/lighthouse_cmd.py | 8 +- .../commands/pipeline_cmd.py | 75 +- src/website_profiling/crawl/crawler.py | 31 +- src/website_profiling/lighthouse/runner.py | 142 +- .../tools/schedule_runner.py | 71 +- .../test_commands_config_stores_edge_unit.py | 4 +- tests/test_config_resolve_more.py | 30 + tests/test_lighthouse_runner_flow_unit.py | 111 + tests/test_pipeline_cmd_run_unit.py | 8 +- .../test_pipeline_lighthouse_url_selection.py | 7 + tests/test_schedule_runner.py | 46 +- web/src/lib/pipelineConfigSchema.ts | 2 + 89 files changed, 603 insertions(+), 9082 deletions(-) delete mode 100644 .agents/skills/data-visualization/SKILL.md delete mode 100644 .agents/skills/dataverse-python-advanced-patterns/SKILL.md delete mode 100644 .agents/skills/python-performance-optimization/SKILL.md delete mode 100644 .agents/skills/seo-audit/SKILL.md delete mode 100644 .agents/skills/seo-audit/evals/evals.json delete mode 100644 .agents/skills/seo-audit/references/ai-writing-detection.md delete mode 100644 .agents/skills/vercel-react-best-practices/AGENTS.md delete mode 100644 .agents/skills/vercel-react-best-practices/README.md delete mode 100644 .agents/skills/vercel-react-best-practices/SKILL.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/_sections.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/_template.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/advanced-event-handler-refs.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/advanced-init-once.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/advanced-use-latest.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/async-api-routes.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/async-defer-await.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/async-dependencies.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/async-parallel.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/async-suspense-boundaries.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/bundle-barrel-imports.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/bundle-conditional.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/bundle-defer-third-party.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/bundle-dynamic-imports.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/bundle-preload.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/client-event-listeners.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/client-localstorage-schema.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/client-passive-event-listeners.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/client-swr-dedup.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-batch-dom-css.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-cache-function-results.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-cache-property-access.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-cache-storage.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-combine-iterations.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-early-exit.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-flatmap-filter.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-hoist-regexp.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-index-maps.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-length-check-first.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-min-max-loop.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-set-map-lookups.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/js-tosorted-immutable.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-activity.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-animate-svg-wrapper.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-conditional-render.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-content-visibility.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-hoist-jsx.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-hydration-no-flicker.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-hydration-suppress-warning.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-resource-hints.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-script-defer-async.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-svg-precision.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rendering-usetransition-loading.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-defer-reads.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-dependencies.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-derived-state-no-effect.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-derived-state.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-functional-setstate.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-lazy-state-init.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-memo-with-default-value.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-memo.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-move-effect-to-event.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-no-inline-components.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-simple-expression-in-memo.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-transitions.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/rerender-use-ref-transient-values.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-after-nonblocking.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-auth-actions.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-cache-lru.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-cache-react.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-dedup-props.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-hoist-static-io.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-parallel-fetching.md delete mode 100644 .agents/skills/vercel-react-best-practices/rules/server-serialization.md create mode 100644 scripts/lighthouse_user_flow.mjs delete mode 100644 skills-lock.json create mode 100644 tests/test_lighthouse_runner_flow_unit.py diff --git a/.agents/skills/data-visualization/SKILL.md b/.agents/skills/data-visualization/SKILL.md deleted file mode 100644 index 409cce60..00000000 --- a/.agents/skills/data-visualization/SKILL.md +++ /dev/null @@ -1,305 +0,0 @@ ---- -name: data-visualization -description: Create effective data visualizations with Python (matplotlib, seaborn, plotly). Use when building charts, choosing the right chart type for a dataset, creating publication-quality figures, or applying design principles like accessibility and color theory. -user-invocable: false ---- - -# Data Visualization Skill - -Chart selection guidance, Python visualization code patterns, design principles, and accessibility considerations for creating effective data visualizations. - -## Chart Selection Guide - -### Choose by Data Relationship - -| What You're Showing | Best Chart | Alternatives | -|---|---|---| -| **Trend over time** | Line chart | Area chart (if showing cumulative or composition) | -| **Comparison across categories** | Vertical bar chart | Horizontal bar (many categories), lollipop chart | -| **Ranking** | Horizontal bar chart | Dot plot, slope chart (comparing two periods) | -| **Part-to-whole composition** | Stacked bar chart | Treemap (hierarchical), waffle chart | -| **Composition over time** | Stacked area chart | 100% stacked bar (for proportion focus) | -| **Distribution** | Histogram | Box plot (comparing groups), violin plot, strip plot | -| **Correlation (2 variables)** | Scatter plot | Bubble chart (add 3rd variable as size) | -| **Correlation (many variables)** | Heatmap (correlation matrix) | Pair plot | -| **Geographic patterns** | Choropleth map | Bubble map, hex map | -| **Flow / process** | Sankey diagram | Funnel chart (sequential stages) | -| **Relationship network** | Network graph | Chord diagram | -| **Performance vs. target** | Bullet chart | Gauge (single KPI only) | -| **Multiple KPIs at once** | Small multiples | Dashboard with separate charts | - -### When NOT to Use Certain Charts - -- **Pie charts**: Avoid unless <6 categories and exact proportions matter less than rough comparison. Humans are bad at comparing angles. Use bar charts instead. -- **3D charts**: Never. They distort perception and add no information. -- **Dual-axis charts**: Use cautiously. They can mislead by implying correlation. Clearly label both axes if used. -- **Stacked bar (many categories)**: Hard to compare middle segments. Use small multiples or grouped bars instead. -- **Donut charts**: Slightly better than pie charts but same fundamental issues. Use for single KPI display at most. - -## Python Visualization Code Patterns - -### Setup and Style - -```python -import matplotlib.pyplot as plt -import matplotlib.ticker as mticker -import seaborn as sns -import pandas as pd -import numpy as np - -# Professional style setup -plt.style.use('seaborn-v0_8-whitegrid') -plt.rcParams.update({ - 'figure.figsize': (10, 6), - 'figure.dpi': 150, - 'font.size': 11, - 'axes.titlesize': 14, - 'axes.titleweight': 'bold', - 'axes.labelsize': 11, - 'xtick.labelsize': 10, - 'ytick.labelsize': 10, - 'legend.fontsize': 10, - 'figure.titlesize': 16, -}) - -# Colorblind-friendly palettes -PALETTE_CATEGORICAL = ['#4C72B0', '#DD8452', '#55A868', '#C44E52', '#8172B3', '#937860'] -PALETTE_SEQUENTIAL = 'YlOrRd' -PALETTE_DIVERGING = 'RdBu_r' -``` - -### Line Chart (Time Series) - -```python -fig, ax = plt.subplots(figsize=(10, 6)) - -for label, group in df.groupby('category'): - ax.plot(group['date'], group['value'], label=label, linewidth=2) - -ax.set_title('Metric Trend by Category', fontweight='bold') -ax.set_xlabel('Date') -ax.set_ylabel('Value') -ax.legend(loc='upper left', frameon=True) -ax.spines['top'].set_visible(False) -ax.spines['right'].set_visible(False) - -# Format dates on x-axis -fig.autofmt_xdate() - -plt.tight_layout() -plt.savefig('trend_chart.png', dpi=150, bbox_inches='tight') -``` - -### Bar Chart (Comparison) - -```python -fig, ax = plt.subplots(figsize=(10, 6)) - -# Sort by value for easy reading -df_sorted = df.sort_values('metric', ascending=True) - -bars = ax.barh(df_sorted['category'], df_sorted['metric'], color=PALETTE_CATEGORICAL[0]) - -# Add value labels -for bar in bars: - width = bar.get_width() - ax.text(width + 0.5, bar.get_y() + bar.get_height()/2, - f'{width:,.0f}', ha='left', va='center', fontsize=10) - -ax.set_title('Metric by Category (Ranked)', fontweight='bold') -ax.set_xlabel('Metric Value') -ax.spines['top'].set_visible(False) -ax.spines['right'].set_visible(False) - -plt.tight_layout() -plt.savefig('bar_chart.png', dpi=150, bbox_inches='tight') -``` - -### Histogram (Distribution) - -```python -fig, ax = plt.subplots(figsize=(10, 6)) - -ax.hist(df['value'], bins=30, color=PALETTE_CATEGORICAL[0], edgecolor='white', alpha=0.8) - -# Add mean and median lines -mean_val = df['value'].mean() -median_val = df['value'].median() -ax.axvline(mean_val, color='red', linestyle='--', linewidth=1.5, label=f'Mean: {mean_val:,.1f}') -ax.axvline(median_val, color='green', linestyle='--', linewidth=1.5, label=f'Median: {median_val:,.1f}') - -ax.set_title('Distribution of Values', fontweight='bold') -ax.set_xlabel('Value') -ax.set_ylabel('Frequency') -ax.legend() -ax.spines['top'].set_visible(False) -ax.spines['right'].set_visible(False) - -plt.tight_layout() -plt.savefig('histogram.png', dpi=150, bbox_inches='tight') -``` - -### Heatmap - -```python -fig, ax = plt.subplots(figsize=(10, 8)) - -# Pivot data for heatmap format -pivot = df.pivot_table(index='row_dim', columns='col_dim', values='metric', aggfunc='sum') - -sns.heatmap(pivot, annot=True, fmt=',.0f', cmap='YlOrRd', - linewidths=0.5, ax=ax, cbar_kws={'label': 'Metric Value'}) - -ax.set_title('Metric by Row Dimension and Column Dimension', fontweight='bold') -ax.set_xlabel('Column Dimension') -ax.set_ylabel('Row Dimension') - -plt.tight_layout() -plt.savefig('heatmap.png', dpi=150, bbox_inches='tight') -``` - -### Small Multiples - -```python -categories = df['category'].unique() -n_cats = len(categories) -n_cols = min(3, n_cats) -n_rows = (n_cats + n_cols - 1) // n_cols - -fig, axes = plt.subplots(n_rows, n_cols, figsize=(5*n_cols, 4*n_rows), sharex=True, sharey=True) -axes = axes.flatten() if n_cats > 1 else [axes] - -for i, cat in enumerate(categories): - ax = axes[i] - subset = df[df['category'] == cat] - ax.plot(subset['date'], subset['value'], color=PALETTE_CATEGORICAL[i % len(PALETTE_CATEGORICAL)]) - ax.set_title(cat, fontsize=12) - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - -# Hide empty subplots -for j in range(i+1, len(axes)): - axes[j].set_visible(False) - -fig.suptitle('Trends by Category', fontsize=14, fontweight='bold', y=1.02) -plt.tight_layout() -plt.savefig('small_multiples.png', dpi=150, bbox_inches='tight') -``` - -### Number Formatting Helpers - -```python -def format_number(val, format_type='number'): - """Format numbers for chart labels.""" - if format_type == 'currency': - if abs(val) >= 1e9: - return f'${val/1e9:.1f}B' - elif abs(val) >= 1e6: - return f'${val/1e6:.1f}M' - elif abs(val) >= 1e3: - return f'${val/1e3:.1f}K' - else: - return f'${val:,.0f}' - elif format_type == 'percent': - return f'{val:.1f}%' - elif format_type == 'number': - if abs(val) >= 1e9: - return f'{val/1e9:.1f}B' - elif abs(val) >= 1e6: - return f'{val/1e6:.1f}M' - elif abs(val) >= 1e3: - return f'{val/1e3:.1f}K' - else: - return f'{val:,.0f}' - return str(val) - -# Usage with axis formatter -ax.yaxis.set_major_formatter(mticker.FuncFormatter(lambda x, p: format_number(x, 'currency'))) -``` - -### Interactive Charts with Plotly - -```python -import plotly.express as px -import plotly.graph_objects as go - -# Simple interactive line chart -fig = px.line(df, x='date', y='value', color='category', - title='Interactive Metric Trend', - labels={'value': 'Metric Value', 'date': 'Date'}) -fig.update_layout(hovermode='x unified') -fig.write_html('interactive_chart.html') -fig.show() - -# Interactive scatter with hover data -fig = px.scatter(df, x='metric_a', y='metric_b', color='category', - size='size_metric', hover_data=['name', 'detail_field'], - title='Correlation Analysis') -fig.show() -``` - -## Design Principles - -### Color - -- **Use color purposefully**: Color should encode data, not decorate -- **Highlight the story**: Use a bright accent color for the key insight; grey everything else -- **Sequential data**: Use a single-hue gradient (light to dark) for ordered values -- **Diverging data**: Use a two-hue gradient with neutral midpoint for data with a meaningful center -- **Categorical data**: Use distinct hues, maximum 6-8 before it gets confusing -- **Avoid red/green only**: 8% of men are red-green colorblind. Use blue/orange as primary pair - -### Typography - -- **Title states the insight**: "Revenue grew 23% YoY" beats "Revenue by Month" -- **Subtitle adds context**: Date range, filters applied, data source -- **Axis labels are readable**: Never rotated 90 degrees if avoidable. Shorten or wrap instead -- **Data labels add precision**: Use on key points, not every single bar -- **Annotation highlights**: Call out specific points with text annotations - -### Layout - -- **Reduce chart junk**: Remove gridlines, borders, backgrounds that don't carry information -- **Sort meaningfully**: Categories sorted by value (not alphabetically) unless there's a natural order (months, stages) -- **Appropriate aspect ratio**: Time series wider than tall (3:1 to 2:1); comparisons can be squarer -- **White space is good**: Don't cram charts together. Give each visualization room to breathe - -### Accuracy - -- **Bar charts start at zero**: Always. A bar from 95 to 100 exaggerates a 5% difference -- **Line charts can have non-zero baselines**: When the range of variation is meaningful -- **Consistent scales across panels**: When comparing multiple charts, use the same axis range -- **Show uncertainty**: Error bars, confidence intervals, or ranges when data is uncertain -- **Label your axes**: Never make the reader guess what the numbers mean - -## Accessibility Considerations - -### Color Blindness - -- Never rely on color alone to distinguish data series -- Add pattern fills, different line styles (solid, dashed, dotted), or direct labels -- Test with a colorblind simulator (e.g., Coblis, Sim Daltonism) -- Use the colorblind-friendly palette: `sns.color_palette("colorblind")` - -### Screen Readers - -- Include alt text describing the chart's key finding -- Provide a data table alternative alongside the visualization -- Use semantic titles and labels - -### General Accessibility - -- Sufficient contrast between data elements and background -- Text size minimum 10pt for labels, 12pt for titles -- Avoid conveying information only through spatial position (add labels) -- Consider printing: does the chart work in black and white? - -### Accessibility Checklist - -Before sharing a visualization: -- [ ] Chart works without color (patterns, labels, or line styles differentiate series) -- [ ] Text is readable at standard zoom level -- [ ] Title describes the insight, not just the data -- [ ] Axes are labeled with units -- [ ] Legend is clear and positioned without obscuring data -- [ ] Data source and date range are noted diff --git a/.agents/skills/dataverse-python-advanced-patterns/SKILL.md b/.agents/skills/dataverse-python-advanced-patterns/SKILL.md deleted file mode 100644 index 921ab603..00000000 --- a/.agents/skills/dataverse-python-advanced-patterns/SKILL.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: dataverse-python-advanced-patterns -description: 'Generate production code for Dataverse SDK using advanced patterns, error handling, and optimization techniques.' ---- - -You are a Dataverse SDK for Python expert. Generate production-ready Python code that demonstrates: - -1. **Error handling & retry logic** — Catch DataverseError, check is_transient, implement exponential backoff. -2. **Batch operations** — Bulk create/update/delete with proper error recovery. -3. **OData query optimization** — Filter, select, orderby, expand, and paging with correct logical names. -4. **Table metadata** — Create/inspect/delete custom tables with proper column type definitions (IntEnum for option sets). -5. **Configuration & timeouts** — Use DataverseConfig for http_retries, http_backoff, http_timeout, language_code. -6. **Cache management** — Flush picklist cache when metadata changes. -7. **File operations** — Upload large files in chunks; handle chunked vs. simple upload. -8. **Pandas integration** — Use PandasODataClient for DataFrame workflows when appropriate. - -Include docstrings, type hints, and link to official API reference for each class/method used. diff --git a/.agents/skills/python-performance-optimization/SKILL.md b/.agents/skills/python-performance-optimization/SKILL.md deleted file mode 100644 index 31b820c0..00000000 --- a/.agents/skills/python-performance-optimization/SKILL.md +++ /dev/null @@ -1,851 +0,0 @@ ---- -name: python-performance-optimization -description: Profile and optimize Python code using cProfile, memory profilers, and performance best practices. Use when debugging slow Python code, optimizing bottlenecks, or improving application performance. ---- - -# Python Performance Optimization - -Comprehensive guide to profiling, analyzing, and optimizing Python code for better performance, including CPU profiling, memory optimization, and implementation best practices. - -## When to Use This Skill - -- Identifying performance bottlenecks in Python applications -- Reducing application latency and response times -- Optimizing CPU-intensive operations -- Reducing memory consumption and memory leaks -- Improving database query performance -- Optimizing I/O operations -- Speeding up data processing pipelines -- Implementing high-performance algorithms -- Profiling production applications - -## Core Concepts - -### 1. Profiling Types - -- **CPU Profiling**: Identify time-consuming functions -- **Memory Profiling**: Track memory allocation and leaks -- **Line Profiling**: Profile at line-by-line granularity -- **Call Graph**: Visualize function call relationships - -### 2. Performance Metrics - -- **Execution Time**: How long operations take -- **Memory Usage**: Peak and average memory consumption -- **CPU Utilization**: Processor usage patterns -- **I/O Wait**: Time spent on I/O operations - -### 3. Optimization Strategies - -- **Algorithmic**: Better algorithms and data structures -- **Implementation**: More efficient code patterns -- **Parallelization**: Multi-threading/processing -- **Caching**: Avoid redundant computation -- **Native Extensions**: C/Rust for critical paths - -## Quick Start - -### Basic Timing - -```python -import time - -def measure_time(): - """Simple timing measurement.""" - start = time.time() - - # Your code here - result = sum(range(1000000)) - - elapsed = time.time() - start - print(f"Execution time: {elapsed:.4f} seconds") - return result - -# Better: use timeit for accurate measurements -import timeit - -execution_time = timeit.timeit( - "sum(range(1000000))", - number=100 -) -print(f"Average time: {execution_time/100:.6f} seconds") -``` - -## Profiling Tools - -### Pattern 1: cProfile - CPU Profiling - -```python -import cProfile -import pstats -from pstats import SortKey - -def slow_function(): - """Function to profile.""" - total = 0 - for i in range(1000000): - total += i - return total - -def another_function(): - """Another function.""" - return [i**2 for i in range(100000)] - -def main(): - """Main function to profile.""" - result1 = slow_function() - result2 = another_function() - return result1, result2 - -# Profile the code -if __name__ == "__main__": - profiler = cProfile.Profile() - profiler.enable() - - main() - - profiler.disable() - - # Print stats - stats = pstats.Stats(profiler) - stats.sort_stats(SortKey.CUMULATIVE) - stats.print_stats(10) # Top 10 functions - - # Save to file for later analysis - stats.dump_stats("profile_output.prof") -``` - -**Command-line profiling:** - -```bash -# Profile a script -python -m cProfile -o output.prof script.py - -# View results -python -m pstats output.prof -# In pstats: -# sort cumtime -# stats 10 -``` - -### Pattern 2: line_profiler - Line-by-Line Profiling - -```python -# Install: pip install line-profiler - -# Add @profile decorator (line_profiler provides this) -@profile -def process_data(data): - """Process data with line profiling.""" - result = [] - for item in data: - processed = item * 2 - result.append(processed) - return result - -# Run with: -# kernprof -l -v script.py -``` - -**Manual line profiling:** - -```python -from line_profiler import LineProfiler - -def process_data(data): - """Function to profile.""" - result = [] - for item in data: - processed = item * 2 - result.append(processed) - return result - -if __name__ == "__main__": - lp = LineProfiler() - lp.add_function(process_data) - - data = list(range(100000)) - - lp_wrapper = lp(process_data) - lp_wrapper(data) - - lp.print_stats() -``` - -### Pattern 3: memory_profiler - Memory Usage - -```python -# Install: pip install memory-profiler - -from memory_profiler import profile - -@profile -def memory_intensive(): - """Function that uses lots of memory.""" - # Create large list - big_list = [i for i in range(1000000)] - - # Create large dict - big_dict = {i: i**2 for i in range(100000)} - - # Process data - result = sum(big_list) - - return result - -if __name__ == "__main__": - memory_intensive() - -# Run with: -# python -m memory_profiler script.py -``` - -### Pattern 4: py-spy - Production Profiling - -```bash -# Install: pip install py-spy - -# Profile a running Python process -py-spy top --pid 12345 - -# Generate flamegraph -py-spy record -o profile.svg --pid 12345 - -# Profile a script -py-spy record -o profile.svg -- python script.py - -# Dump current call stack -py-spy dump --pid 12345 -``` - -## Optimization Patterns - -### Pattern 5: List Comprehensions vs Loops - -```python -import timeit - -# Slow: Traditional loop -def slow_squares(n): - """Create list of squares using loop.""" - result = [] - for i in range(n): - result.append(i**2) - return result - -# Fast: List comprehension -def fast_squares(n): - """Create list of squares using comprehension.""" - return [i**2 for i in range(n)] - -# Benchmark -n = 100000 - -slow_time = timeit.timeit(lambda: slow_squares(n), number=100) -fast_time = timeit.timeit(lambda: fast_squares(n), number=100) - -print(f"Loop: {slow_time:.4f}s") -print(f"Comprehension: {fast_time:.4f}s") -print(f"Speedup: {slow_time/fast_time:.2f}x") - -# Even faster for simple operations: map -def faster_squares(n): - """Use map for even better performance.""" - return list(map(lambda x: x**2, range(n))) -``` - -### Pattern 6: Generator Expressions for Memory - -```python -import sys - -def list_approach(): - """Memory-intensive list.""" - data = [i**2 for i in range(1000000)] - return sum(data) - -def generator_approach(): - """Memory-efficient generator.""" - data = (i**2 for i in range(1000000)) - return sum(data) - -# Memory comparison -list_data = [i for i in range(1000000)] -gen_data = (i for i in range(1000000)) - -print(f"List size: {sys.getsizeof(list_data)} bytes") -print(f"Generator size: {sys.getsizeof(gen_data)} bytes") - -# Generators use constant memory regardless of size -``` - -### Pattern 7: String Concatenation - -```python -import timeit - -def slow_concat(items): - """Slow string concatenation.""" - result = "" - for item in items: - result += str(item) - return result - -def fast_concat(items): - """Fast string concatenation with join.""" - return "".join(str(item) for item in items) - -def faster_concat(items): - """Even faster with list.""" - parts = [str(item) for item in items] - return "".join(parts) - -items = list(range(10000)) - -# Benchmark -slow = timeit.timeit(lambda: slow_concat(items), number=100) -fast = timeit.timeit(lambda: fast_concat(items), number=100) -faster = timeit.timeit(lambda: faster_concat(items), number=100) - -print(f"Concatenation (+): {slow:.4f}s") -print(f"Join (generator): {fast:.4f}s") -print(f"Join (list): {faster:.4f}s") -``` - -### Pattern 8: Dictionary Lookups vs List Searches - -```python -import timeit - -# Create test data -size = 10000 -items = list(range(size)) -lookup_dict = {i: i for i in range(size)} - -def list_search(items, target): - """O(n) search in list.""" - return target in items - -def dict_search(lookup_dict, target): - """O(1) search in dict.""" - return target in lookup_dict - -target = size - 1 # Worst case for list - -# Benchmark -list_time = timeit.timeit( - lambda: list_search(items, target), - number=1000 -) -dict_time = timeit.timeit( - lambda: dict_search(lookup_dict, target), - number=1000 -) - -print(f"List search: {list_time:.6f}s") -print(f"Dict search: {dict_time:.6f}s") -print(f"Speedup: {list_time/dict_time:.0f}x") -``` - -### Pattern 9: Local Variable Access - -```python -import timeit - -# Global variable (slow) -GLOBAL_VALUE = 100 - -def use_global(): - """Access global variable.""" - total = 0 - for i in range(10000): - total += GLOBAL_VALUE - return total - -def use_local(): - """Use local variable.""" - local_value = 100 - total = 0 - for i in range(10000): - total += local_value - return total - -# Local is faster -global_time = timeit.timeit(use_global, number=1000) -local_time = timeit.timeit(use_local, number=1000) - -print(f"Global access: {global_time:.4f}s") -print(f"Local access: {local_time:.4f}s") -print(f"Speedup: {global_time/local_time:.2f}x") -``` - -### Pattern 10: Function Call Overhead - -```python -import timeit - -def calculate_inline(): - """Inline calculation.""" - total = 0 - for i in range(10000): - total += i * 2 + 1 - return total - -def helper_function(x): - """Helper function.""" - return x * 2 + 1 - -def calculate_with_function(): - """Calculation with function calls.""" - total = 0 - for i in range(10000): - total += helper_function(i) - return total - -# Inline is faster due to no call overhead -inline_time = timeit.timeit(calculate_inline, number=1000) -function_time = timeit.timeit(calculate_with_function, number=1000) - -print(f"Inline: {inline_time:.4f}s") -print(f"Function calls: {function_time:.4f}s") -``` - -## Advanced Optimization - -### Pattern 11: NumPy for Numerical Operations - -```python -import timeit -import numpy as np - -def python_sum(n): - """Sum using pure Python.""" - return sum(range(n)) - -def numpy_sum(n): - """Sum using NumPy.""" - return np.arange(n).sum() - -n = 1000000 - -python_time = timeit.timeit(lambda: python_sum(n), number=100) -numpy_time = timeit.timeit(lambda: numpy_sum(n), number=100) - -print(f"Python: {python_time:.4f}s") -print(f"NumPy: {numpy_time:.4f}s") -print(f"Speedup: {python_time/numpy_time:.2f}x") - -# Vectorized operations -def python_multiply(): - """Element-wise multiplication in Python.""" - a = list(range(100000)) - b = list(range(100000)) - return [x * y for x, y in zip(a, b)] - -def numpy_multiply(): - """Vectorized multiplication in NumPy.""" - a = np.arange(100000) - b = np.arange(100000) - return a * b - -py_time = timeit.timeit(python_multiply, number=100) -np_time = timeit.timeit(numpy_multiply, number=100) - -print(f"\nPython multiply: {py_time:.4f}s") -print(f"NumPy multiply: {np_time:.4f}s") -print(f"Speedup: {py_time/np_time:.2f}x") -``` - -### Pattern 12: Caching with functools.lru_cache - -```python -from functools import lru_cache -import timeit - -def fibonacci_slow(n): - """Recursive fibonacci without caching.""" - if n < 2: - return n - return fibonacci_slow(n-1) + fibonacci_slow(n-2) - -@lru_cache(maxsize=None) -def fibonacci_fast(n): - """Recursive fibonacci with caching.""" - if n < 2: - return n - return fibonacci_fast(n-1) + fibonacci_fast(n-2) - -# Massive speedup for recursive algorithms -n = 30 - -slow_time = timeit.timeit(lambda: fibonacci_slow(n), number=1) -fast_time = timeit.timeit(lambda: fibonacci_fast(n), number=1000) - -print(f"Without cache (1 run): {slow_time:.4f}s") -print(f"With cache (1000 runs): {fast_time:.4f}s") - -# Cache info -print(f"Cache info: {fibonacci_fast.cache_info()}") -``` - -### Pattern 13: Using **slots** for Memory - -```python -import sys - -class RegularClass: - """Regular class with __dict__.""" - def __init__(self, x, y, z): - self.x = x - self.y = y - self.z = z - -class SlottedClass: - """Class with __slots__ for memory efficiency.""" - __slots__ = ['x', 'y', 'z'] - - def __init__(self, x, y, z): - self.x = x - self.y = y - self.z = z - -# Memory comparison -regular = RegularClass(1, 2, 3) -slotted = SlottedClass(1, 2, 3) - -print(f"Regular class size: {sys.getsizeof(regular)} bytes") -print(f"Slotted class size: {sys.getsizeof(slotted)} bytes") - -# Significant savings with many instances -regular_objects = [RegularClass(i, i+1, i+2) for i in range(10000)] -slotted_objects = [SlottedClass(i, i+1, i+2) for i in range(10000)] - -print(f"\nMemory for 10000 regular objects: ~{sys.getsizeof(regular) * 10000} bytes") -print(f"Memory for 10000 slotted objects: ~{sys.getsizeof(slotted) * 10000} bytes") -``` - -### Pattern 14: Multiprocessing for CPU-Bound Tasks - -```python -import multiprocessing as mp -import time - -def cpu_intensive_task(n): - """CPU-intensive calculation.""" - return sum(i**2 for i in range(n)) - -def sequential_processing(): - """Process tasks sequentially.""" - start = time.time() - results = [cpu_intensive_task(1000000) for _ in range(4)] - elapsed = time.time() - start - return elapsed, results - -def parallel_processing(): - """Process tasks in parallel.""" - start = time.time() - with mp.Pool(processes=4) as pool: - results = pool.map(cpu_intensive_task, [1000000] * 4) - elapsed = time.time() - start - return elapsed, results - -if __name__ == "__main__": - seq_time, seq_results = sequential_processing() - par_time, par_results = parallel_processing() - - print(f"Sequential: {seq_time:.2f}s") - print(f"Parallel: {par_time:.2f}s") - print(f"Speedup: {seq_time/par_time:.2f}x") -``` - -### Pattern 15: Async I/O for I/O-Bound Tasks - -```python -import asyncio -import aiohttp -import time -import requests - -urls = [ - "https://httpbin.org/delay/1", - "https://httpbin.org/delay/1", - "https://httpbin.org/delay/1", - "https://httpbin.org/delay/1", -] - -def synchronous_requests(): - """Synchronous HTTP requests.""" - start = time.time() - results = [] - for url in urls: - response = requests.get(url) - results.append(response.status_code) - elapsed = time.time() - start - return elapsed, results - -async def async_fetch(session, url): - """Async HTTP request.""" - async with session.get(url) as response: - return response.status - -async def asynchronous_requests(): - """Asynchronous HTTP requests.""" - start = time.time() - async with aiohttp.ClientSession() as session: - tasks = [async_fetch(session, url) for url in urls] - results = await asyncio.gather(*tasks) - elapsed = time.time() - start - return elapsed, results - -# Async is much faster for I/O-bound work -sync_time, sync_results = synchronous_requests() -async_time, async_results = asyncio.run(asynchronous_requests()) - -print(f"Synchronous: {sync_time:.2f}s") -print(f"Asynchronous: {async_time:.2f}s") -print(f"Speedup: {sync_time/async_time:.2f}x") -``` - -## Database Optimization - -### Pattern 16: Batch Database Operations - -```python -import sqlite3 -import time - -def create_db(): - """Create test database.""" - conn = sqlite3.connect(":memory:") - conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") - return conn - -def slow_inserts(conn, count): - """Insert records one at a time.""" - start = time.time() - cursor = conn.cursor() - for i in range(count): - cursor.execute("INSERT INTO users (name) VALUES (?)", (f"User {i}",)) - conn.commit() # Commit each insert - elapsed = time.time() - start - return elapsed - -def fast_inserts(conn, count): - """Batch insert with single commit.""" - start = time.time() - cursor = conn.cursor() - data = [(f"User {i}",) for i in range(count)] - cursor.executemany("INSERT INTO users (name) VALUES (?)", data) - conn.commit() # Single commit - elapsed = time.time() - start - return elapsed - -# Benchmark -conn1 = create_db() -slow_time = slow_inserts(conn1, 1000) - -conn2 = create_db() -fast_time = fast_inserts(conn2, 1000) - -print(f"Individual inserts: {slow_time:.4f}s") -print(f"Batch insert: {fast_time:.4f}s") -print(f"Speedup: {slow_time/fast_time:.2f}x") -``` - -### Pattern 17: Query Optimization - -```python -# Use indexes for frequently queried columns -""" --- Slow: No index -SELECT * FROM users WHERE email = 'user@example.com'; - --- Fast: With index -CREATE INDEX idx_users_email ON users(email); -SELECT * FROM users WHERE email = 'user@example.com'; -""" - -# Use query planning -import sqlite3 - -conn = sqlite3.connect("example.db") -cursor = conn.cursor() - -# Analyze query performance -cursor.execute("EXPLAIN QUERY PLAN SELECT * FROM users WHERE email = ?", ("test@example.com",)) -print(cursor.fetchall()) - -# Use SELECT only needed columns -# Slow: SELECT * -# Fast: SELECT id, name -``` - -## Memory Optimization - -### Pattern 18: Detecting Memory Leaks - -```python -import tracemalloc -import gc - -def memory_leak_example(): - """Example that leaks memory.""" - leaked_objects = [] - - for i in range(100000): - # Objects added but never removed - leaked_objects.append([i] * 100) - - # In real code, this would be an unintended reference - -def track_memory_usage(): - """Track memory allocations.""" - tracemalloc.start() - - # Take snapshot before - snapshot1 = tracemalloc.take_snapshot() - - # Run code - memory_leak_example() - - # Take snapshot after - snapshot2 = tracemalloc.take_snapshot() - - # Compare - top_stats = snapshot2.compare_to(snapshot1, 'lineno') - - print("Top 10 memory allocations:") - for stat in top_stats[:10]: - print(stat) - - tracemalloc.stop() - -# Monitor memory -track_memory_usage() - -# Force garbage collection -gc.collect() -``` - -### Pattern 19: Iterators vs Lists - -```python -import sys - -def process_file_list(filename): - """Load entire file into memory.""" - with open(filename) as f: - lines = f.readlines() # Loads all lines - return sum(1 for line in lines if line.strip()) - -def process_file_iterator(filename): - """Process file line by line.""" - with open(filename) as f: - return sum(1 for line in f if line.strip()) - -# Iterator uses constant memory -# List loads entire file into memory -``` - -### Pattern 20: Weakref for Caches - -```python -import weakref - -class CachedResource: - """Resource that can be garbage collected.""" - def __init__(self, data): - self.data = data - -# Regular cache prevents garbage collection -regular_cache = {} - -def get_resource_regular(key): - """Get resource from regular cache.""" - if key not in regular_cache: - regular_cache[key] = CachedResource(f"Data for {key}") - return regular_cache[key] - -# Weak reference cache allows garbage collection -weak_cache = weakref.WeakValueDictionary() - -def get_resource_weak(key): - """Get resource from weak cache.""" - resource = weak_cache.get(key) - if resource is None: - resource = CachedResource(f"Data for {key}") - weak_cache[key] = resource - return resource - -# When no strong references exist, objects can be GC'd -``` - -## Benchmarking Tools - -### Custom Benchmark Decorator - -```python -import time -from functools import wraps - -def benchmark(func): - """Decorator to benchmark function execution.""" - @wraps(func) - def wrapper(*args, **kwargs): - start = time.perf_counter() - result = func(*args, **kwargs) - elapsed = time.perf_counter() - start - print(f"{func.__name__} took {elapsed:.6f} seconds") - return result - return wrapper - -@benchmark -def slow_function(): - """Function to benchmark.""" - time.sleep(0.5) - return sum(range(1000000)) - -result = slow_function() -``` - -### Performance Testing with pytest-benchmark - -```python -# Install: pip install pytest-benchmark - -def test_list_comprehension(benchmark): - """Benchmark list comprehension.""" - result = benchmark(lambda: [i**2 for i in range(10000)]) - assert len(result) == 10000 - -def test_map_function(benchmark): - """Benchmark map function.""" - result = benchmark(lambda: list(map(lambda x: x**2, range(10000)))) - assert len(result) == 10000 - -# Run with: pytest test_performance.py --benchmark-compare -``` - -## Best Practices - -1. **Profile before optimizing** - Measure to find real bottlenecks -2. **Focus on hot paths** - Optimize code that runs most frequently -3. **Use appropriate data structures** - Dict for lookups, set for membership -4. **Avoid premature optimization** - Clarity first, then optimize -5. **Use built-in functions** - They're implemented in C -6. **Cache expensive computations** - Use lru_cache -7. **Batch I/O operations** - Reduce system calls -8. **Use generators** for large datasets -9. **Consider NumPy** for numerical operations -10. **Profile production code** - Use py-spy for live systems - -## Common Pitfalls - -- Optimizing without profiling -- Using global variables unnecessarily -- Not using appropriate data structures -- Creating unnecessary copies of data -- Not using connection pooling for databases -- Ignoring algorithmic complexity -- Over-optimizing rare code paths -- Not considering memory usage diff --git a/.agents/skills/seo-audit/SKILL.md b/.agents/skills/seo-audit/SKILL.md deleted file mode 100644 index 1dbe4de9..00000000 --- a/.agents/skills/seo-audit/SKILL.md +++ /dev/null @@ -1,412 +0,0 @@ ---- -name: seo-audit -description: When the user wants to audit, review, or diagnose SEO issues on their site. Also use when the user mentions "SEO audit," "technical SEO," "why am I not ranking," "SEO issues," "on-page SEO," "meta tags review," "SEO health check," "my traffic dropped," "lost rankings," "not showing up in Google," "site isn't ranking," "Google update hit me," "page speed," "core web vitals," "crawl errors," or "indexing issues." Use this even if the user just says something vague like "my SEO is bad" or "help with SEO" — start with an audit. For building pages at scale to target keywords, see programmatic-seo. For adding structured data, see schema-markup. For AI search optimization, see ai-seo. -metadata: - version: 1.1.0 ---- - -# SEO Audit - -You are an expert in search engine optimization. Your goal is to identify SEO issues and provide actionable recommendations to improve organic search performance. - -## Initial Assessment - -**Check for product marketing context first:** -If `.agents/product-marketing-context.md` exists (or `.claude/product-marketing-context.md` in older setups), read it before asking questions. Use that context and only ask for information not already covered or specific to this task. - -Before auditing, understand: - -1. **Site Context** - - What type of site? (SaaS, e-commerce, blog, etc.) - - What's the primary business goal for SEO? - - What keywords/topics are priorities? - -2. **Current State** - - Any known issues or concerns? - - Current organic traffic level? - - Recent changes or migrations? - -3. **Scope** - - Full site audit or specific pages? - - Technical + on-page, or one focus area? - - Access to Search Console / analytics? - ---- - -## Audit Framework - -### Schema Markup Detection Limitation - -**`web_fetch` and `curl` cannot reliably detect structured data / schema markup.** - -Many CMS plugins (AIOSEO, Yoast, RankMath) inject JSON-LD via client-side JavaScript — it won't appear in static HTML or `web_fetch` output (which strips `