Custom crawling

.github/workflows/ci.yml

@@ -47,13 +47,14 @@ jobs:
           pytest tests/test_alert_checker.py tests/test_schedule_runner.py tests/test_export_audit.py \
             tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_audit_tools_expanded.py \
             tests/test_audit_tools_coverage.py tests/test_audit_tools_dispatch_coverage.py \
-            tests/test_export_custom_coverage.py tests/test_export_artifacts_coverage.py \
-            tests/test_export_compare_coverage.py tests/test_export_tools_coverage.py \
-            tests/test_image_tools.py tests/test_export_custom.py tests/test_export_artifacts.py \
-            tests/test_export_compare.py \
-            tests/test_mcp_registry.py tests/test_mcp_resources.py \
+            tests/test_audit_tools_links_extras.py tests/test_export_custom_coverage.py \
+            tests/test_export_artifacts_coverage.py tests/test_export_compare_coverage.py \
+            tests/test_export_tools_coverage.py tests/test_image_tools.py tests/test_export_custom.py \
+            tests/test_export_artifacts.py tests/test_export_compare.py tests/test_export_workbook.py \
+            tests/test_export_sitemap.py tests/test_mcp_registry.py tests/test_mcp_resources.py \
+            tests/test_tools_branch_coverage.py \
             --cov=website_profiling.tools --cov-config=.coveragerc.tools \
-            --cov-report=term-missing --cov-fail-under=95 -q -o addopts=
+            --cov-report=term-missing --cov-fail-under=100 -q -o addopts=
       - name: CLI smoke
         run: python -m src --help
 

alembic/versions/013_crawl_discovery_and_link_edges.py

@@ -0,0 +1,56 @@
+"""Crawl discovery mode on crawl_runs and rich link_edges table.
+
+Revision ID: 013_crawl_discovery_edges
+Revises: 012_chat_sessions
+"""
+from __future__ import annotations
+
+from alembic import op
+
+revision = "013_crawl_discovery_edges"
+down_revision = "012_chat_sessions"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute("""
+        ALTER TABLE crawl_runs
+            ADD COLUMN IF NOT EXISTS discovery_mode TEXT DEFAULT 'spider';
+
+        CREATE TABLE IF NOT EXISTS link_edges (
+            crawl_run_id BIGINT NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE,
+            from_url TEXT NOT NULL,
+            to_url TEXT NOT NULL,
+            anchor_text TEXT NOT NULL DEFAULT '',
+            rel TEXT NOT NULL DEFAULT '',
+            is_nofollow BOOLEAN NOT NULL DEFAULT FALSE,
+            is_sponsored BOOLEAN NOT NULL DEFAULT FALSE,
+            is_ugc BOOLEAN NOT NULL DEFAULT FALSE,
+            link_type TEXT NOT NULL DEFAULT 'internal',
+            PRIMARY KEY (crawl_run_id, from_url, to_url, anchor_text, rel)
+        );
+        CREATE INDEX IF NOT EXISTS idx_link_edges_run_from
+            ON link_edges(crawl_run_id, from_url);
+        CREATE INDEX IF NOT EXISTS idx_link_edges_run_to
+            ON link_edges(crawl_run_id, to_url);
+
+        CREATE TABLE IF NOT EXISTS saved_crawl_filters (
+            id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            property_id BIGINT NOT NULL REFERENCES properties(id) ON DELETE CASCADE,
+            name TEXT NOT NULL,
+            filter_json JSONB NOT NULL DEFAULT '{}',
+            created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+            UNIQUE (property_id, name)
+        );
+        CREATE INDEX IF NOT EXISTS idx_saved_crawl_filters_property
+            ON saved_crawl_filters(property_id);
+    """)
+
+
+def downgrade() -> None:
+    op.execute("""
+        DROP TABLE IF EXISTS saved_crawl_filters CASCADE;
+        DROP TABLE IF EXISTS link_edges CASCADE;
+        ALTER TABLE crawl_runs DROP COLUMN IF EXISTS discovery_mode;
+    """)

docs/GLOSSARY.md

@@ -39,17 +39,21 @@ UI terms agencies recognize, mapped to internal keys and data sources.
 | Scheduled audits | `properties.schedule_cron`, `/api/schedule/check` | Cron + pipeline spawn | Recurring site audit — see [OPS.md](OPS.md) |
 | Property alerts | `alert_webhook_url`, `/api/alerts/check` | Health snapshot rules | Ops notifications |
 | Content brief | Keywords Brief button, `/api/keywords/content-brief` | LLM or deterministic | Content planning |
-| AI issue fix | `llm_recommendation`, `/api/issues/fix-suggestion` | LLM on demand + report build | Actionable remediation |
+| AI fix suggestions | `llm_recommendation`, `/api/ai/fix-suggestion`, `/api/issues/fix-suggestion` (legacy) | LLM on demand + report build | Actionable remediation across Issues, Lighthouse, Security, and other surfaces |
 | AI Chat | `/chat`, `/api/chat`, `chat_sessions` | LLM + read-only audit tools | Conversational site audit queries |
 | MCP tools | `python -m website_profiling.mcp` | Same `audit_tools` as chat | Cursor / Claude Desktop integration — see [MCP.md](MCP.md) |
 | Read-only session | `AUTH_DEFAULT_ROLE=client-readonly`, `/api/auth/session` | Session cookie | Client view-only access |
 | Export executive summary | `export_audit_html/pdf/csv`, `export_audit_report` (chat/MCP), Export view | Report payload + optional AI | Client deliverable |
+| ads.txt / security.txt | `site_level`, `get_ads_txt_status`, `get_security_txt_status` | Root file fetch at report build | Publisher / contact file hygiene |
+| Subdomain inventory | `subdomains`, `list_subdomains`, `/subdomains` view | Crawl + GSC + optional crt.sh | Host footprint vs crawl scope |
+| Contact intelligence | `contact_intelligence`, `get_contact_intelligence`, `/contacts` view | Crawl schema/mailto + security.txt + RDAP org | Business identity consistency |
 
 ## Metric names
 
 | UI | Field | Source |
 |----|-------|--------|
-| Inlinks | `inlinks` | Crawl graph |
+| Impact score | `impact_score` on issues | GSC clicks + GA4 sessions + priority weight (see below) |
+| Link edges | `link_edges`, `link_rel_summary` | Crawl anchor/rel attributes |
 | Outlinks | `outlinks` | Crawl graph |
 | Status code | `status` | HTTP |
 | Crawl rendering | `crawl_render_mode` on run; `fetch_method` per URL | `static`, `javascript`, or `auto` crawl config; `static` vs `rendered` per page |
@@ -62,6 +66,8 @@ UI terms agencies recognize, mapped to internal keys and data sources.
 | On-site frequency | `volume` (heuristic) | Estimated from crawl |
 | Sessions | GA4 metrics | Analytics |
 
+**Impact score:** `priority_weight + (gsc_clicks × 10) + (ga4_sessions × 5)` with Critical=1000, High=100, Medium=10, Low=1.
+
 ## Provenance badges
 
 | Badge | Meaning |

docs/MCP.md

@@ -43,11 +43,11 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings):
 | `audit://glossary` | Excerpt from `docs/GLOSSARY.md` |
 | `audit://tools` | Tool catalog grouped by SEO domain |
 
-## Tools (171 read-only + export)
+## Tools (176 read-only + export)
 
 ### Export and deliverables
 
-`export_audit_report`, `export_compare_csv`, `export_list_as_csv`, `compose_custom_report`, `export_custom_report`, `list_export_formats`
+`export_audit_report`, `export_compare_csv`, `export_list_as_csv`, `export_sitemap_xml`, `validate_rich_results`, `compose_custom_report`, `export_custom_report`, `list_export_formats`
 
 Full audit exports reuse the same generators as the Export view (PDF requires `reportlab`). Export tools store files as artifacts (24h TTL); in-app chat renders download buttons via `/api/chat/artifacts/{id}`.
 
@@ -79,7 +79,7 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu
 
 ### Links and architecture
 
-`list_orphan_pages`, `get_top_linked_pages`, `get_top_crawled_pages`, `get_outbound_link_domains`, `get_link_graph_summary`, `get_url_fingerprints`, `list_broken_link_sources`, `get_mime_type_breakdown`, `get_title_length_distribution`, `get_domain_link_distribution`, `get_outlink_distribution`
+`get_link_rel_summary`, `get_inlink_anchors`, `list_nofollow_internal_links`, `list_orphan_pages`, `get_top_linked_pages`, `get_top_crawled_pages`, `get_outbound_link_domains`, `get_link_graph_summary`, `get_url_fingerprints`, `list_broken_link_sources`, `get_mime_type_breakdown`, `get_title_length_distribution`, `get_domain_link_distribution`, `get_outlink_distribution`
 
 ### Indexation and international
 
@@ -115,12 +115,13 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu
 
 ## Future pipeline items (not yet exposed as tools)
 
-These require additional crawl or third-party integrations before dedicated tools are useful:
+These require additional third-party integrations or product scope beyond current crawl data:
 
-- Google Rich Results / schema validation API
-- Full backlink index and anchor-text analytics
-- axe / color-contrast accessibility audits
+- Full backlink index and anchor-text analytics (beyond GSC Links import)
 - SERP rank tracking beyond GSC position snapshots
+- Standalone Google Rich Results Test API (current `validate_rich_results` uses crawl heuristics + GSC URL Inspection)
+
+Already available: `validate_rich_results`, `export_sitemap_xml`, workbook export, axe audits via `enable_axe` on browser crawls.
 
 ## Example prompts
 

input.txt.example

@@ -18,6 +18,16 @@ content_excerpt_max_chars = 4096
 preserve_crawl_history = true
 crawl_stream_to_db = false
 crawl_exclude_urls = 
+crawl_discovery_mode = spider
+crawl_url_list = 
+crawl_user_agent_preset = default
+crawl_user_agent_custom = 
+crawl_auth_username = 
+crawl_auth_password = 
+crawl_extra_headers = 
+crawl_cookies = 
+crawl_robots_txt_override = 
+custom_extractors = 
 # crawl_render_mode: static | javascript | auto (auto = static first, browser when SPA heuristics match)
 crawl_render_mode = static
 crawl_js_concurrency = 3
@@ -46,6 +56,9 @@ max_image_probe_urls = 500
 image_probe_concurrency = 6
 image_probe_timeout = 8
 image_unoptimized_min_kb = 200
+enable_subdomain_discovery = true
+subdomain_ct_lookup = true
+enable_rdap_org_lookup = true
 
 # --- Lighthouse ---
 lighthouse_url = 
@@ -56,6 +69,13 @@ lighthouse_iterations = 1
 run_lighthouse = true
 run_lighthouse_on_pages = true
 enable_crux = false
+enable_rich_results_validation = false
+google_rich_results_api_key =
+enable_axe = false
+enable_spell_check = false
+enable_html_validation = false
+enable_amp_audit = false
+enable_wayback_lookup = false
 competitor_domains =
 bing_webmaster_api_key =
 serp_api_key =

pipeline-config.example.txt

@@ -19,6 +19,16 @@ content_excerpt_max_chars = 4096
 preserve_crawl_history = true
 crawl_stream_to_db = false
 crawl_exclude_urls = 
+crawl_discovery_mode = spider
+crawl_url_list = 
+crawl_user_agent_preset = default
+crawl_user_agent_custom = 
+crawl_auth_username = 
+crawl_auth_password = 
+crawl_extra_headers = 
+crawl_cookies = 
+crawl_robots_txt_override = 
+custom_extractors = 
 # crawl_render_mode: static | javascript | auto
 crawl_render_mode = static
 crawl_js_concurrency = 3
@@ -47,6 +57,9 @@ max_image_probe_urls = 500
 image_probe_concurrency = 6
 image_probe_timeout = 8
 image_unoptimized_min_kb = 200
+enable_subdomain_discovery = true
+subdomain_ct_lookup = true
+enable_rdap_org_lookup = true
 
 # --- Lighthouse ---
 lighthouse_url = 
@@ -57,6 +70,13 @@ lighthouse_iterations = 1
 run_lighthouse = true
 run_lighthouse_on_pages = true
 enable_crux = false
+enable_rich_results_validation = false
+google_rich_results_api_key =
+enable_axe = false
+enable_spell_check = false
+enable_html_validation = false
+enable_amp_audit = false
+enable_wayback_lookup = false
 competitor_domains =
 bing_webmaster_api_key =
 serp_api_key =

requirements-optional.txt

@@ -0,0 +1,3 @@
+# Optional audit dependencies (install when enabling spell-check / HTML validation extras)
+pyspellchecker>=0.8.1
+html5lib>=1.1

scripts/local-test.ps1

@@ -245,7 +245,7 @@ function Invoke-PytestReporting {
 }
 
 function Invoke-PytestTools {
-    Write-Log "Pytest (tools coverage gate, 95%)"
+    Write-Log "Pytest (tools coverage gate, 100%)"
     & $VENV_PYTEST `
         tests/test_alert_checker.py `
         tests/test_schedule_runner.py `
@@ -265,10 +265,11 @@ function Invoke-PytestTools {
         tests/test_export_compare.py `
         tests/test_mcp_registry.py `
         tests/test_mcp_resources.py `
+        tests/test_tools_branch_coverage.py `
         --cov=website_profiling.tools `
         --cov-config=.coveragerc.tools `
         --cov-report=term-missing `
-        --cov-fail-under=95 `
+        --cov-fail-under=100 `
         -q `
         -o addopts=
     Assert-LastExitCode "Tools coverage gate failed"

scripts/local-test.sh

@@ -119,10 +119,14 @@ run_pytest_reporting() {
     tests/test_categories_roadmap.py \
     tests/test_report_categories_golden.py \
     tests/test_categories_coverage.py \
+    tests/test_contrast_issues.py \
     tests/test_indexation_coverage.py \
     tests/test_crawl_segments.py \
     tests/test_terminology.py \
     tests/test_compare_payload.py \
+    tests/test_optional_audits.py \
+    tests/test_property_profile.py \
+    tests/test_reporting_gaps.py \
     --cov=website_profiling.reporting \
     --cov-config=.coveragerc.reporting \
     --cov-report=term-missing \
@@ -133,7 +137,7 @@ run_pytest_reporting() {
 
 run_pytest_tools() {
   [[ "$PYTEST_NO_COV" -eq 1 ]] && return 0
-  log "Pytest (tools coverage gate, 95%)"
+  log "Pytest (tools coverage gate, 100%)"
   "$VENV/bin/pytest" \
     tests/test_alert_checker.py \
     tests/test_schedule_runner.py \
@@ -143,6 +147,7 @@ run_pytest_tools() {
     tests/test_audit_tools_expanded.py \
     tests/test_audit_tools_coverage.py \
     tests/test_audit_tools_dispatch_coverage.py \
+    tests/test_audit_tools_links_extras.py \
     tests/test_export_custom_coverage.py \
     tests/test_export_artifacts_coverage.py \
     tests/test_export_compare_coverage.py \
@@ -151,12 +156,15 @@ run_pytest_tools() {
     tests/test_export_custom.py \
     tests/test_export_artifacts.py \
     tests/test_export_compare.py \
+    tests/test_export_workbook.py \
+    tests/test_export_sitemap.py \
     tests/test_mcp_registry.py \
     tests/test_mcp_resources.py \
+    tests/test_tools_branch_coverage.py \
     --cov=website_profiling.tools \
     --cov-config=.coveragerc.tools \
     --cov-report=term-missing \
-    --cov-fail-under=95 \
+    --cov-fail-under=100 \
     -q \
     -o addopts=
 }