From 03e95b072b293b1ce0f3a81562c8a9ea091707e4 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 01:56:54 +0530 Subject: [PATCH 1/5] more core improvement and predictable --- README.md | 4 + .../014_pipeline_job_log_truncated.py | 27 +++ requirements.txt | 54 +++--- .../integrations/google/auth.py | 27 ++- tests/test_property_google.py | 43 +++++ web/app/api/jobs/[id]/route.ts | 1 + web/app/api/jobs/route.ts | 12 +- web/app/api/run/route.ts | 11 +- .../components/GoogleIntegrationsPanel.tsx | 2 +- .../integrations/PropertyOpsSection.tsx | 28 ++- .../overview/OverviewSummaryTab.tsx | 17 +- .../components/pipeline/PipelineLogViewer.tsx | 9 + .../components/pipeline/PipelineRunPanel.tsx | 2 + web/src/context/PipelineContext.tsx | 36 +++- web/src/lib/pipelineDebug.ts | 6 + web/src/lib/pipelineJobEvents.ts | 11 +- web/src/server/jobsRoute.test.ts | 22 +-- web/src/server/pipelineJobs.test.ts | 151 +++++++++++++++ web/src/server/pipelineJobs.ts | 180 ++++++++++++------ web/src/server/pipelineJobsDb.ts | 138 ++++++++++---- web/src/strings.json | 6 + web/src/types/api.ts | 2 + web/src/views/CompareReports.tsx | 5 +- 23 files changed, 608 insertions(+), 186 deletions(-) create mode 100644 alembic/versions/014_pipeline_job_log_truncated.py create mode 100644 web/src/server/pipelineJobs.test.ts diff --git a/README.md b/README.md index d531cd4f..aa7dea34 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,10 @@ Open [http://localhost:3000/home](http://localhost:3000/home). ./local-run stop # stop Postgres container ``` +`requirements.txt` pins direct Python dependencies to versions verified by `./local-test python`. Re-run the full test suite after intentional upgrades. + +Pipeline jobs: stuck `running` rows are reconciled after **1 hour** by default (`PIPELINE_JOB_STALE_HOURS`). Orphan jobs with no live server process are cleared after **5 minutes** (`PIPELINE_JOB_ORPHAN_MINUTES`). Increase `PIPELINE_JOB_STALE_HOURS` for crawls that routinely run longer than an hour. + **Tests** ```bash diff --git a/alembic/versions/014_pipeline_job_log_truncated.py b/alembic/versions/014_pipeline_job_log_truncated.py new file mode 100644 index 00000000..55cb6ec1 --- /dev/null +++ b/alembic/versions/014_pipeline_job_log_truncated.py @@ -0,0 +1,27 @@ +"""Add log_truncated flag to pipeline_jobs. + +Revision ID: 014_pipeline_log_truncated +Revises: 013_crawl_discovery_edges +""" +from __future__ import annotations + +from alembic import op + +revision = "014_pipeline_log_truncated" +down_revision = "013_crawl_discovery_edges" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE pipeline_jobs + ADD COLUMN IF NOT EXISTS log_truncated BOOLEAN NOT NULL DEFAULT false; + """) + + +def downgrade() -> None: + op.execute(""" + ALTER TABLE pipeline_jobs + DROP COLUMN IF EXISTS log_truncated; + """) diff --git a/requirements.txt b/requirements.txt index 9138a6e1..002458fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,21 @@ -requests>=2.28.0 -beautifulsoup4>=4.11.0 -lxml>=4.9.0 -pandas>=1.5.0 -tqdm>=4.64.0 -networkx>=2.8.0 -python-Wappalyzer>=0.3.1 +requests==2.34.2 +beautifulsoup4==4.14.3 +lxml==6.1.1 +pandas==3.0.3 +tqdm==4.67.3 +networkx==3.6.1 +python-Wappalyzer==0.3.1 # Local content analysis (duplicates, language) -rapidfuzz>=3.0.0 -langdetect>=1.0.9 +rapidfuzz==3.14.5 +langdetect==1.0.9 # Google Search Console + GA4 integration (optional; required for `python -m src google`) -google-auth>=2.0.0 -google-auth-oauthlib>=1.0.0 -google-api-python-client>=2.0.0 -google-analytics-data>=0.18.0 -google-analytics-admin>=0.22.0 +google-auth==2.53.0 +google-auth-oauthlib==1.4.0 +google-api-python-client==2.197.0 +google-analytics-data==0.23.0 +google-analytics-admin==0.30.0 # Keywords Explorer — Google Suggest + Wikipedia + Datamuse (all free, no auth needed) # requests already listed above @@ -23,28 +23,28 @@ google-analytics-admin>=0.22.0 # pytrends>=4.9,<5 # PostgreSQL -psycopg[binary,pool]>=3.2 -sqlalchemy>=2.0.0 -alembic>=1.13 +psycopg[binary,pool]==3.3.4 +sqlalchemy==2.0.50 +alembic==1.18.4 # Audit export (PDF) -reportlab>=4.0.0 +reportlab==4.5.1 # JavaScript rendering crawl (headless Chromium via Playwright) -playwright>=1.49.0 +playwright==1.60.0 # LLM providers for AI enrichment (configure via web UI AI tab) -httpx>=0.27.0 -openai>=1.0.0 -anthropic>=0.25.0 +httpx==0.28.1 +openai==2.41.0 +anthropic==0.107.0 # Spell-check / HTML validation extras -pyspellchecker>=0.8.1 -html5lib>=1.1 +pyspellchecker==0.9.0 +html5lib==1.1 # MCP server for Cursor / Claude Desktop -mcp>=1.0.0 +mcp~=1.0.0 # Dev / test -pytest>=7.0.0 -pytest-cov>=5.0.0 +pytest==9.0.3 +pytest-cov==7.1.0 diff --git a/src/website_profiling/integrations/google/auth.py b/src/website_profiling/integrations/google/auth.py index 6364f077..6eda7087 100644 --- a/src/website_profiling/integrations/google/auth.py +++ b/src/website_profiling/integrations/google/auth.py @@ -36,7 +36,7 @@ def _app_client_credentials() -> tuple[str, str]: return app_client_credentials() -def _property_refresh_token(property_id: int) -> tuple[str, str | None]: +def _property_google_auth(property_id: int) -> tuple[str, str | None, str]: from ...db import db_session from ...db.property_store import get_property_by_id @@ -44,21 +44,16 @@ def _property_refresh_token(property_id: int) -> tuple[str, str | None]: prop = get_property_by_id(conn, property_id) if not prop: raise RuntimeError(f"Property id {property_id} not found.") - domain = prop.get("canonical_domain") or "this site" token = (prop.get("google_refresh_token") or "").strip() - if not token: - raise RuntimeError( - f"Google not connected for {domain}. " - "Set Site URL, open Integrations, and click Connect with Google for this site." - ) - return token, prop.get("google_auth_mode") + domain = prop.get("canonical_domain") or "this site" + return token, prop.get("google_auth_mode"), domain def build_credentials(property_id: int | None = None): """ Load Google OAuth2 credentials. property_id is required for OAuth user tokens. - Service account uses google_app_settings.service_account_json when property_id is None. + Service account uses google_app_settings.service_account_json (app-wide). """ try: from google.oauth2.credentials import Credentials @@ -66,11 +61,17 @@ def build_credentials(property_id: int | None = None): except ImportError as e: raise ImportError(f"{INSTALL_HINT}\n({e})") from e + from ...db.google_app_store import has_service_account, build_service_account_credentials + if property_id is not None: - refresh_token, prop_auth_mode = _property_refresh_token(property_id) - if prop_auth_mode == "service_account": + refresh_token, prop_auth_mode, domain = _property_google_auth(property_id) + if prop_auth_mode == "service_account" or (not refresh_token and has_service_account()): + return build_service_account_credentials() + if not refresh_token: raise RuntimeError( - "Per-property service account is not implemented yet. Use OAuth Connect." + f"Google not connected for {domain}. " + "Set Site URL, open Integrations, and click Connect with Google for this site, " + "or upload an app-wide service account JSON in Integrations." ) client_id, client_secret = _app_client_credentials() creds = Credentials( @@ -83,8 +84,6 @@ def build_credentials(property_id: int | None = None): creds.refresh(Request()) return creds - from ...db.google_app_store import has_service_account, build_service_account_credentials - if has_service_account(): return build_service_account_credentials() diff --git a/tests/test_property_google.py b/tests/test_property_google.py index 4e7df8f5..72843dea 100644 --- a/tests/test_property_google.py +++ b/tests/test_property_google.py @@ -1,6 +1,8 @@ """Per-property Google resolution and scoped google_data.""" from __future__ import annotations +from unittest.mock import patch + from website_profiling.db.property_store import canonical_domain_from_start_url @@ -26,3 +28,44 @@ def test_build_credentials_accepts_property_id(): sig = inspect.signature(build_credentials) assert "property_id" in sig.parameters + + +def test_build_credentials_property_service_account_uses_app_sa(): + from website_profiling.integrations.google.auth import build_credentials + + fake_sa = {"type": "service_account", "project_id": "p"} + with ( + patch( + "website_profiling.integrations.google.auth._property_google_auth", + return_value=("", "service_account", "example.com"), + ), + patch( + "website_profiling.db.google_app_store.build_service_account_credentials", + return_value={"ok": True}, + ) as mock_sa, + ): + creds = build_credentials(property_id=42) + assert creds == {"ok": True} + mock_sa.assert_called_once() + + +def test_build_credentials_property_no_oauth_falls_back_to_app_sa(): + from website_profiling.integrations.google.auth import build_credentials + + with ( + patch( + "website_profiling.integrations.google.auth._property_google_auth", + return_value=("", "oauth", "example.com"), + ), + patch( + "website_profiling.db.google_app_store.has_service_account", + return_value=True, + ), + patch( + "website_profiling.db.google_app_store.build_service_account_credentials", + return_value={"sa": True}, + ) as mock_sa, + ): + creds = build_credentials(property_id=7) + assert creds == {"sa": True} + mock_sa.assert_called_once() diff --git a/web/app/api/jobs/[id]/route.ts b/web/app/api/jobs/[id]/route.ts index e0849ade..54b94077 100644 --- a/web/app/api/jobs/[id]/route.ts +++ b/web/app/api/jobs/[id]/route.ts @@ -21,5 +21,6 @@ export const GET: ApiRouteHandlerWithParams<{ id: string }> = async ( exitCode: job.exitCode, log: job.log, error: job.error ?? null, + logTruncated: job.logTruncated ?? false, }); }; diff --git a/web/app/api/jobs/route.ts b/web/app/api/jobs/route.ts index 378047fa..8f9e4bd0 100644 --- a/web/app/api/jobs/route.ts +++ b/web/app/api/jobs/route.ts @@ -1,10 +1,6 @@ import { NextResponse, type NextRequest } from 'next/server'; import { forbiddenIfNotLocal } from '@/server/localOnly'; -import { - getActiveRunningJob, - listRecentPipelineJobs, - reconcileStaleRunningJobs, -} from '@/server/pipelineJobsDb'; +import { listPipelineJobsForApi } from '@/server/pipelineJobs'; import type { ApiRouteHandler } from '@/types/api'; export const runtime = 'nodejs'; @@ -24,11 +20,7 @@ export const GET: ApiRouteHandler = async (request: NextRequest): Promise {}); + }).catch((err) => logPipelineDbError('writeAuditLog', err)); return NextResponse.json({ jobId: id }); } catch (e) { const msg = e instanceof Error ? e.message : String(e); - return NextResponse.json({ error: msg }, { status: 400 }); + const isSlotTaken = msg === 'An audit job is already running'; + return NextResponse.json({ error: msg }, { status: isSlotTaken ? 400 : 500 }); } }; diff --git a/web/src/components/GoogleIntegrationsPanel.tsx b/web/src/components/GoogleIntegrationsPanel.tsx index 84602e6b..2c181f7d 100644 --- a/web/src/components/GoogleIntegrationsPanel.tsx +++ b/web/src/components/GoogleIntegrationsPanel.tsx @@ -796,7 +796,7 @@ export default function GoogleIntegrationsPanel({ ) : null; const infoBannerText = - 'Google Client ID/Secret and service account keys are stored in the database. Each site keeps its own OAuth connection and Search Console / Analytics property IDs.'; + 'Google Client ID/Secret and service account JSON are stored app-wide in the database. Upload a service account for API access without per-site OAuth, or connect each site with OAuth for user-delegated access. Search Console and Analytics property IDs remain per site.'; const infoBanner = (

diff --git a/web/src/components/integrations/PropertyOpsSection.tsx b/web/src/components/integrations/PropertyOpsSection.tsx index ba0c1fa6..388ddad0 100644 --- a/web/src/components/integrations/PropertyOpsSection.tsx +++ b/web/src/components/integrations/PropertyOpsSection.tsx @@ -27,14 +27,20 @@ export default function PropertyOpsSection({ propertyId }: PropertyOpsSectionPro let cancelled = false; setLoading(true); void fetch(apiUrl(`/properties/${propertyId}/ops`)) - .then((res) => (res.ok ? res.json() : null)) - .then((data) => { - if (cancelled || !data) return; + .then(async (res) => { + if (cancelled) return; + if (!res.ok) { + setMessage(s.loadFailed); + return; + } + const data = await res.json(); setScheduleCron(String(data.schedule_cron || '')); setAlertWebhookUrl(String(data.alert_webhook_url || '')); setAlertEmail(String(data.alert_email || '')); }) - .catch(() => {}) + .catch(() => { + if (!cancelled) setMessage(s.loadFailed); + }) .finally(() => { if (!cancelled) setLoading(false); }); @@ -47,9 +53,11 @@ export default function PropertyOpsSection({ propertyId }: PropertyOpsSectionPro if (propertyId == null) return undefined; let cancelled = false; void fetch(apiUrl(`/properties/${propertyId}/google/links/status`)) - .then((res) => (res.ok ? res.json() : null)) - .then((status) => { - if (cancelled || !status) return; + .then(async (res) => { + if (cancelled) return; + if (!res.ok) return; + const status = await res.json(); + if (!status) return; if (!status.hasData) { setGscLinksStale(s.gscLinksMissing); return; @@ -63,11 +71,13 @@ export default function PropertyOpsSection({ propertyId }: PropertyOpsSectionPro setGscLinksStale(null); } }) - .catch(() => {}); + .catch(() => { + if (!cancelled) setGscLinksStale(s.loadFailed); + }); return () => { cancelled = true; }; - }, [propertyId, s.gscLinksMissing, s.gscLinksStale]); + }, [propertyId, s.gscLinksMissing, s.gscLinksStale, s.loadFailed]); const handleSave = useCallback(async () => { if (propertyId == null || readOnly) return; diff --git a/web/src/components/overview/OverviewSummaryTab.tsx b/web/src/components/overview/OverviewSummaryTab.tsx index 2484d69b..80864535 100644 --- a/web/src/components/overview/OverviewSummaryTab.tsx +++ b/web/src/components/overview/OverviewSummaryTab.tsx @@ -44,6 +44,7 @@ export function OverviewSummaryTab({ data, exportHref, compareHref, reportCount const sj = strings.common; const searchParams = useSearchParams(); const [healthDelta, setHealthDelta] = useState(null); + const [historyError, setHistoryError] = useState(null); const keywordsHref = useMemo( () => buildKeywordsHref(searchParams.toString()), [searchParams], @@ -62,16 +63,21 @@ export function OverviewSummaryTab({ data, exportHref, compareHref, reportCount useEffect(() => { const domain = data.site_name || ''; if (!domain) return; + setHistoryError(null); void fetch(`/api/report/history?domain=${encodeURIComponent(domain)}&limit=2`) - .then((r) => r.json()) - .then((payload: { history?: Array<{ healthScore?: number | null }> }) => { + .then(async (r) => { + if (!r.ok) { + setHistoryError(vo.historyTrendUnavailable ?? 'Could not load health trend.'); + return; + } + const payload = (await r.json()) as { history?: Array<{ healthScore?: number | null }> }; const hist = payload.history || []; if (hist.length >= 2 && currentHealth != null && hist[1]?.healthScore != null) { setHealthDelta(currentHealth - Number(hist[1].healthScore)); } }) - .catch(() => {}); - }, [data.site_name, currentHealth]); + .catch(() => setHistoryError(vo.historyTrendUnavailable ?? 'Could not load health trend.')); + }, [data.site_name, currentHealth, vo.historyTrendUnavailable]); const execTopIssues = (data.executive_summary?.top_issues || []).slice(0, 5); const execPriorities = (data.executive_summary?.priorities || []).filter(Boolean); @@ -126,6 +132,9 @@ export function OverviewSummaryTab({ data, exportHref, compareHref, reportCount {healthDelta} vs prior run) )} + {historyError ? ( + {historyError} + ) : null}

)} {execSummary ? ( diff --git a/web/src/components/pipeline/PipelineLogViewer.tsx b/web/src/components/pipeline/PipelineLogViewer.tsx index 0220480e..92963069 100644 --- a/web/src/components/pipeline/PipelineLogViewer.tsx +++ b/web/src/components/pipeline/PipelineLogViewer.tsx @@ -28,6 +28,7 @@ export interface PipelineLogViewerProps { log: string; autoScroll?: boolean; status?: PipelineJobStatus | ''; + logTruncated?: boolean; className?: string; } @@ -206,6 +207,7 @@ export default function PipelineLogViewer({ log, autoScroll = true, status = '', + logTruncated = false, className = '', }: PipelineLogViewerProps) { const scrollRef = useRef(null); @@ -395,6 +397,13 @@ export default function PipelineLogViewer({ ) : null} + {logTruncated ? ( +
+ + {strings.pipelineRunner.logTruncatedBanner} +
+ ) : null} +
diff --git a/web/src/context/PipelineContext.tsx b/web/src/context/PipelineContext.tsx index 84004cbd..72e2fa9e 100644 --- a/web/src/context/PipelineContext.tsx +++ b/web/src/context/PipelineContext.tsx @@ -63,6 +63,7 @@ export interface PipelineContextValue { stopping: boolean; activeJobId: string; log: string; + logTruncated: boolean; status: PipelineJobStatus | ''; backgroundMode: boolean; startUrl: string; @@ -125,6 +126,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { const [busy, setBusy] = useState(false); const [stopping, setStopping] = useState(false); const [log, setLog] = useState(''); + const [logTruncated, setLogTruncated] = useState(false); const [status, setStatus] = useState(''); const [backgroundMode, setBackgroundMode] = useState(false); const [configLoaded, setConfigLoaded] = useState(false); @@ -191,6 +193,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { setBusy(true); setStopping(false); setLog(''); + setLogTruncated(false); setStatus('running'); setBackgroundMode(false); if (jobCommand) { @@ -210,6 +213,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { pollStopRef.current = pollPipelineJob(jobId, (job) => { const displayLog = formatPipelineJobLog(job.log, job.error); setLog(displayLog); + setLogTruncated(Boolean(job.logTruncated)); setStatus(job.status); if (job.status === 'success' || job.status === 'error') { stopPoll(); @@ -310,13 +314,23 @@ export function PipelineProvider({ children }: { children: ReactNode }) { try { const res = await fetch(apiUrl('/jobs?limit=1')); const data = await res.json().catch(() => ({})); - if (cancelled || !res.ok) return; - const active = data.active as { id?: string; jobType?: string } | null; - if (active?.id) { + if (cancelled) return; + if (!res.ok) { + const errMsg = String(data.error || res.statusText || s.resumeJobFailed); + setLoadError(errMsg); + logPipelineFailure('Resume active job failed', { status: res.status, error: errMsg }); + return; + } + const active = data.active as { id?: string; jobType?: string; status?: string } | null; + if (active?.id && active.status === 'running') { watchJob(active.id, { navigate: false, jobCommand: active.jobType || '' }); } - } catch { - /* non-fatal */ + } catch (e) { + if (!cancelled) { + const errMsg = e instanceof Error ? e.message : s.resumeJobFailed; + setLoadError(errMsg); + logPipelineFailure('Resume active job failed', { error: e, message: errMsg }); + } } })(); return () => { @@ -398,7 +412,14 @@ export function PipelineProvider({ children }: { children: ReactNode }) { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ preset }), - }).catch(() => {}); + }) + .then(async (res) => { + if (!res.ok) { + const data = await res.json().catch(() => ({})); + setSaveMsg(String(data.error || s.presetSaveFailed)); + } + }) + .catch(() => setSaveMsg(s.presetSaveFailed)); } }, [configState.active_property_id]); @@ -492,6 +513,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { stopPoll(); setBusy(true); setLog(''); + setLogTruncated(false); setStatus('starting'); setBackgroundMode(false); try { @@ -597,6 +619,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { stopping, activeJobId: activeJobIdRef.current, log, + logTruncated, status, backgroundMode, startUrl: String(configState.start_url ?? ''), @@ -640,6 +663,7 @@ export function PipelineProvider({ children }: { children: ReactNode }) { busy, stopping, log, + logTruncated, status, backgroundMode, browserCrawlStatus, diff --git a/web/src/lib/pipelineDebug.ts b/web/src/lib/pipelineDebug.ts index 79607aca..42646bf3 100644 --- a/web/src/lib/pipelineDebug.ts +++ b/web/src/lib/pipelineDebug.ts @@ -1,3 +1,9 @@ +/** Server-side pipeline DB write failures (visible in Next.js terminal). */ +export function logPipelineDbError(action: string, err: unknown): void { + const msg = err instanceof Error ? err.message : String(err); + console.error(`[pipeline-db] ${action}: ${msg}`); +} + /** Structured pipeline errors in the browser console (devTools). */ export function logPipelineFailure( context: string, diff --git a/web/src/lib/pipelineJobEvents.ts b/web/src/lib/pipelineJobEvents.ts index f9fb09db..d52a73fa 100644 --- a/web/src/lib/pipelineJobEvents.ts +++ b/web/src/lib/pipelineJobEvents.ts @@ -39,6 +39,7 @@ type JobPollUpdate = { status: PipelineJob['status']; log: string; error?: string | null; + logTruncated?: boolean; }; /** @@ -64,8 +65,12 @@ export function pollPipelineJob( if (cancelled) return; try { const res = await fetch(jobPath); - const data: { status?: string; log?: string; error?: string | null } = - await res.json().catch(() => ({})); + const data: { + status?: string; + log?: string; + error?: string | null; + logTruncated?: boolean; + } = await res.json().catch(() => ({})); if (cancelled) return; if (!res.ok) { const errMsg = data.error || res.statusText; @@ -81,7 +86,7 @@ export function pollPipelineJob( const status = (data.status as PipelineJob['status']) || 'error'; const log = data.log || ''; const error = data.error ?? null; - onUpdate({ status, log, error }); + onUpdate({ status, log, error, logTruncated: Boolean(data.logTruncated) }); if (status === 'success' || status === 'error') { finish(); } diff --git a/web/src/server/jobsRoute.test.ts b/web/src/server/jobsRoute.test.ts index 38904a99..9f36d72f 100644 --- a/web/src/server/jobsRoute.test.ts +++ b/web/src/server/jobsRoute.test.ts @@ -1,25 +1,21 @@ import { describe, expect, it, vi, beforeEach } from 'vitest'; import { localRequest, remoteRequest } from '@/server/testHelpers/routeTestUtils'; -const reconcileMock = vi.fn(); -const listJobsMock = vi.fn(); -const activeJobMock = vi.fn(); +const listForApiMock = vi.fn(); -vi.mock('@/server/pipelineJobsDb', () => ({ - reconcileStaleRunningJobs: (...args: unknown[]) => reconcileMock(...args), - listRecentPipelineJobs: (...args: unknown[]) => listJobsMock(...args), - getActiveRunningJob: (...args: unknown[]) => activeJobMock(...args), +vi.mock('@/server/pipelineJobs', () => ({ + listPipelineJobsForApi: (...args: unknown[]) => listForApiMock(...args), })); describe('jobs route', () => { beforeEach(() => { - reconcileMock.mockReset(); - listJobsMock.mockReset(); - activeJobMock.mockReset(); + listForApiMock.mockReset(); vi.resetModules(); - reconcileMock.mockResolvedValue(0); - listJobsMock.mockResolvedValue([{ id: 'j1', status: 'completed' }]); - activeJobMock.mockResolvedValue(null); + listForApiMock.mockResolvedValue({ + jobs: [{ id: 'j1', status: 'completed' }], + active: null, + reconciled: 0, + }); }); it('returns 403 for non-local host', async () => { diff --git a/web/src/server/pipelineJobs.test.ts b/web/src/server/pipelineJobs.test.ts new file mode 100644 index 00000000..6ec54fd9 --- /dev/null +++ b/web/src/server/pipelineJobs.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; +import { PIPELINE_LOG_MAX, PIPELINE_LOG_TRIM } from '@/server/pipelineJobsDb'; + +const tryClaimMock = vi.fn(); +const reconcileMock = vi.fn(); +const appendLogMock = vi.fn(); +const finishMock = vi.fn(); +const getJobFromDbMock = vi.fn(); +const getActiveMock = vi.fn(); +const listRecentMock = vi.fn(); +const markOrphanMock = vi.fn(); + +vi.mock('@/server/pipelineJobsDb', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + tryClaimRunningPipelineJob: (...args: unknown[]) => tryClaimMock(...args), + reconcileStaleRunningJobs: (...args: unknown[]) => reconcileMock(...args), + appendPipelineJobLog: (...args: unknown[]) => appendLogMock(...args), + finishPipelineJob: (...args: unknown[]) => finishMock(...args), + getPipelineJobFromDb: (...args: unknown[]) => getJobFromDbMock(...args), + getActiveRunningJob: (...args: unknown[]) => getActiveMock(...args), + listRecentPipelineJobs: (...args: unknown[]) => listRecentMock(...args), + markRunningJobOrphaned: (...args: unknown[]) => markOrphanMock(...args), + }; +}); + +const spawnMock = vi.fn(); + +vi.mock('child_process', () => ({ + spawn: (...args: unknown[]) => spawnMock(...args), +})); + +function makeProc() { + const handlers: Record void>> = {}; + return { + stdout: { on: vi.fn() }, + stderr: { on: vi.fn() }, + killed: false, + kill: vi.fn(), + on(event: string, fn: (arg?: unknown) => void) { + handlers[event] = handlers[event] || []; + handlers[event].push(fn); + }, + emit(event: string, arg?: unknown) { + for (const fn of handlers[event] || []) fn(arg); + }, + }; +} + +describe('pipelineJobs', () => { + beforeEach(() => { + vi.resetModules(); + delete process.env.DATABASE_URL; + delete globalThis.__websiteProfilingPipelineJobs; + delete globalThis.__websiteProfilingPipelineProcesses; + tryClaimMock.mockReset(); + reconcileMock.mockReset(); + appendLogMock.mockReset(); + finishMock.mockReset(); + getJobFromDbMock.mockReset(); + getActiveMock.mockReset(); + listRecentMock.mockReset(); + markOrphanMock.mockReset(); + spawnMock.mockReset(); + reconcileMock.mockResolvedValue(0); + listRecentMock.mockResolvedValue([]); + getActiveMock.mockResolvedValue(null); + markOrphanMock.mockResolvedValue(false); + finishMock.mockResolvedValue(undefined); + appendLogMock.mockResolvedValue(false); + }); + + afterEach(() => { + delete process.env.DATABASE_URL; + }); + + it('startPipelineJobAsync throws when in-memory job already running (no DB)', async () => { + const { startPipelineJobAsync } = await import('@/server/pipelineJobs'); + const proc = makeProc(); + spawnMock.mockReturnValue(proc); + + const id = await startPipelineJobAsync('crawl', null, {}); + expect(id).toBeTruthy(); + + await expect(startPipelineJobAsync('crawl', null, {})).rejects.toThrow( + /already running/i, + ); + }); + + it('startPipelineJobAsync rejects when atomic claim fails', async () => { + process.env.DATABASE_URL = 'postgres://local/test'; + tryClaimMock.mockResolvedValueOnce(true).mockResolvedValueOnce(false); + const proc = makeProc(); + spawnMock.mockReturnValue(proc); + + const { startPipelineJobAsync } = await import('@/server/pipelineJobs'); + const first = await startPipelineJobAsync('crawl', null, {}); + expect(first).toBeTruthy(); + proc.emit('close', 0); + await new Promise((r) => setTimeout(r, 0)); + + await expect(startPipelineJobAsync('crawl', null, {})).rejects.toThrow( + /already running/i, + ); + expect(tryClaimMock).toHaveBeenCalledTimes(2); + expect(spawnMock).toHaveBeenCalledTimes(1); + }); + + it('sets logTruncated when in-memory log exceeds cap', async () => { + const proc = makeProc(); + spawnMock.mockReturnValue(proc); + + const { startPipelineJobAsync, getJobSync } = await import('@/server/pipelineJobs'); + const id = await startPipelineJobAsync('crawl', null, {}); + const dataCall = proc.stdout.on.mock.calls.find((call) => call[0] === 'data'); + const dataHandler = dataCall?.[1] as ((c: Buffer) => void) | undefined; + expect(dataHandler).toBeDefined(); + dataHandler?.(Buffer.from('x'.repeat(PIPELINE_LOG_MAX + 1))); + + const updated = getJobSync(id); + expect(updated?.log.length).toBeLessThanOrEqual(PIPELINE_LOG_TRIM); + expect(updated?.logTruncated).toBe(true); + }); + + it('listPipelineJobsForApi reconciles orphan jobs without live process', async () => { + process.env.DATABASE_URL = 'postgres://local/test'; + const startedAt = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + reconcileMock.mockResolvedValue(0); + getActiveMock + .mockResolvedValueOnce({ + id: 'job-orphan', + jobType: 'crawl', + status: 'running', + propertyId: null, + startedAt, + finishedAt: null, + exitCode: null, + error: null, + }) + .mockResolvedValueOnce(null); + markOrphanMock.mockResolvedValue(true); + listRecentMock.mockResolvedValue([]); + + const { listPipelineJobsForApi } = await import('@/server/pipelineJobs'); + const result = await listPipelineJobsForApi(5); + expect(markOrphanMock).toHaveBeenCalledWith('job-orphan'); + expect(result.reconciled).toBe(1); + expect(result.active).toBeNull(); + }); +}); diff --git a/web/src/server/pipelineJobs.ts b/web/src/server/pipelineJobs.ts index ab7a63e3..689eac4c 100644 --- a/web/src/server/pipelineJobs.ts +++ b/web/src/server/pipelineJobs.ts @@ -5,14 +5,20 @@ import { randomUUID } from 'crypto'; import { getPipelineSpawnEnv } from '@/server/pipelineSpawnEnv'; import { formatPythonSpawnError, resolvePythonExecutable } from '@/server/resolvePython'; import { buildPipelineJobErrorMessage } from '@/lib/pipelineJobErrorMessage'; +import { logPipelineDbError } from '@/lib/pipelineDebug'; import { appendPipelineJobLog, cancelPipelineJobInDb, finishPipelineJob, + getActiveRunningJob, getPipelineJobFromDb, - insertPipelineJob, - isAnyPipelineJobRunning, + listRecentPipelineJobs, + markRunningJobOrphaned, + PIPELINE_LOG_MAX, + PIPELINE_LOG_TRIM, reconcileStaleRunningJobs, + tryClaimRunningPipelineJob, + type PipelineJobListItem, } from '@/server/pipelineJobsDb'; import type { PipelineJob, PipelineJobEntry, PipelineJobStore } from '@/types/api'; @@ -22,6 +28,7 @@ function isDbJobsEnabled(): boolean { const WEB_CWD = process.cwd(); const DEFAULT_REPO_ROOT = process.env.WEBSITE_PROFILING_ROOT || path.resolve(WEB_CWD, '..'); +const ORPHAN_JOB_MINUTES = Number(process.env.PIPELINE_JOB_ORPHAN_MINUTES || '5'); const ALLOWED_COMMANDS = new Set([ null, @@ -38,11 +45,6 @@ const ALLOWED_COMMANDS = new Set([ 'google', ]); -/** - * Next may bundle this module into separate server chunks per API route, so module-level - * `Map` instances are not shared. Persist store on globalThis so POST /api/run and GET - * /api/jobs/[id] always see the same jobs (also survives dev Fast Refresh better). - */ function getStore(): PipelineJobStore { if (!globalThis.__websiteProfilingPipelineJobs) { globalThis.__websiteProfilingPipelineJobs = { @@ -62,6 +64,14 @@ function getProcessMap(): Map { const CANCELLED_MESSAGE = 'Cancelled by user'; +function trimInMemoryLog(entry: PipelineJobEntry, chunk: string): void { + entry.log += chunk; + if (entry.log.length > PIPELINE_LOG_MAX) { + entry.log = entry.log.slice(-PIPELINE_LOG_TRIM); + entry.logTruncated = true; + } +} + function markJobFinished( id: string, entry: PipelineJobEntry, @@ -77,7 +87,9 @@ function markJobFinished( getStore().running = false; getProcessMap().delete(id); if (isDbJobsEnabled()) { - void finishPipelineJob(id, status, exitCode, error).catch(() => {}); + void finishPipelineJob(id, status, exitCode, error, entry.logTruncated).catch((err) => + logPipelineDbError('finishPipelineJob', err), + ); } } @@ -103,13 +115,6 @@ function resolveRepoRoot(override: string | undefined | null): string { return normalized; } -/** - * Resolve and validate an absolute config file path. - * - * Allowed locations: - * - Under repoRoot (always) - * - Under DATA_DIR — the data volume (/data) in Docker - */ function validateConfigPath(absPath: string, repoRoot: string): string { const normalized = path.resolve(absPath); @@ -127,40 +132,70 @@ export interface StartPipelineJobOptions { propertyId?: number | null; } -/** - * Start a pipeline job. When configAbsPath is omitted (the normal UI flow), - * Python picks up settings from the pipeline_config table via DATABASE_URL. - */ -export async function assertNoRunningJob(): Promise { - if (isDbJobsEnabled()) { - await reconcileStaleRunningJobs(); - if (await isAnyPipelineJobRunning()) { - throw new Error('An audit job is already running'); +function hasLiveProcess(jobId: string): boolean { + const proc = getProcessMap().get(jobId); + return Boolean(proc && !proc.killed); +} + +async function reconcileOrphanedActiveJob(active: PipelineJobListItem): Promise { + if (hasLiveProcess(active.id)) return false; + const started = new Date(active.startedAt).getTime(); + if (Number.isNaN(started) || Date.now() - started < ORPHAN_JOB_MINUTES * 60 * 1000) { + return false; + } + const updated = await markRunningJobOrphaned(active.id); + if (updated) { + getStore().running = false; + const entry = getStore().jobs.get(active.id); + if (entry && !entry.finished) { + markJobFinished(active.id, entry, 'error', -1, 'Job process not found (server restarted)'); } - return; } - if (getStore().running) { - throw new Error('An audit job is already running'); + return updated; +} + +export interface PipelineJobsListResult { + jobs: PipelineJobListItem[]; + active: PipelineJobListItem | null; + reconciled: number; +} + +/** List jobs for GET /api/jobs with stale + orphan reconciliation. */ +export async function listPipelineJobsForApi(limit: number): Promise { + let reconciled = await reconcileStaleRunningJobs(); + let active = await getActiveRunningJob(); + if (active) { + const orphanReconciled = await reconcileOrphanedActiveJob(active); + if (orphanReconciled) { + reconciled += 1; + active = await getActiveRunningJob(); + } } + const jobs = await listRecentPipelineJobs(limit); + return { jobs, active, reconciled }; } -export function startPipelineJob( +/** + * Start a pipeline job. When configAbsPath is omitted (the normal UI flow), + * Python picks up settings from the pipeline_config table via DATABASE_URL. + */ +export async function startPipelineJobAsync( command: string | null | undefined, configAbsPath: string | null | undefined, options: StartPipelineJobOptions = {}, -): string { +): Promise { if (command != null && command !== '' && !ALLOWED_COMMANDS.has(command)) { throw new Error('Invalid command'); } + const store = getStore(); - if (!isDbJobsEnabled() && store.running) { + if (store.running) { throw new Error('An audit job is already running'); } const repoRoot = resolveRepoRoot(options.repoRoot); const pythonExe = sanitizePython(options.python, repoRoot); - // Validate config path only when explicitly provided let cfgPath: string | null = null; if (configAbsPath != null && String(configAbsPath).trim() !== '') { cfgPath = validateConfigPath(String(configAbsPath), repoRoot); @@ -170,23 +205,31 @@ export function startPipelineJob( } const id = randomUUID(); + const jobType = command?.split(/\s+/)[0] || 'full'; + + if (isDbJobsEnabled()) { + const claimed = await tryClaimRunningPipelineJob( + id, + jobType, + options.propertyId ?? null, + null, + ); + if (!claimed) { + throw new Error('An audit job is already running'); + } + } + const entry: PipelineJobEntry = { status: 'running', exitCode: null, log: '', + logTruncated: false, }; store.jobs.set(id, entry); store.running = true; - const jobType = command?.split(/\s+/)[0] || 'full'; - if (isDbJobsEnabled()) { - void insertPipelineJob(id, jobType, options.propertyId ?? null, null).catch(() => {}); - } - - // When no explicit config path is given, Python reads from PostgreSQL via DATABASE_URL. const args = ['-m', 'src']; if (cfgPath) args.push('--config', cfgPath); - // Support multi-word commands like "keywords --enrich-google" if (command) args.push(...command.split(/\s+/).filter(Boolean)); const proc = spawn(pythonExe, args, { @@ -198,12 +241,13 @@ export function startPipelineJob( const append = (chunk: Buffer | string): void => { const text = chunk.toString(); - entry.log += text; - if (entry.log.length > 256_000) { - entry.log = entry.log.slice(-200_000); - } + trimInMemoryLog(entry, text); if (isDbJobsEnabled()) { - void appendPipelineJobLog(id, text).catch(() => {}); + void appendPipelineJobLog(id, text) + .then((truncated) => { + if (truncated) entry.logTruncated = true; + }) + .catch((err) => logPipelineDbError('appendPipelineJobLog', err)); } }; @@ -233,17 +277,45 @@ export function startPipelineJob( return id; } +/** @deprecated Use startPipelineJobAsync */ +export async function assertNoRunningJob(): Promise { + if (isDbJobsEnabled()) { + await reconcileStaleRunningJobs(); + const active = await getActiveRunningJob(); + if (active) { + await reconcileOrphanedActiveJob(active); + const stillActive = await getActiveRunningJob(); + if (stillActive) throw new Error('An audit job is already running'); + } + return; + } + if (getStore().running) { + throw new Error('An audit job is already running'); + } +} + export async function getJob(id: string): Promise { + const memory = getStore().jobs.get(id); if (isDbJobsEnabled()) { const fromDb = await getPipelineJobFromDb(id); - if (fromDb) return fromDb; - // Fall back to in-memory for jobs started in this process before DB insert completes. - return getStore().jobs.get(id) ?? null; + if (fromDb) { + if (memory) { + return { + ...fromDb, + log: memory.log.length >= fromDb.log.length ? memory.log : fromDb.log, + logTruncated: memory.logTruncated || fromDb.logTruncated, + status: memory.finished ? memory.status : fromDb.status, + exitCode: memory.finished ? memory.exitCode : fromDb.exitCode, + error: memory.error ?? fromDb.error, + }; + } + return fromDb; + } + return memory ?? null; } - return getStore().jobs.get(id) ?? null; + return memory ?? null; } -/** Sync read from in-memory cache only (legacy). */ export function getJobSync(id: string): PipelineJob | null { return getStore().jobs.get(id) ?? null; } @@ -254,10 +326,6 @@ export interface CancelPipelineJobResult { error?: string; } -/** - * Stop a running pipeline job. Kills the child process when this server instance - * spawned it; otherwise marks the DB row cancelled (best effort after restart). - */ export async function cancelPipelineJob(id: string): Promise { const trimmed = id.trim(); if (!trimmed) { @@ -272,9 +340,13 @@ export async function cancelPipelineJob(id: string): Promise {}); + void appendPipelineJobLog(trimmed, cancelLine) + .then((truncated) => { + if (truncated) entry.logTruncated = true; + }) + .catch((err) => logPipelineDbError('appendPipelineJobLog', err)); } try { proc.kill(); diff --git a/web/src/server/pipelineJobsDb.ts b/web/src/server/pipelineJobsDb.ts index 6a5491d6..73ad2512 100644 --- a/web/src/server/pipelineJobsDb.ts +++ b/web/src/server/pipelineJobsDb.ts @@ -3,40 +3,107 @@ import type { PoolClient } from 'pg'; import { withDb } from '@/server/db'; import type { PipelineJob } from '@/types/api'; -const LOG_MAX = 256_000; -const LOG_TRIM = 200_000; +export const PIPELINE_LOG_MAX = 256_000; +export const PIPELINE_LOG_TRIM = 200_000; + +const STALE_JOB_HOURS = Number(process.env.PIPELINE_JOB_STALE_HOURS || '1'); export function hashConfig(configPath: string | null): string | null { if (!configPath) return null; return createHash('sha256').update(configPath).digest('hex').slice(0, 16); } -export async function insertPipelineJob( +function trimPipelineLog(log: string): { log: string; truncated: boolean } { + if (log.length <= PIPELINE_LOG_MAX) { + return { log, truncated: false }; + } + return { log: log.slice(-PIPELINE_LOG_TRIM), truncated: true }; +} + +async function reconcileStaleRunningJobsWithClient(client: PoolClient): Promise { + const cur = await client.query<{ id: string }>( + `UPDATE pipeline_jobs + SET status = 'error', + error_text = COALESCE(error_text, 'Job interrupted (server restart or timeout)'), + finished_at = now() + WHERE status = 'running' + AND started_at < now() - ($1::text || ' hours')::interval + RETURNING id::text`, + [String(STALE_JOB_HOURS)], + ); + return cur.rowCount ?? 0; +} + +/** Mark jobs stuck in running state as error (e.g. after server restart). */ +export async function reconcileStaleRunningJobs(): Promise { + return withDb(async (client) => reconcileStaleRunningJobsWithClient(client)); +} + +/** + * Atomically claim the single running pipeline slot. + * Reconciles stale jobs in the same transaction before insert. + */ +export async function tryClaimRunningPipelineJob( id: string, jobType: string, propertyId: number | null, configHash: string | null, -): Promise { - await withDb(async (client) => { - await client.query( - `INSERT INTO pipeline_jobs (id, job_type, status, property_id, config_hash) - VALUES ($1::uuid, $2, 'running', $3, $4)`, - [id, jobType, propertyId, configHash], +): Promise { + return withDb(async (client) => { + await client.query('BEGIN'); + try { + await reconcileStaleRunningJobsWithClient(client); + const cur = await client.query<{ id: string }>( + `INSERT INTO pipeline_jobs (id, job_type, status, property_id, config_hash) + SELECT $1::uuid, $2, 'running', $3, $4 + WHERE NOT EXISTS (SELECT 1 FROM pipeline_jobs WHERE status = 'running') + RETURNING id::text`, + [id, jobType, propertyId, configHash], + ); + await client.query('COMMIT'); + return (cur.rowCount ?? 0) > 0; + } catch (e) { + await client.query('ROLLBACK'); + throw e; + } + }); +} + +export async function markRunningJobOrphaned( + id: string, + message = 'Job process not found (server restarted)', +): Promise { + return withDb(async (client) => { + const cur = await client.query<{ id: string }>( + `UPDATE pipeline_jobs + SET status = 'error', + error_text = $2, + exit_code = -1, + finished_at = now() + WHERE id = $1::uuid AND status = 'running' + RETURNING id::text`, + [id, message], ); + return (cur.rowCount ?? 0) > 0; }); } -export async function appendPipelineJobLog(id: string, chunk: string): Promise { - await withDb(async (client) => { - const cur = await client.query<{ log_text: string }>( - `SELECT log_text FROM pipeline_jobs WHERE id = $1::uuid FOR UPDATE`, +export async function appendPipelineJobLog(id: string, chunk: string): Promise { + return withDb(async (client) => { + const cur = await client.query<{ log_text: string; log_truncated: boolean }>( + `SELECT log_text, log_truncated FROM pipeline_jobs WHERE id = $1::uuid FOR UPDATE`, [id], ); const row = cur.rows[0]; - if (!row) return; - let log = (row.log_text || '') + chunk; - if (log.length > LOG_MAX) log = log.slice(-LOG_TRIM); - await client.query(`UPDATE pipeline_jobs SET log_text = $2 WHERE id = $1::uuid`, [id, log]); + if (!row) return false; + const combined = (row.log_text || '') + chunk; + const { log, truncated } = trimPipelineLog(combined); + const logTruncated = row.log_truncated || truncated; + await client.query( + `UPDATE pipeline_jobs SET log_text = $2, log_truncated = $3 WHERE id = $1::uuid`, + [id, log, logTruncated], + ); + return logTruncated; }); } @@ -64,13 +131,23 @@ export async function finishPipelineJob( status: 'success' | 'error', exitCode: number | null, error?: string, + logTruncated?: boolean, ): Promise { await withDb(async (client) => { + if (logTruncated === undefined) { + await client.query( + `UPDATE pipeline_jobs + SET status = $2, exit_code = $3, error_text = $4, finished_at = now() + WHERE id = $1::uuid`, + [id, status, exitCode, error ?? null], + ); + return; + } await client.query( `UPDATE pipeline_jobs - SET status = $2, exit_code = $3, error_text = $4, finished_at = now() + SET status = $2, exit_code = $3, error_text = $4, finished_at = now(), log_truncated = $5 WHERE id = $1::uuid`, - [id, status, exitCode, error ?? null], + [id, status, exitCode, error ?? null, logTruncated], ); }); } @@ -82,8 +159,9 @@ export async function getPipelineJobFromDb(id: string): Promise( - `SELECT status, exit_code, log_text, error_text FROM pipeline_jobs WHERE id = $1::uuid`, + `SELECT status, exit_code, log_text, error_text, log_truncated FROM pipeline_jobs WHERE id = $1::uuid`, [id], ); const row = cur.rows[0]; @@ -94,6 +172,7 @@ export async function getPipelineJobFromDb(id: string): Promise { - return withDb(async (client) => { - const cur = await client.query<{ id: string }>( - `UPDATE pipeline_jobs - SET status = 'error', - error_text = COALESCE(error_text, 'Job interrupted (server restart or timeout)'), - finished_at = now() - WHERE status = 'running' - AND started_at < now() - ($1::text || ' hours')::interval - RETURNING id::text`, - [String(STALE_JOB_HOURS)], - ); - return cur.rowCount ?? 0; - }); -} - export async function listRecentPipelineJobs(limit = 20): Promise { return withDb(async (client) => { const cur = await client.query<{ diff --git a/web/src/strings.json b/web/src/strings.json index 1b5010e7..ddfc2687 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -200,6 +200,9 @@ "stopJobFailed": "Could not stop the audit: {message}", "outputLabel": "Audit run log", "outputTitle": "Audit run log", + "logTruncatedBanner": "Older log lines were trimmed to save memory. Download log for the tail snapshot.", + "presetSaveFailed": "Could not save crawl preset for this property.", + "resumeJobFailed": "Could not resume the active audit job.", "setupStepsAria": "Run audit setup steps", "consoleFilterHint": "Site Audit run", "statusLabel": "Status", @@ -306,6 +309,7 @@ "saving": "Saving…", "saved": "Ops settings saved.", "saveFailed": "Could not save ops settings.", + "loadFailed": "Could not load ops settings.", "cronEndpointsHint": "Wire cron: curl -X POST http://localhost:3000/api/schedule/check and /api/alerts/check?propertyId={id}", "gscLinksMissing": "No GSC Links CSV imported yet — upload from Search Console → Links for backlink velocity and competitor gap.", "gscLinksStale": "GSC Links import is {days} days old — re-import weekly for accurate velocity." @@ -1990,6 +1994,7 @@ "baselineLabel": "Baseline (older)", "needTwoReports": "Run at least two full site audits (not crawl-only) for this domain to compare snapshots.", "pickBaseline": "Select a baseline report to compare against the current report.", + "exportCsvFailed": "Could not export issue diff CSV.", "noDifferences": "No URL-level changes between these two audits.", "noneInCategory": "None in this category.", "urlChangeListsUnavailable": "Per-URL change lists are missing from one or both audits. Site-wide metrics below may still apply. Run a full site audit to enable URL-level comparison.", @@ -2126,6 +2131,7 @@ "browserPageErrorsLine": "{pages} page(s) had uncaught JavaScript errors during rendering.", "viewJavaScriptErrors": "View JavaScript errors" }, + "historyTrendUnavailable": "Could not load health trend vs prior run.", "googleStaleWarning": "Search Console & Analytics data is older than 7 days. Refresh from Integrations.", "googlePartialWarning": "Search Console or Analytics data is missing from the last sync.", "mlErrors": "AI insights reported {count} error{plural}", diff --git a/web/src/types/api.ts b/web/src/types/api.ts index 66f75c23..fe4e7629 100644 --- a/web/src/types/api.ts +++ b/web/src/types/api.ts @@ -23,6 +23,7 @@ export interface PipelineJob { exitCode: number | null; log: string; error?: string; + logTruncated?: boolean; } /** In-memory job entry (server only). */ @@ -81,6 +82,7 @@ export interface JobStatusResponse { exitCode: number | null; log: string; error: string | null; + logTruncated?: boolean; } export interface RunPostResponse { diff --git a/web/src/views/CompareReports.tsx b/web/src/views/CompareReports.tsx index 5257949d..5caaa229 100644 --- a/web/src/views/CompareReports.tsx +++ b/web/src/views/CompareReports.tsx @@ -343,11 +343,14 @@ export default function CompareReports({ searchQuery = '' }: ViewProps) { a.click(); URL.revokeObjectURL(url); }) - .catch(() => {}); + .catch(() => setCopyHint(vc.exportCsvFailed)); }} > Export issue diff (CSV) + {copyHint ? ( + {copyHint} + ) : null}
) : null} {reportList.length >= 2 && compareReportId == null && !loading && !error ? ( From 74941cc88f99713ce43cf40952ac9fa7cfbca903 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 04:08:36 +0530 Subject: [PATCH 2/5] updatwed --- .github/workflows/ci.yml | 2 + scripts/local-test.ps1 | 4 + scripts/local-test.sh | 4 + src/website_profiling/common.py | 626 +-------- src/website_profiling/crawl/config.py | 117 ++ src/website_profiling/crawl/crawler.py | 772 +++-------- src/website_profiling/crawl/db_writer.py | 59 + src/website_profiling/crawl/frontier.py | 156 +++ src/website_profiling/crawl/page_record.py | 226 +++ src/website_profiling/crawl/schema.py | 154 +++ src/website_profiling/lighthouse/config.py | 166 +++ .../lighthouse/result_parser.py | 107 ++ src/website_profiling/lighthouse/runner.py | 174 +-- src/website_profiling/parsing/__init__.py | 1 + src/website_profiling/parsing/content.py | 108 ++ src/website_profiling/parsing/io.py | 72 + src/website_profiling/parsing/links.py | 155 +++ src/website_profiling/parsing/robots.py | 17 + src/website_profiling/parsing/seo.py | 138 ++ src/website_profiling/parsing/tech.py | 116 ++ src/website_profiling/reporting/builder.py | 1217 +---------------- src/website_profiling/reporting/categories.py | 1053 -------------- .../reporting/categories/__init__.py | 113 ++ .../reporting/categories/_helpers.py | 252 ++++ .../reporting/categories/accessibility.py | 188 +++ .../reporting/categories/intelligence.py | 68 + .../reporting/categories/link_health.py | 89 ++ .../reporting/categories/mobile.py | 62 + .../reporting/categories/performance.py | 160 +++ .../reporting/categories/security.py | 117 ++ .../reporting/categories/technical_seo.py | 211 +++ .../reporting/content_analytics.py | 434 ++++++ .../reporting/edges_report.py | 114 ++ .../reporting/lighthouse_report.py | 195 +++ .../reporting/report_metadata.py | 239 ++++ .../reporting/seo_summary.py | 166 +++ src/website_profiling/reporting/site_level.py | 49 + src/website_profiling/tools/export_audit.py | 560 +------- .../tools/export_audit_data.py | 216 +++ .../tools/export_audit_html.py | 343 +++++ tests/test_categories_coverage.py | 2 +- tests/test_crawl_frontier.py | 70 + tests/test_crawl_gap_coverage.py | 4 +- tests/test_crawler_deep.py | 17 +- tests/test_crawler_unit.py | 10 +- tests/test_page_record.py | 50 + tests/test_property_profile.py | 2 +- tests/test_reporting_builder_modules.py | 839 ++++++++++++ 48 files changed, 5925 insertions(+), 4089 deletions(-) create mode 100644 src/website_profiling/crawl/config.py create mode 100644 src/website_profiling/crawl/db_writer.py create mode 100644 src/website_profiling/crawl/frontier.py create mode 100644 src/website_profiling/crawl/page_record.py create mode 100644 src/website_profiling/crawl/schema.py create mode 100644 src/website_profiling/lighthouse/config.py create mode 100644 src/website_profiling/lighthouse/result_parser.py create mode 100644 src/website_profiling/parsing/__init__.py create mode 100644 src/website_profiling/parsing/content.py create mode 100644 src/website_profiling/parsing/io.py create mode 100644 src/website_profiling/parsing/links.py create mode 100644 src/website_profiling/parsing/robots.py create mode 100644 src/website_profiling/parsing/seo.py create mode 100644 src/website_profiling/parsing/tech.py delete mode 100644 src/website_profiling/reporting/categories.py create mode 100644 src/website_profiling/reporting/categories/__init__.py create mode 100644 src/website_profiling/reporting/categories/_helpers.py create mode 100644 src/website_profiling/reporting/categories/accessibility.py create mode 100644 src/website_profiling/reporting/categories/intelligence.py create mode 100644 src/website_profiling/reporting/categories/link_health.py create mode 100644 src/website_profiling/reporting/categories/mobile.py create mode 100644 src/website_profiling/reporting/categories/performance.py create mode 100644 src/website_profiling/reporting/categories/security.py create mode 100644 src/website_profiling/reporting/categories/technical_seo.py create mode 100644 src/website_profiling/reporting/content_analytics.py create mode 100644 src/website_profiling/reporting/edges_report.py create mode 100644 src/website_profiling/reporting/lighthouse_report.py create mode 100644 src/website_profiling/reporting/report_metadata.py create mode 100644 src/website_profiling/reporting/seo_summary.py create mode 100644 src/website_profiling/reporting/site_level.py create mode 100644 src/website_profiling/tools/export_audit_data.py create mode 100644 src/website_profiling/tools/export_audit_html.py create mode 100644 tests/test_crawl_frontier.py create mode 100644 tests/test_page_record.py create mode 100644 tests/test_reporting_builder_modules.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a4389d5..a515a0a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,8 @@ jobs: tests/test_indexation_coverage.py tests/test_crawl_segments.py \ tests/test_terminology.py tests/test_compare_payload.py \ tests/test_optional_audits.py tests/test_property_profile.py tests/test_reporting_gaps.py \ + tests/test_text_content_analysis.py tests/test_builder_image_buckets.py \ + tests/test_pipeline_report_pool_unit.py tests/test_reporting_builder_modules.py \ --cov=website_profiling.reporting --cov-config=.coveragerc.reporting \ --cov-report=term-missing --cov-fail-under=100 -q -o addopts= - name: Pytest (tools coverage gate) diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1 index ef9c7297..db9e8dc4 100644 --- a/scripts/local-test.ps1 +++ b/scripts/local-test.ps1 @@ -237,6 +237,10 @@ function Invoke-PytestReporting { tests/test_optional_audits.py ` tests/test_property_profile.py ` tests/test_reporting_gaps.py ` + tests/test_text_content_analysis.py ` + tests/test_builder_image_buckets.py ` + tests/test_pipeline_report_pool_unit.py ` + tests/test_reporting_builder_modules.py ` --cov=website_profiling.reporting ` --cov-config=.coveragerc.reporting ` --cov-report=term-missing ` diff --git a/scripts/local-test.sh b/scripts/local-test.sh index 3d32f7e5..b28d24bc 100755 --- a/scripts/local-test.sh +++ b/scripts/local-test.sh @@ -126,6 +126,10 @@ run_pytest_reporting() { tests/test_optional_audits.py \ tests/test_property_profile.py \ tests/test_reporting_gaps.py \ + tests/test_text_content_analysis.py \ + tests/test_builder_image_buckets.py \ + tests/test_pipeline_report_pool_unit.py \ + tests/test_reporting_builder_modules.py \ --cov=website_profiling.reporting \ --cov-config=.coveragerc.reporting \ --cov-report=term-missing \ diff --git a/src/website_profiling/common.py b/src/website_profiling/common.py index b19c528b..899ebe09 100644 --- a/src/website_profiling/common.py +++ b/src/website_profiling/common.py @@ -1,575 +1,57 @@ """ -Shared helpers for crawler and report/plot scripts. +Shared helpers for crawler and report/plot scripts (re-export facade). """ -import json -import os -import warnings -from urllib.parse import urljoin, urldefrag, urlparse -import urllib.robotparser as robotparser -import ast -import math - -import pandas as pd -from bs4 import BeautifulSoup - - -def load_dataframe(path: str) -> pd.DataFrame: - """Load a DataFrame from CSV or JSON (by extension).""" - if not os.path.isfile(path): - raise FileNotFoundError(path) - path_lower = path.lower() - if path_lower.endswith(".json"): - return pd.read_json(path, orient="records") - return pd.read_csv(path) - - -def save_dataframe(df: pd.DataFrame, path: str) -> None: - """Save a DataFrame to CSV or JSON (by extension). Uses default_handler for JSON to avoid numpy types.""" - path_lower = path.lower() - if path_lower.endswith(".json"): - df.to_json(path, orient="records", indent=2, date_format="iso", default_handler=str) - else: - df.to_csv(path, index=False) - - -def load_edges(path: str) -> list[tuple[str, str]]: - """Load edge list from CSV or JSON (by extension). Returns list of (from_url, to_url).""" - if not os.path.isfile(path): - return [] - path_lower = path.lower() - try: - if path_lower.endswith(".json"): - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - if isinstance(data, list) and data and isinstance(data[0], dict): - return [(str(o.get("from", "")), str(o.get("to", ""))) for o in data if o.get("from") and o.get("to")] - return [] - edf = pd.read_csv(path) - if {"from", "to"}.issubset(edf.columns): - return [(str(a).rstrip("/"), str(b).rstrip("/")) for a, b in edf[["from", "to"]].values] - except Exception: - pass - return [] - - -def save_edges(edges: list[tuple[str, str]], path: str) -> None: - """Save edge list to CSV or JSON (by extension).""" - path_lower = path.lower() - if path_lower.endswith(".json"): - data = [{"from": a, "to": b} for a, b in edges] - with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - else: - pd.DataFrame(edges, columns=["from", "to"]).to_csv(path, index=False) - - -_TRACKING_PARAM_PREFIXES = ("utm_",) -_FACET_PARAM_NAMES = frozenset({"sort", "filter", "page", "offset", "limit"}) - - -def strip_crawl_query_params(url: str, ignore_params: list[str] | None = None) -> str: - """Remove tracking and facet query params for crawl deduplication.""" - parsed = urlparse(url) - if not parsed.query: - return url.rstrip("/") - ignore = {p.lower() for p in (ignore_params or [])} - parts = [] - for pair in parsed.query.split("&"): - if not pair: - continue - key = pair.split("=", 1)[0].lower() - if key in ignore: - continue - if any(key.startswith(p) for p in _TRACKING_PARAM_PREFIXES): - continue - if key in _FACET_PARAM_NAMES: - continue - parts.append(pair) - query = "&".join(parts) - rebuilt = parsed._replace(query=query).geturl() - return rebuilt.rstrip("/") - - -def normalize_link( - base: str, - href: str, - strip_params: bool = True, - ignore_params: list[str] | None = None, -) -> str | None: - if not href: - return None - href = href.strip() - if href.startswith(("mailto:", "javascript:", "tel:", "data:")): - return None - joined = urljoin(base, href) - joined, _ = urldefrag(joined) - parsed = urlparse(joined) - if parsed.scheme not in ("http", "https"): - return None - out = joined.rstrip("/") - if strip_params: - out = strip_crawl_query_params(out, ignore_params) - return out - - -def _parse_rel_flags(rel_raw: str) -> tuple[bool, bool, bool]: - parts = {p.strip().lower() for p in (rel_raw or "").split() if p.strip()} - return ("nofollow" in parts, "sponsored" in parts, "ugc" in parts) - - -def _anchor_text_from_tag(a) -> str: - parts: list[str] = [] - for child in a.children: - if getattr(child, "name", None) == "img": - parts.append("[image]") - elif isinstance(child, str): - t = child.strip() - if t: - parts.append(t) - text = " ".join(parts).strip() or a.get_text(separator=" ", strip=True) - return (text or "")[:500] - - -def parse_link_edges(base_url: str, html_text: str) -> tuple[str, list[dict]]: - """Extract title and rich outbound link records from HTML.""" - soup = BeautifulSoup(html_text, "lxml") - title_tag = ( - soup.title.string.strip() - if soup.title and soup.title.string - else "" - ) - start_netloc = urlparse(base_url).netloc - edges: list[dict] = [] - for a in soup.find_all("a", href=True): - ln = normalize_link(base_url, a["href"]) - if not ln: - continue - rel_raw = a.get("rel") or "" - if isinstance(rel_raw, list): - rel_str = " ".join(str(x) for x in rel_raw) - else: - rel_str = str(rel_raw) - nofollow, sponsored, ugc = _parse_rel_flags(rel_str) - link_type = "internal" if urlparse(ln).netloc == start_netloc else "external" - edges.append({ - "to_url": ln.rstrip("/"), - "anchor_text": _anchor_text_from_tag(a), - "rel": rel_str.strip(), - "is_nofollow": nofollow, - "is_sponsored": sponsored, - "is_ugc": ugc, - "link_type": link_type, - }) - return title_tag, edges - - -def parse_links(base_url: str, html_text: str) -> tuple[str, set[str]]: - """Extract page title and set of absolute links from HTML. Returns (title, links).""" - title, edges = parse_link_edges(base_url, html_text) - return title, {e["to_url"] for e in edges} - - -def parse_seo(base_url: str, html_text: str) -> tuple[str, int, str, int, str]: - """ - Extract SEO-related fields from HTML. - Returns (meta_description, meta_description_len, h1_text, h1_count, canonical_url). - """ - soup = BeautifulSoup(html_text, "lxml") - meta_desc = "" - meta = soup.find("meta", attrs={"name": "description"}) - if meta and meta.get("content"): - meta_desc = (meta["content"] or "").strip() - if not meta_desc: - og = soup.find("meta", attrs={"property": "og:description"}) - if og and og.get("content"): - meta_desc = (og["content"] or "").strip() - meta_desc_len = len(meta_desc) - - h1_tags = soup.find_all("h1") - h1_count = len(h1_tags) - h1_text = (h1_tags[0].get_text(separator=" ", strip=True) if h1_tags else "") or "" - - canonical_url = "" - link_canonical = soup.find("link", attrs={"rel": "canonical"}) - if link_canonical and link_canonical.get("href"): - canonical_url = normalize_link(base_url, link_canonical["href"]) or "" - - return meta_desc, meta_desc_len, h1_text, h1_count, canonical_url - - -def parse_seo_extended(html_text: str, base_url: str) -> dict: - """ - Extract extended SEO/accessibility/performance-related fields from HTML. - Returns a dict with: viewport_present, viewport_content, noindex, has_schema, - heading_sequence, images_without_alt, images_total, img_without_lazy, img_without_dimensions, - aria_count, mixed_content_count. - """ - soup = BeautifulSoup(html_text, "lxml") - out = { - "viewport_present": False, - "viewport_content": "", - "noindex": False, - "has_schema": False, - "heading_sequence": [], - "heading_text": [], - "images_without_alt": 0, - "images_total": 0, - "img_without_lazy": 0, - "img_without_dimensions": 0, - "aria_count": 0, - "mixed_content_count": 0, - } - # Viewport - viewport = soup.find("meta", attrs={"name": "viewport"}) - if viewport and viewport.get("content"): - out["viewport_present"] = True - out["viewport_content"] = (viewport["content"] or "").strip() - # noindex - robots = soup.find("meta", attrs={"name": "robots"}) - if robots and robots.get("content"): - content = (robots["content"] or "").lower() - out["noindex"] = "noindex" in content - # Structured data: JSON-LD or microdata - if soup.find("script", type="application/ld+json"): - out["has_schema"] = True - if soup.find(attrs={"itemscope": True}): - out["has_schema"] = True - # Heading order (h1..h6 tag names) and visible heading copy (for keywords / fingerprints) - for tag in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]): - if tag.name: - out["heading_sequence"].append(tag.name) - text = (tag.get_text(separator=" ", strip=True) or "").strip() - if text: - out["heading_text"].append(text) - # Images: alt, lazy, dimensions - base_scheme = urlparse(base_url).scheme.lower() - for img in soup.find_all("img"): - out["images_total"] += 1 - if not img.get("alt") and not img.get("aria-label"): - out["images_without_alt"] += 1 - loading = (img.get("loading") or "").strip().lower() - if loading != "lazy": - out["img_without_lazy"] += 1 - if not img.get("width") and not img.get("height"): - out["img_without_dimensions"] += 1 - src = img.get("src") or "" - if base_scheme == "https" and src.strip().lower().startswith("http://"): - out["mixed_content_count"] += 1 - # ARIA: count elements with any aria- attribute - for el in soup.find_all(True): - if getattr(el, "attrs", None) and any(k.startswith("aria-") for k in el.attrs): - out["aria_count"] += 1 - # Mixed content: links and other src/href - for tag in soup.find_all(True): - for attr in ("href", "src", "srcset"): - val = tag.get(attr) - if not val or base_scheme != "https": - continue - val = str(val).strip().lower() - if val.startswith("http://"): - out["mixed_content_count"] += 1 - elif attr == "srcset": - for part in val.split(","): - part = part.strip().split()[0] if part.strip() else "" - if part.startswith("http://"): - out["mixed_content_count"] += 1 - return out - - -_STOP_WORDS = frozenset({ - "the", "and", "for", "that", "this", "with", "from", "your", "have", "are", - "was", "were", "been", "will", "would", "could", "should", "about", "which", - "their", "there", "what", "when", "where", "more", "some", "than", "them", - "other", "into", "over", "also", "just", "after", "before", "only", "then", - "very", "most", "each", "such", "like", "does", "here", "because", "being", - "well", "while", "these", "those", "both", "many", "much", "even", "back", - "through", "still", "between", "every", "under", "last", "long", "great", - "make", "same", "come", "take", "know", "they", "page", "site", "home", - "click", "read", "view", "next", "menu", "main", "skip", "content", "link", - "http", "https", "www", "html", "class", "none", "true", "false", "null", -}) - - -def _count_syllables(word: str) -> int: - word = word.lower().strip() - if len(word) <= 3: - return 1 - vowels = "aeiouy" - count = 0 - prev_vowel = False - for ch in word: - is_vowel = ch in vowels - if is_vowel and not prev_vowel: - count += 1 - prev_vowel = is_vowel - if word.endswith("e") and count > 1: - count -= 1 - return max(1, count) - - -def parse_content_text(soup, raw_html: str, excerpt_max_chars: int = 0) -> dict: - """Extract content analytics: word count, reading level, content-to-HTML ratio, top keywords. - - excerpt_max_chars: when > 0, strip script/style from body and store a whitespace-normalized - plain-text excerpt (truncated) in ``content_excerpt`` for analysis / AI / UI. - """ - import re - from collections import Counter - - body = soup.find("body") - if body: - for tag in body.find_all(["script", "style", "noscript"]): - tag.decompose() - body_text = body.get_text(separator=" ", strip=True) if body else "" - words = [w for w in re.findall(r"[a-zA-Z]+", body_text) if len(w) >= 2] - word_count = len(words) - - sentences = [s.strip() for s in re.split(r"[.!?]+", body_text) if len(s.strip()) > 5] - sentence_count = max(1, len(sentences)) - - total_syllables = sum(_count_syllables(w) for w in words) if words else 0 - - reading_level = 0.0 - if word_count > 30: - reading_level = ( - 0.39 * (word_count / sentence_count) - + 11.8 * (total_syllables / max(1, word_count)) - - 15.59 - ) - reading_level = max(0.0, min(18.0, round(reading_level, 1))) - - html_len = max(1, len(raw_html)) - content_html_ratio = round(len(body_text) / html_len * 100, 1) - - keyword_words = [w.lower() for w in words if len(w) >= 4 and w.lower() not in _STOP_WORDS] - top_keywords = Counter(keyword_words).most_common(10) - max_kw = top_keywords[0][1] if top_keywords else 0 - kw_rows = [] - for w, c in top_keywords: - score = round(100 * c / max_kw) if max_kw else 0 - kw_rows.append({"word": w, "count": c, "score": int(score)}) - - excerpt = "" - if excerpt_max_chars and excerpt_max_chars > 0 and body_text: - excerpt = re.sub(r"\s+", " ", body_text.strip()) - if len(excerpt) > excerpt_max_chars: - excerpt = excerpt[: excerpt_max_chars].rsplit(" ", 1)[0].strip() or excerpt[:excerpt_max_chars] - - return { - "word_count": word_count, - "reading_level": reading_level, - "content_html_ratio": content_html_ratio, - "top_keywords": json.dumps(kw_rows), - "content_excerpt": excerpt, - } - - -def parse_social_meta(soup) -> dict: - """Extract Open Graph and Twitter Card meta tags.""" - def _meta_content(attrs: dict) -> str: - tag = soup.find("meta", attrs=attrs) - return (tag.get("content") or "").strip() if tag else "" - - return { - "og_title": _meta_content({"property": "og:title"}), - "og_description": _meta_content({"property": "og:description"}), - "og_image": _meta_content({"property": "og:image"}), - "og_type": _meta_content({"property": "og:type"}), - "twitter_card": _meta_content({"name": "twitter:card"}), - "twitter_title": _meta_content({"name": "twitter:title"}), - "twitter_image": _meta_content({"name": "twitter:image"}), - } - - -_TECH_PATTERNS = [ - ("WordPress", "html", "/wp-content/"), - ("WordPress", "html", "/wp-includes/"), - ("Drupal", "meta_generator", "Drupal"), - ("Joomla", "meta_generator", "Joomla"), - ("Shopify", "html", "cdn.shopify.com"), - ("Squarespace", "html", "squarespace.com"), - ("Wix", "html", "wix.com"), - ("Next.js", "html", "__NEXT_DATA__"), - ("Next.js", "html", "_next/static"), - ("Nuxt.js", "html", "__NUXT__"), - ("Gatsby", "html", "gatsby-"), - ("React", "html", "data-reactroot"), - ("React", "html", "__REACT_DEVTOOLS"), - ("React", "html", "react.production.min"), - ("Vue.js", "html", "__vue"), - ("Vue.js", "html", "vue.min.js"), - ("Angular", "html", "ng-version"), - ("Angular", "html", "ng-app"), - ("Svelte", "html", "svelte"), - ("jQuery", "html", "jquery"), - ("Bootstrap", "html", "bootstrap"), - ("Tailwind CSS", "html", "tailwindcss"), - ("Google Analytics", "html", "google-analytics.com/analytics.js"), - ("Google Analytics", "html", "googletagmanager.com/gtag"), - ("Google Tag Manager", "html", "googletagmanager.com/gtm.js"), - ("Facebook Pixel", "html", "connect.facebook.net"), - ("Hotjar", "html", "hotjar.com"), - ("Google Fonts", "html", "fonts.googleapis.com"), - ("Font Awesome", "html", "fontawesome"), - ("Cloudflare", "header", "cf-ray"), - ("Nginx", "header_server", "nginx"), - ("Apache", "header_server", "apache"), - ("LiteSpeed", "header_server", "litespeed"), - ("Vercel", "header_server", "vercel"), - ("Netlify", "header_server", "netlify"), - ("Amazon CloudFront", "header", "x-amz-cf-id"), - ("AWS", "header_server", "amazons3"), -] - -# Module-level cache for Wappalyzer instance (avoids reloading technologies file per page). -_wappalyzer_instance = None -_wappalyzer_disabled = False - - -def _is_wappalyzer_regex_warning(msg: str) -> bool: - lower = msg.lower() - return "compiling regex" in lower and "unbalanced parenthesis" in lower - - -def detect_tech_wappalyzer( - url: str, - html: str, - headers: dict, - soup, - wappalyzer=None, -) -> str: - """ - Detect technologies using python-Wappalyzer from existing HTML and headers. - Returns JSON list of tech names. On any failure, falls back to parse_tech_stack(soup, headers, url). - """ - global _wappalyzer_instance, _wappalyzer_disabled - if _wappalyzer_disabled: - return parse_tech_stack(soup, headers, url) - try: - from Wappalyzer import Wappalyzer, WebPage - except ImportError: - return parse_tech_stack(soup, headers, url) - try: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - instance = wappalyzer if wappalyzer is not None else _wappalyzer_instance - if instance is None: - instance = Wappalyzer.latest() - if wappalyzer is None: - _wappalyzer_instance = instance - webpage = WebPage(url, html=html, headers=headers) - detected = instance.analyze(webpage) - if any(_is_wappalyzer_regex_warning(str(w.message)) for w in caught): - _wappalyzer_disabled = True - _wappalyzer_instance = None - return parse_tech_stack(soup, headers, url) - return json.dumps(sorted(detected)) - except Exception: - return parse_tech_stack(soup, headers, url) - - -def parse_tech_stack(soup, headers: dict, url: str) -> str: - """Detect technologies from HTML patterns and HTTP headers. Returns JSON list of tech names.""" - detected = set() - html_str = str(soup).lower() - meta_gen = soup.find("meta", attrs={"name": "generator"}) - generator = (meta_gen.get("content") or "").strip().lower() if meta_gen else "" - server_header = (headers.get("Server") or headers.get("server") or "").lower() - - for name, source, pattern in _TECH_PATTERNS: - pat = pattern.lower() - if source == "html" and pat in html_str: - detected.add(name) - elif source == "meta_generator" and pat in generator: - detected.add(name) - elif source == "header": - for v in headers.values(): - if isinstance(v, str) and pat in v.lower(): - detected.add(name) - break - elif source == "header_server" and pat in server_header: - detected.add(name) - - return json.dumps(sorted(detected)) - - -def parse_resources(html_text: str, base_url: str) -> dict: - """ - Extract script/link resource counts and total sizes (same-origin only, no fetch). - Returns dict: script_count, link_stylesheet_count, script_urls, stylesheet_urls - (URLs for optional later HEAD/GET). Does not fetch; caller may fetch with limit. - """ - soup = BeautifulSoup(html_text, "lxml") - parsed_base = urlparse(base_url) - script_urls = [] - for s in soup.find_all("script", src=True): - url = normalize_link(base_url, s["src"]) - if url and urlparse(url).netloc == parsed_base.netloc: - script_urls.append(url) - stylesheet_urls = [] - for link in soup.find_all("link", rel=lambda r: r and "stylesheet" in (r.lower() if isinstance(r, str) else "")): - url = link.get("href") and normalize_link(base_url, link["href"]) - if url and urlparse(url).netloc == parsed_base.netloc: - stylesheet_urls.append(url) - return { - "script_count": len(script_urls), - "link_stylesheet_count": len(stylesheet_urls), - "script_urls": script_urls, - "stylesheet_urls": stylesheet_urls, - } - - -def _is_empty(raw) -> bool: - if raw is None: - return True - if isinstance(raw, float) and math.isnan(raw): - return True - if raw == "": - return True - return False - - -def parse_links_serialized(raw) -> list[str]: - """ - Parse a serialized list of URLs from CSV/DataFrame (string list repr, comma-separated, or list). - """ - if _is_empty(raw): - return [] - if isinstance(raw, list): - return [str(x).strip().rstrip("/") for x in raw if x] - s = str(raw).strip() - if not s: - return [] - if s.startswith("[") and s.endswith("]"): - try: - v = ast.literal_eval(s) - if isinstance(v, (list, tuple)): - return [str(x).strip().rstrip("/") for x in v if x] - except Exception: - pass - return [t.strip().rstrip("/") for t in s.split(",") if t.strip()] - - -def load_robots(start_url: str): - """Load robots.txt for the given URL; returns RobotFileParser or None on error.""" - parsed = urlparse(start_url) - robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt" - rp = robotparser.RobotFileParser() - rp.set_url(robots_url) - try: - rp.read() - return rp - except Exception: - return None - - -# Column names that may contain serialized outlink lists (for building edges from crawl CSV) -LINK_COLUMN_NAMES = ( - "links", - "edges", - "outlinks", - "outlink_targets", - "targets", - "link_targets", - "links_list", +from .parsing import tech as _tech +from .parsing.content import parse_content_text, parse_social_meta +from .parsing.io import load_dataframe, load_edges, save_dataframe, save_edges +from .parsing.links import ( + LINK_COLUMN_NAMES, + normalize_link, + parse_link_edges, + parse_links, + parse_links_serialized, + strip_crawl_query_params, + _is_empty, ) +from .parsing.robots import load_robots +from .parsing.seo import parse_resources, parse_seo, parse_seo_extended +from .parsing.tech import parse_tech_stack, _is_wappalyzer_regex_warning + +_wappalyzer_instance = _tech._wappalyzer_instance +_wappalyzer_disabled = _tech._wappalyzer_disabled + + +def detect_tech_wappalyzer(url, html, headers, soup, wappalyzer=None): + """Detect technologies; syncs wappalyzer module state with this facade for tests.""" + _tech._wappalyzer_disabled = _wappalyzer_disabled + _tech._wappalyzer_instance = _wappalyzer_instance + result = _tech.detect_tech_wappalyzer(url, html, headers, soup, wappalyzer) + globals()["_wappalyzer_disabled"] = _tech._wappalyzer_disabled + globals()["_wappalyzer_instance"] = _tech._wappalyzer_instance + return result + + +__all__ = [ + "load_dataframe", + "save_dataframe", + "load_edges", + "save_edges", + "strip_crawl_query_params", + "normalize_link", + "parse_link_edges", + "parse_links", + "parse_seo", + "parse_seo_extended", + "parse_content_text", + "parse_social_meta", + "detect_tech_wappalyzer", + "parse_tech_stack", + "parse_resources", + "parse_links_serialized", + "load_robots", + "LINK_COLUMN_NAMES", + "_is_wappalyzer_regex_warning", + "_is_empty", + "_wappalyzer_disabled", + "_wappalyzer_instance", +] diff --git a/src/website_profiling/crawl/config.py b/src/website_profiling/crawl/config.py new file mode 100644 index 00000000..22f09898 --- /dev/null +++ b/src/website_profiling/crawl/config.py @@ -0,0 +1,117 @@ +"""Crawler configuration dataclass.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from .discovery import follow_links_for_mode, normalize_discovery_mode + +DEFAULT_USER_AGENT = "WebsiteProfilingCrawler/1.0" +MOBILE_USER_AGENT = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" +) + + +def resolve_crawl_user_agent( + preset: str | None, custom: str | None, default: str | None = None +) -> str: + p = (preset or "default").strip().lower() + if p == "mobile": + return MOBILE_USER_AGENT + if p == "custom" and custom and str(custom).strip(): + return str(custom).strip() + return (default or DEFAULT_USER_AGENT).strip() or DEFAULT_USER_AGENT + + +@dataclass +class CrawlConfig: + start_url: str + max_pages: Optional[int] = None + concurrency: int = 6 + timeout: int = 12 + ignore_robots: bool = False + allow_external: bool = False + max_depth: Optional[int] = None + user_agent: Optional[str] = None + polite_delay: float = 0.0 + store_outlinks: bool = False + exclude_urls: Optional[list[str]] = None + use_wappalyzer: bool = True + store_content_excerpt: bool = False + content_excerpt_max_chars: int = 4096 + render_mode: str = "static" + js_concurrency: int = 3 + js_timeout: int = 30 + js_wait_until: str = "domcontentloaded" + js_extra_wait_ms: int = 1500 + js_block_resources: bool = True + capture_console: bool = True + js_console_levels: str = "error,warning" + capture_failed_requests: bool = False + console_max_per_page: int = 20 + custom_extraction_regex: str = "" + crawl_ignore_params: Optional[list[str]] = None + discovery_mode: str = "spider" + crawl_url_list: Optional[list[str]] = None + crawl_user_agent_preset: str = "default" + crawl_user_agent_custom: str = "" + crawl_auth_username: str = "" + crawl_auth_password: str = "" + crawl_extra_headers: str = "" + crawl_cookies: str = "" + crawl_robots_txt_override: str = "" + custom_extractors: Optional[list[dict]] = None + enable_axe: bool = False + + @classmethod + def from_kwargs(cls, **kwargs: object) -> CrawlConfig: + """Build config from Crawler keyword arguments (unknown keys ignored).""" + fields = {f.name for f in cls.__dataclass_fields__.values()} + return cls(**{k: v for k, v in kwargs.items() if k in fields}) + + def normalized(self) -> CrawlConfig: + """Apply normalized derived fields in-place.""" + self.start_url = self.start_url.rstrip("/") + self.discovery_mode = normalize_discovery_mode(self.discovery_mode) + self.render_mode = (self.render_mode or "static").strip().lower() + self.js_concurrency = max(1, int(self.js_concurrency)) + self.max_pages = ( + self.max_pages if (self.max_pages is not None and self.max_pages > 0) else float("inf") + ) + self.max_depth = None if self.max_depth is None else int(self.max_depth) + self.polite_delay = max(0.0, float(self.polite_delay)) + self.exclude_urls = list(self.exclude_urls) if self.exclude_urls else [] + self.store_content_excerpt = bool(self.store_content_excerpt) + self.content_excerpt_max_chars = max(0, int(self.content_excerpt_max_chars or 0)) + self.custom_extraction_regex = (self.custom_extraction_regex or "").strip() + self.custom_extractors = list(self.custom_extractors or []) + self.crawl_ignore_params = list(self.crawl_ignore_params or []) + self.crawl_url_list = [ + u.rstrip("/") for u in (self.crawl_url_list or []) if u and str(u).strip() + ] + self.user_agent = resolve_crawl_user_agent( + self.crawl_user_agent_preset, + self.crawl_user_agent_custom, + self.user_agent, + ) + return self + + @property + def effective_concurrency(self) -> int: + if self.render_mode == "javascript": + return self.js_concurrency + return max(1, int(self.concurrency)) + + @property + def follow_links(self) -> bool: + return follow_links_for_mode(self.discovery_mode) + + @property + def fetcher_render_mode(self) -> str: + if self.render_mode == "javascript": + return "javascript" + if self.render_mode == "auto": + return "auto" + return "static" diff --git a/src/website_profiling/crawl/crawler.py b/src/website_profiling/crawl/crawler.py index 39b0ac51..faa62b54 100644 --- a/src/website_profiling/crawl/crawler.py +++ b/src/website_profiling/crawl/crawler.py @@ -1,73 +1,47 @@ """ Website crawler: threaded, respects robots.txt, returns DataFrame and optional CSV. """ +from __future__ import annotations + import json -import threading import time from concurrent.futures import ThreadPoolExecutor -from queue import Queue from typing import Optional -from urllib.parse import urlparse - - -def _url_matches_exclude(url: str, exclude_urls: list[str]) -> bool: - """True if url equals or is under any exclude prefix (trailing-slash normalized).""" - if not exclude_urls: - return False - u = url.rstrip("/") - for prefix in exclude_urls: - p = prefix.strip().rstrip("/") - if not p: - continue - if u == p or u.startswith(p + "/"): - return True - return False import pandas as pd import requests from tqdm.auto import tqdm from ..console_io import console_print -from ..common import ( - detect_tech_wappalyzer, - load_robots, - normalize_link, - parse_content_text, - parse_link_edges, - parse_resources, - parse_seo, - parse_seo_extended, - parse_social_meta, - parse_tech_stack, +from ..common import strip_crawl_query_params +from .config import ( + DEFAULT_USER_AGENT, + MOBILE_USER_AGENT, + CrawlConfig, + resolve_crawl_user_agent, ) -from ..analysis.page import analyze_html -from .discovery import ( - follow_links_for_mode, - normalize_discovery_mode, - seed_sitemap_for_mode, -) -from .extraction import parse_extractors_config, run_extractors +from .db_writer import CrawlDbWriter, _CrawlDbWriter +from .discovery import normalize_discovery_mode from .fetchers import build_fetcher +from .sitemap import discover_sitemap_urls from .fetchers.base import FetchResult -from .fetchers.browser_diagnostics import merge_browser_into_page_analysis from .fetchers.hybrid import HybridFetcher -from .fetchers.spa_heuristics import needs_js_render_after_parse -from .sitemap import discover_sitemap_urls - -DEFAULT_USER_AGENT = "WebsiteProfilingCrawler/1.0" -MOBILE_USER_AGENT = ( - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " - "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" -) +from .frontier import CrawlFrontier, url_matches_exclude +from .page_record import PageRecordBuilder +from .schema import crawl_dataframe_columns, empty_crawl_row +# Re-export for backward compatibility. +_url_matches_exclude = url_matches_exclude -def resolve_crawl_user_agent(preset: str | None, custom: str | None, default: str | None = None) -> str: - p = (preset or "default").strip().lower() - if p == "mobile": - return MOBILE_USER_AGENT - if p == "custom" and custom and str(custom).strip(): - return str(custom).strip() - return (default or DEFAULT_USER_AGENT).strip() or DEFAULT_USER_AGENT +__all__ = [ + "Crawler", + "run_crawler", + "resolve_crawl_user_agent", + "DEFAULT_USER_AGENT", + "MOBILE_USER_AGENT", + "_url_matches_exclude", + "_CrawlDbWriter", +] class Crawler: @@ -110,331 +84,166 @@ def __init__( crawl_robots_txt_override: str = "", custom_extractors: Optional[list[dict]] = None, enable_axe: bool = False, + *, + config: Optional[CrawlConfig] = None, ): - self.start_url = start_url.rstrip("/") - self.start_netloc = urlparse(self.start_url).netloc - self.discovery_mode = normalize_discovery_mode(discovery_mode) - self.follow_links = follow_links_for_mode(self.discovery_mode) - self.crawl_url_list = [u.rstrip("/") for u in (crawl_url_list or []) if u and str(u).strip()] + if config is None: + config = CrawlConfig.from_kwargs( + start_url=start_url, + max_pages=max_pages, + concurrency=concurrency, + timeout=timeout, + ignore_robots=ignore_robots, + allow_external=allow_external, + max_depth=max_depth, + user_agent=user_agent, + polite_delay=polite_delay, + store_outlinks=store_outlinks, + exclude_urls=exclude_urls, + use_wappalyzer=use_wappalyzer, + store_content_excerpt=store_content_excerpt, + content_excerpt_max_chars=content_excerpt_max_chars, + render_mode=render_mode, + js_concurrency=js_concurrency, + js_timeout=js_timeout, + js_wait_until=js_wait_until, + js_extra_wait_ms=js_extra_wait_ms, + js_block_resources=js_block_resources, + capture_console=capture_console, + js_console_levels=js_console_levels, + capture_failed_requests=capture_failed_requests, + console_max_per_page=console_max_per_page, + custom_extraction_regex=custom_extraction_regex, + crawl_ignore_params=crawl_ignore_params, + discovery_mode=discovery_mode, + crawl_url_list=crawl_url_list, + crawl_user_agent_preset=crawl_user_agent_preset, + crawl_user_agent_custom=crawl_user_agent_custom, + crawl_auth_username=crawl_auth_username, + crawl_auth_password=crawl_auth_password, + crawl_extra_headers=crawl_extra_headers, + crawl_cookies=crawl_cookies, + crawl_robots_txt_override=crawl_robots_txt_override, + custom_extractors=custom_extractors, + enable_axe=enable_axe, + ) + config.normalized() + self.config = config + + self.start_url = config.start_url + self.discovery_mode = config.discovery_mode + self.follow_links = config.follow_links + self.crawl_url_list = config.crawl_url_list self.link_edges_accum: list[dict] = [] - self.render_mode = (render_mode or "static").strip().lower() - self.js_concurrency = max(1, int(js_concurrency)) - effective_concurrency = ( - self.js_concurrency - if self.render_mode == "javascript" - else max(1, int(concurrency)) - ) - self.max_pages = ( - max_pages if (max_pages is not None and max_pages > 0) else float("inf") + self.render_mode = config.render_mode + self.max_pages = config.max_pages + self.concurrency = config.effective_concurrency + self.timeout = config.timeout + self.polite_delay = config.polite_delay + self.store_outlinks = config.store_outlinks + self.exclude_urls = config.exclude_urls + self.crawl_ignore_params = config.crawl_ignore_params + self.custom_extraction_regex = config.custom_extraction_regex + self.custom_extractors = config.custom_extractors + + self.page_builder = PageRecordBuilder( + use_wappalyzer=config.use_wappalyzer, + store_content_excerpt=config.store_content_excerpt, + content_excerpt_max_chars=config.content_excerpt_max_chars, + custom_extraction_regex=config.custom_extraction_regex, + custom_extractors=config.custom_extractors, ) - self.concurrency = effective_concurrency - self.timeout = timeout - self.ignore_robots = ignore_robots - self.allow_external = allow_external - self.max_depth = None if max_depth is None else int(max_depth) - self.user_agent = resolve_crawl_user_agent( - crawl_user_agent_preset, crawl_user_agent_custom, user_agent + + self.frontier = CrawlFrontier( + config.start_url, + allow_external=config.allow_external, + max_depth=config.max_depth, + exclude_urls=config.exclude_urls, + follow_links=config.follow_links, + ignore_robots=config.ignore_robots, + user_agent=config.user_agent or DEFAULT_USER_AGENT, + crawl_robots_txt_override=config.crawl_robots_txt_override, ) - self.polite_delay = max(0.0, float(polite_delay)) - self.store_outlinks = store_outlinks - self.exclude_urls = list(exclude_urls) if exclude_urls else [] - self.use_wappalyzer = use_wappalyzer - self.store_content_excerpt = bool(store_content_excerpt) - self.content_excerpt_max_chars = max(0, int(content_excerpt_max_chars or 0)) - self._wappalyzer_instance = None - self.custom_extraction_regex = (custom_extraction_regex or "").strip() - self.custom_extractors = list(custom_extractors or []) - self.crawl_ignore_params = list(crawl_ignore_params or []) + self.queue = self.frontier.queue + self.depths = self.frontier.depths + self.visited = self.frontier.visited + self.lock = self.frontier.lock - self.queue = Queue() - self.depths: dict[str, int] = {} - self.visited = set() - self.results = [] - self.lock = threading.Lock() + self.results: list[dict] = [] self.session = requests.Session() - self.session.headers.update({"User-Agent": self.user_agent}) - if crawl_auth_username: - self.session.auth = (crawl_auth_username, crawl_auth_password or "") - for line in (crawl_extra_headers or "").replace("\r", "").split("\n"): + self.session.headers.update({"User-Agent": config.user_agent}) + if config.crawl_auth_username: + self.session.auth = (config.crawl_auth_username, config.crawl_auth_password or "") + for line in (config.crawl_extra_headers or "").replace("\r", "").split("\n"): if ":" in line: key, val = line.split(":", 1) k, v = key.strip(), val.strip() if k: self.session.headers[k] = v - if crawl_cookies and str(crawl_cookies).strip(): - self.session.headers["Cookie"] = str(crawl_cookies).strip() - self.rp = None - if not self.ignore_robots: - override = (crawl_robots_txt_override or "").strip() - if override: - import io - import urllib.robotparser as robotparser + if config.crawl_cookies and str(config.crawl_cookies).strip(): + self.session.headers["Cookie"] = str(config.crawl_cookies).strip() - self.rp = robotparser.RobotFileParser() - self.rp.parse(override.splitlines()) - else: - self.rp = load_robots(self.start_url) self.fetcher = build_fetcher( - render_mode="javascript" if self.render_mode == "javascript" else ("auto" if self.render_mode == "auto" else "static"), - timeout=timeout, - user_agent=self.user_agent, + render_mode=config.fetcher_render_mode, + timeout=config.timeout, + user_agent=config.user_agent, session=self.session, - js_concurrency=self.js_concurrency, - js_timeout=js_timeout, - js_wait_until=js_wait_until, - js_extra_wait_ms=js_extra_wait_ms, - js_block_resources=js_block_resources, - capture_console=capture_console, - js_console_levels=js_console_levels, - capture_failed_requests=capture_failed_requests, - console_max_per_page=console_max_per_page, - run_axe=enable_axe, + js_concurrency=config.js_concurrency, + js_timeout=config.js_timeout, + js_wait_until=config.js_wait_until, + js_extra_wait_ms=config.js_extra_wait_ms, + js_block_resources=config.js_block_resources, + capture_console=config.capture_console, + js_console_levels=config.js_console_levels, + capture_failed_requests=config.capture_failed_requests, + console_max_per_page=config.console_max_per_page, + run_axe=config.enable_axe, ) self._hybrid_fetcher = ( self.fetcher if isinstance(self.fetcher, HybridFetcher) else None ) - self._seed_initial_urls(timeout) + self.frontier.seed_initial_urls( + discovery_mode=config.discovery_mode, + crawl_url_list=config.crawl_url_list, + timeout=config.timeout, + session=self.session, + ) - def _enqueue_seed(self, url: str, depth: int = 0) -> None: - u = url.rstrip("/") - if _url_matches_exclude(u, self.exclude_urls): - return - if not self.allow_external and not self.same_domain(u): - return - if u in self.depths: - return - self.queue.put(u) - self.depths[u] = depth + @property + def rp(self): + return self.frontier.rp - def _seed_initial_urls(self, timeout: int) -> None: - mode = self.discovery_mode - if mode in ("list", "hybrid"): - for url in self.crawl_url_list: - self._enqueue_seed(url, 0) - if mode in ("spider", "hybrid"): - self._enqueue_seed(self.start_url, 0) - if seed_sitemap_for_mode(mode): - self._seed_sitemap_urls(timeout) + @rp.setter + def rp(self, value) -> None: + self.frontier.rp = value - def same_domain(self, url): - return urlparse(url).netloc == self.start_netloc + def same_domain(self, url: str) -> bool: + return self.frontier.same_domain(url) - def allowed_by_robots(self, url): - if self.ignore_robots or not self.rp: - return True - try: - return self.rp.can_fetch(self.user_agent, url) - except Exception: - return True + def allowed_by_robots(self, url: str) -> bool: + return self.frontier.allowed_by_robots(url) - def _seed_sitemap_urls(self, timeout: int) -> None: - try: - seeds = discover_sitemap_urls( - self.start_url, - timeout=timeout, - session=self.session, - ) - except Exception: - return - for url in seeds: - self._enqueue_seed(url, 0) + def _queue_contains(self, item: str) -> bool: + return self.frontier.queue_contains(item) - def fetch(self, url) -> FetchResult: + def fetch(self, url: str) -> FetchResult: return self.fetcher.fetch(url) - def _empty_seo(self, url: str, headers_dict: Optional[dict] = None, redirect_chain_length: int = 0) -> dict: - """Default SEO/performance fields when no HTML or error.""" - h = headers_dict or {} - return { - "response_time_ms": "", - "content_length": 0, - "final_url": url, - "meta_description": "", - "meta_description_len": 0, - "h1": "", - "h1_count": 0, - "canonical_url": "", - "viewport_present": False, - "viewport_content": "", - "noindex": False, - "has_schema": False, - "heading_sequence": "", - "heading_text": "", - "images_without_alt": 0, - "images_total": 0, - "img_without_lazy": 0, - "img_without_dimensions": 0, - "aria_count": 0, - "mixed_content_count": 0, - "redirect_chain_length": redirect_chain_length, - "cache_control": h.get("Cache-Control", ""), - "etag": h.get("ETag", ""), - "x_robots_tag": h.get("X-Robots-Tag", ""), - "strict_transport_security": h.get("Strict-Transport-Security", ""), - "x_content_type_options": h.get("X-Content-Type-Options", ""), - "x_frame_options": h.get("X-Frame-Options", ""), - "content_security_policy": h.get("Content-Security-Policy", ""), - "script_count": 0, - "link_stylesheet_count": 0, - "total_js_bytes": 0, - "total_css_bytes": 0, - "word_count": 0, - "reading_level": 0.0, - "content_html_ratio": 0.0, - "top_keywords": "[]", - "content_excerpt": "", - "og_title": "", - "og_description": "", - "og_image": "", - "og_type": "", - "twitter_card": "", - "twitter_title": "", - "twitter_image": "", - "tech_stack": "[]", - "depth": None, - "page_analysis": "{}", - } - - def _parse_page_content( - self, - url: str, - text: str, - final_url: str, - headers_dict: dict, - redirect_chain_length: int, - ) -> dict: - """Extract title, links, and SEO/content fields from HTML.""" - ext = self._empty_seo(url, headers_dict, redirect_chain_length) - title, link_edge_rows = parse_link_edges(url, text) - links = {e["to_url"] for e in link_edge_rows} - meta_description, meta_description_len, h1_text, h1_count, canonical_url = ( - parse_seo(url, text) - ) - seo_ext = parse_seo_extended(text, final_url or url) - ext["viewport_present"] = seo_ext.get("viewport_present", False) - ext["viewport_content"] = seo_ext.get("viewport_content", "") - ext["noindex"] = seo_ext.get("noindex", False) - if (headers_dict.get("X-Robots-Tag") or "").lower().find("noindex") >= 0: - ext["noindex"] = True - ext["has_schema"] = seo_ext.get("has_schema", False) - ext["heading_sequence"] = ",".join(seo_ext.get("heading_sequence") or []) - ext["heading_text"] = " | ".join(seo_ext.get("heading_text") or []) - ext["images_without_alt"] = seo_ext.get("images_without_alt", 0) - ext["images_total"] = seo_ext.get("images_total", 0) - ext["img_without_lazy"] = seo_ext.get("img_without_lazy", 0) - ext["img_without_dimensions"] = seo_ext.get("img_without_dimensions", 0) - ext["aria_count"] = seo_ext.get("aria_count", 0) - ext["mixed_content_count"] = seo_ext.get("mixed_content_count", 0) - res_res = parse_resources(text, final_url or url) - ext["script_count"] = res_res.get("script_count", 0) - ext["link_stylesheet_count"] = res_res.get("link_stylesheet_count", 0) - from bs4 import BeautifulSoup as _BS - - _soup = _BS(text, "lxml") - excerpt_max = self.content_excerpt_max_chars if self.store_content_excerpt else 0 - ct_data = parse_content_text(_soup, text, excerpt_max_chars=excerpt_max) - ext["word_count"] = ct_data.get("word_count", 0) - ext["reading_level"] = ct_data.get("reading_level", 0.0) - ext["content_html_ratio"] = ct_data.get("content_html_ratio", 0.0) - ext["top_keywords"] = ct_data.get("top_keywords", "[]") - ext["content_excerpt"] = ct_data.get("content_excerpt") or "" - social = parse_social_meta(_soup) - ext["og_title"] = social.get("og_title", "") - ext["og_description"] = social.get("og_description", "") - ext["og_image"] = social.get("og_image", "") - ext["og_type"] = social.get("og_type", "") - ext["twitter_card"] = social.get("twitter_card", "") - ext["twitter_title"] = social.get("twitter_title", "") - ext["twitter_image"] = social.get("twitter_image", "") - if self.use_wappalyzer: - ext["tech_stack"] = detect_tech_wappalyzer( - final_url or url, text, headers_dict, _soup, self._wappalyzer_instance - ) - else: - ext["tech_stack"] = parse_tech_stack(_soup, headers_dict, final_url or url) - ext["page_analysis"] = json.dumps( - analyze_html(text, final_url or url, final_url or url, canonical_url) - ) - return { - "title": title, - "links": links, - "link_edges": link_edge_rows, - "meta_description": meta_description, - "meta_description_len": meta_description_len, - "h1_text": h1_text, - "h1_count": h1_count, - "canonical_url": canonical_url, - "ext": ext, - } - - def _maybe_refetch_after_parse( - self, - url: str, - result: FetchResult, - *, - link_count: int, - same_domain_link_count: int, - ) -> FetchResult: - """Post-parse auto-mode fallback when static HTML has too few links.""" - if self.render_mode != "auto" or self._hybrid_fetcher is None: - return result - if result.fetch_method != "static": - return result - if not needs_js_render_after_parse( - result, - link_count=link_count, - same_domain_link_count=same_domain_link_count, - ): - return result - rendered = self._hybrid_fetcher.refetch_rendered(url) - if rendered.status == 200 and rendered.text: - return rendered - return result - - @staticmethod - def _sync_from_fetch_result( - result: FetchResult, - url: str, - *, - text: Optional[str], - fetch_method: str, - final_url: str, - content_length: int, - response_time_ms: Optional[int], - headers_dict: dict, - redirect_chain_length: int, - status: Optional[int], - ct: Optional[str], - ) -> dict: - """Copy all FetchResult fields after a post-parse browser refetch.""" - return { - "text": result.text, - "fetch_method": result.fetch_method, - "final_url": result.final_url or url, - "content_length": result.content_length or content_length, - "response_time_ms": result.response_time_ms, - "headers_dict": result.headers_dict or headers_dict, - "redirect_chain_length": result.redirect_chain_length, - "status": result.status, - "ct": result.content_type, - } - - def worker(self, url): + def worker(self, url: str) -> dict: if not self.allowed_by_robots(url): - out = { - "url": url, - "status": "blocked_by_robots", - "content_type": "", - "title": "", - "outlinks": 0, - "fetch_method": "static", - **self._empty_seo(url), - } - if self.store_outlinks: - out["outlink_targets"] = "[]" - return out + return PageRecordBuilder.build_robots_blocked_row( + url, store_outlinks=self.store_outlinks + ) result = self.fetch(url) + if result.status is None: + return PageRecordBuilder.build_fetch_error_row( + url, + result, + fetch_method=result.fetch_method, + store_outlinks=self.store_outlinks, + ) + status = result.status ct = result.content_type text = result.text @@ -445,59 +254,33 @@ def worker(self, url): redirect_chain_length = result.redirect_chain_length fetch_method = result.fetch_method - if status is None: - out = { - "url": url, - "status": "error", - "content_type": "", - "title": "", - "outlinks": 0, - "fetch_method": fetch_method, - **self._empty_seo(url, headers_dict, redirect_chain_length), - } - if self.store_outlinks: - out["outlink_targets"] = "[]" - if result.browser_diagnostics: - out["page_analysis"] = merge_browser_into_page_analysis( - None, result.browser_diagnostics - ) - return out - title = "" outlinks_count = 0 - outlink_list = [] + outlink_list: list[str] = [] meta_description = "" meta_description_len = 0 h1_text = "" h1_count = 0 canonical_url = "" - ext = self._empty_seo(url, headers_dict, redirect_chain_length) + ext = self.page_builder.empty_ext(url, headers_dict, redirect_chain_length) if text: - parsed = self._parse_page_content( + parsed = self.page_builder.parse_page_content( url, text, final_url or url, headers_dict, redirect_chain_length ) links = parsed["links"] same_domain_link_count = sum(1 for link in links if self.same_domain(link)) - result = self._maybe_refetch_after_parse( + result = PageRecordBuilder.maybe_refetch_after_parse( url, result, + render_mode=self.render_mode, + hybrid_fetcher=self._hybrid_fetcher, link_count=len(links), same_domain_link_count=same_domain_link_count, ) if result.text and result.text != text: - synced = self._sync_from_fetch_result( - result, - url, - text=text, - fetch_method=fetch_method, - final_url=final_url, - content_length=content_length, - response_time_ms=response_time_ms, - headers_dict=headers_dict, - redirect_chain_length=redirect_chain_length, - status=status, - ct=ct, + synced = PageRecordBuilder.sync_from_fetch_result( + result, url, content_length=content_length, headers_dict=headers_dict ) text = synced["text"] fetch_method = synced["fetch_method"] @@ -508,7 +291,7 @@ def worker(self, url): redirect_chain_length = synced["redirect_chain_length"] status = synced["status"] ct = synced["ct"] - parsed = self._parse_page_content( + parsed = self.page_builder.parse_page_content( url, text, final_url, headers_dict, redirect_chain_length ) links = parsed["links"] @@ -523,8 +306,6 @@ def worker(self, url): ext = parsed["ext"] if self.crawl_ignore_params: - from ..common import strip_crawl_query_params - links = [strip_crawl_query_params(l, self.crawl_ignore_params) for l in links] link_edge_rows = parsed.get("link_edges") or [] @@ -533,23 +314,7 @@ def worker(self, url): if self.store_outlinks: outlink_list.append(link) self.link_edges_accum.append({"from_url": url, **edge}) - if not self.follow_links: - continue - if _url_matches_exclude(link, self.exclude_urls): - continue - if not self.allow_external and not self.same_domain(link): - continue - cur_depth = self.depths.get(url, 0) - if self.max_depth is not None and cur_depth >= self.max_depth: - continue - with self.lock: - if ( - link not in self.visited - and link not in self.depths - and not self._queue_contains(link) - ): - self.queue.put(link) - self.depths[link] = cur_depth + 1 + self.frontier.try_enqueue_link(link, url) ext["response_time_ms"] = response_time_ms if response_time_ms is not None else "" ext["content_length"] = content_length or 0 @@ -566,31 +331,14 @@ def worker(self, url): ext["x_content_type_options"] = headers_dict.get("X-Content-Type-Options", "") ext["x_frame_options"] = headers_dict.get("X-Frame-Options", "") ext["content_security_policy"] = headers_dict.get("Content-Security-Policy", "") - ext["depth"] = self.depths.get(url) - if self.custom_extraction_regex and text: - import re - - try: - match = re.search(self.custom_extraction_regex, text) - if match: - ext["custom_extract"] = match.group(1) if match.lastindex else match.group(0) - except re.error: - pass - - if self.custom_extractors and text: - fields = run_extractors(text, self.custom_extractors) - if fields: - ext["custom_fields"] = json.dumps(fields) + self.page_builder.apply_custom_extractions(ext, text) if self.polite_delay: time.sleep(self.polite_delay) - if result.browser_diagnostics: - ext["page_analysis"] = merge_browser_into_page_analysis( - ext.get("page_analysis"), result.browser_diagnostics - ) + PageRecordBuilder.merge_browser_diagnostics(ext, result) res = { "url": url, @@ -605,18 +353,12 @@ def worker(self, url): res["outlink_targets"] = json.dumps(list(outlink_list)) return res - def _queue_contains(self, item): - try: - return item in list(self.queue.queue) - except Exception: - return False - def crawl( self, show_progress: bool = True, stream_crawl_run_id: Optional[int] = None, stream_batch_size: int = 500, - ): + ) -> pd.DataFrame: start_time = time.time() from ..progress import CrawlProgressTracker, emit_phase_start @@ -628,7 +370,7 @@ def crawl( ) emit_phase_start("crawl", message="Crawling pages") futures = [] - db_writer: Optional[_CrawlDbWriter] = None + db_writer: Optional[CrawlDbWriter] = None pages_crawled = 0 if stream_crawl_run_id is not None: db_writer = _CrawlDbWriter(stream_crawl_run_id, stream_batch_size) @@ -650,12 +392,10 @@ def crawl( and len(self.results) + len(futures) < self.max_pages ): url = self.queue.get() - if _url_matches_exclude(url, self.exclude_urls): + if self.frontier.should_skip_dequeued(url): + continue + if not self.frontier.mark_visited(url): continue - with self.lock: - if url in self.visited: - continue - self.visited.add(url) futures.append(ex.submit(self.worker, url)) remaining = [] @@ -664,60 +404,7 @@ def crawl( try: res = f.result() except Exception: - res = { - "url": None, - "status": "error", - "content_type": "", - "title": "", - "outlinks": 0, - "response_time_ms": "", - "content_length": 0, - "final_url": "", - "meta_description": "", - "meta_description_len": 0, - "h1": "", - "h1_count": 0, - "canonical_url": "", - "viewport_present": False, - "viewport_content": "", - "noindex": False, - "has_schema": False, - "heading_sequence": "", - "heading_text": "", - "images_without_alt": 0, - "images_total": 0, - "img_without_lazy": 0, - "img_without_dimensions": 0, - "aria_count": 0, - "mixed_content_count": 0, - "redirect_chain_length": 0, - "cache_control": "", - "etag": "", - "x_robots_tag": "", - "strict_transport_security": "", - "x_content_type_options": "", - "x_frame_options": "", - "content_security_policy": "", - "script_count": 0, - "link_stylesheet_count": 0, - "total_js_bytes": 0, - "total_css_bytes": 0, - "word_count": 0, - "reading_level": 0.0, - "content_html_ratio": 0.0, - "top_keywords": "[]", - "content_excerpt": "", - "og_title": "", - "og_description": "", - "og_image": "", - "og_type": "", - "twitter_card": "", - "twitter_title": "", - "twitter_image": "", - "tech_stack": "[]", - "depth": None, - "page_analysis": "{}", - } + res = empty_crawl_row(status="error") if self.store_outlinks: res["outlink_targets"] = "[]" self.results.append(res) @@ -729,10 +416,7 @@ def crawl( db_writer.enqueue(res) if use_tqdm: pbar.update(1) - progress_tracker.maybe_emit( - pages_crawled, - page_url, - ) + progress_tracker.maybe_emit(pages_crawled, page_url) else: remaining.append(f) futures = remaining @@ -758,114 +442,11 @@ def crawl( elapsed = time.time() - start_time df = pd.DataFrame(self.results) if df.empty: - cols = [ - "url", - "status", - "content_type", - "title", - "outlinks", - "response_time_ms", - "content_length", - "final_url", - "meta_description", - "meta_description_len", - "h1", - "h1_count", - "canonical_url", - "viewport_present", - "viewport_content", - "noindex", - "has_schema", - "heading_sequence", - "heading_text", - "images_without_alt", - "images_total", - "img_without_lazy", - "img_without_dimensions", - "aria_count", - "mixed_content_count", - "redirect_chain_length", - "cache_control", - "etag", - "x_robots_tag", - "strict_transport_security", - "x_content_type_options", - "x_frame_options", - "content_security_policy", - "script_count", - "link_stylesheet_count", - "total_js_bytes", - "total_css_bytes", - "word_count", - "reading_level", - "content_html_ratio", - "top_keywords", - "content_excerpt", - "og_title", - "og_description", - "og_image", - "og_type", - "twitter_card", - "twitter_title", - "twitter_image", - "tech_stack", - "depth", - "page_analysis", - "fetch_method", - ] - if self.store_outlinks: - cols.append("outlink_targets") - df = pd.DataFrame(columns=cols) + df = pd.DataFrame(columns=crawl_dataframe_columns(store_outlinks=self.store_outlinks)) df["crawl_time_s"] = elapsed return df -class _CrawlDbWriter(threading.Thread): - """Background thread: batch-insert crawl rows via PostgreSQL connection pool.""" - - def __init__(self, crawl_run_id: int, batch_size: int = 500) -> None: - super().__init__(daemon=True) - self.crawl_run_id = crawl_run_id - self.batch_size = max(50, batch_size) - self._queue: Queue = Queue() - self._error: Optional[BaseException] = None - - def enqueue(self, record: dict) -> None: - self._queue.put(record) - - def finish(self) -> None: - self._queue.put(None) - - def run(self) -> None: - from ..db import db_session - from ..db.crawl_store import _crawl_rows_from_df, write_crawl_batch - - buffer: list[dict] = [] - try: - while True: - item = self._queue.get() - if item is None: - if buffer: - chunk = pd.DataFrame(buffer) - with db_session() as conn: - rows = _crawl_rows_from_df(chunk, self.crawl_run_id) - write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) - break - buffer.append(item) - if len(buffer) >= self.batch_size: - chunk = pd.DataFrame(buffer) - buffer = [] - with db_session() as conn: - rows = _crawl_rows_from_df(chunk, self.crawl_run_id) - write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) - except BaseException as e: - self._error = e - - def raise_if_failed(self) -> None: - if self._error is not None: - raise self._error - - def run_crawler( start_url: str, max_pages: Optional[int] = None, @@ -998,7 +579,6 @@ def run_crawler( with db_session() as conn: write_link_edges(conn, crawler.link_edges_accum, crawl_run_id=run_id) if output_db and not df.empty and stream_run_id is None: - import sys console_print(" Writing crawl results to DB...", flush=True) from ..db import backup_db_if_exists, create_crawl_run, db_session, read_historical_data, restore_historical_data, write_crawl from ..db.storage import ensure_crawl_tables_cleared diff --git a/src/website_profiling/crawl/db_writer.py b/src/website_profiling/crawl/db_writer.py new file mode 100644 index 00000000..ac78af99 --- /dev/null +++ b/src/website_profiling/crawl/db_writer.py @@ -0,0 +1,59 @@ +"""Background thread: batch-insert crawl rows via PostgreSQL connection pool.""" + +from __future__ import annotations + +import threading +from queue import Queue +from typing import Optional + +import pandas as pd + + +class CrawlDbWriter(threading.Thread): + """Background thread: batch-insert crawl rows via PostgreSQL connection pool.""" + + def __init__(self, crawl_run_id: int, batch_size: int = 500) -> None: + super().__init__(daemon=True) + self.crawl_run_id = crawl_run_id + self.batch_size = max(50, batch_size) + self._queue: Queue = Queue() + self._error: Optional[BaseException] = None + + def enqueue(self, record: dict) -> None: + self._queue.put(record) + + def finish(self) -> None: + self._queue.put(None) + + def run(self) -> None: + from ..db import db_session + from ..db.crawl_store import _crawl_rows_from_df, write_crawl_batch + + buffer: list[dict] = [] + try: + while True: + item = self._queue.get() + if item is None: + if buffer: + chunk = pd.DataFrame(buffer) + with db_session() as conn: + rows = _crawl_rows_from_df(chunk, self.crawl_run_id) + write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) + break + buffer.append(item) + if len(buffer) >= self.batch_size: + chunk = pd.DataFrame(buffer) + buffer = [] + with db_session() as conn: + rows = _crawl_rows_from_df(chunk, self.crawl_run_id) + write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) + except BaseException as e: + self._error = e + + def raise_if_failed(self) -> None: + if self._error is not None: + raise self._error + + +# Backward-compatible alias for tests and internal imports. +_CrawlDbWriter = CrawlDbWriter diff --git a/src/website_profiling/crawl/frontier.py b/src/website_profiling/crawl/frontier.py new file mode 100644 index 00000000..cb1e65ac --- /dev/null +++ b/src/website_profiling/crawl/frontier.py @@ -0,0 +1,156 @@ +"""Crawl frontier: URL queue, depth tracking, and link discovery.""" + +from __future__ import annotations + +import threading +from queue import Queue +from typing import Optional +from urllib.parse import urlparse + +import requests + +from ..common import load_robots +from . import sitemap +from .discovery import seed_sitemap_for_mode + + +def url_matches_exclude(url: str, exclude_urls: list[str]) -> bool: + """True if url equals or is under any exclude prefix (trailing-slash normalized).""" + if not exclude_urls: + return False + u = url.rstrip("/") + for prefix in exclude_urls: + p = prefix.strip().rstrip("/") + if not p: + continue + if u == p or u.startswith(p + "/"): + return True + return False + + +class CrawlFrontier: + """Manages crawl queue, visited set, depth limits, and robots rules.""" + + def __init__( + self, + start_url: str, + *, + allow_external: bool = False, + max_depth: Optional[int] = None, + exclude_urls: Optional[list[str]] = None, + follow_links: bool = True, + ignore_robots: bool = False, + user_agent: str = "", + crawl_robots_txt_override: str = "", + ) -> None: + self.start_url = start_url.rstrip("/") + self.start_netloc = urlparse(self.start_url).netloc + self.allow_external = allow_external + self.max_depth = max_depth + self.exclude_urls = list(exclude_urls) if exclude_urls else [] + self.follow_links = follow_links + self.user_agent = user_agent + self.queue: Queue = Queue() + self.depths: dict[str, int] = {} + self.visited: set[str] = set() + self.lock = threading.Lock() + self.rp = None + if not ignore_robots: + override = (crawl_robots_txt_override or "").strip() + if override: + import urllib.robotparser as robotparser + + self.rp = robotparser.RobotFileParser() + self.rp.parse(override.splitlines()) + else: + self.rp = load_robots(self.start_url) + + def same_domain(self, url: str) -> bool: + return urlparse(url).netloc == self.start_netloc + + def allowed_by_robots(self, url: str) -> bool: + if not self.rp: + return True + try: + return self.rp.can_fetch(self.user_agent, url) + except Exception: + return True + + def queue_contains(self, item: str) -> bool: + try: + return item in list(self.queue.queue) + except Exception: + return False + + def enqueue_seed(self, url: str, depth: int = 0) -> None: + u = url.rstrip("/") + if url_matches_exclude(u, self.exclude_urls): + return + if not self.allow_external and not self.same_domain(u): + return + if u in self.depths: + return + self.queue.put(u) + self.depths[u] = depth + + def seed_initial_urls( + self, + *, + discovery_mode: str, + crawl_url_list: list[str], + timeout: int, + session: requests.Session, + ) -> None: + mode = discovery_mode + if mode in ("list", "hybrid"): + for url in crawl_url_list: + self.enqueue_seed(url, 0) + if mode in ("spider", "hybrid"): + self.enqueue_seed(self.start_url, 0) + if seed_sitemap_for_mode(mode): + self.seed_sitemap_urls(timeout, session) + + def seed_sitemap_urls(self, timeout: int, session: requests.Session) -> None: + try: + seeds = sitemap.discover_sitemap_urls( + self.start_url, + timeout=timeout, + session=session, + ) + except Exception: + return + for url in seeds: + self.enqueue_seed(url, 0) + + def try_enqueue_link(self, link: str, from_url: str) -> bool: + """Enqueue a discovered link if frontier rules allow. Returns True if enqueued.""" + if not self.follow_links: + return False + if url_matches_exclude(link, self.exclude_urls): + return False + if not self.allow_external and not self.same_domain(link): + return False + cur_depth = self.depths.get(from_url, 0) + if self.max_depth is not None and cur_depth >= self.max_depth: + return False + with self.lock: + if ( + link not in self.visited + and link not in self.depths + and not self.queue_contains(link) + ): + self.queue.put(link) + self.depths[link] = cur_depth + 1 + return True + return False + + def mark_visited(self, url: str) -> bool: + """Mark URL visited; return False if already visited.""" + with self.lock: + if url in self.visited: + return False + self.visited.add(url) + return True + + def should_skip_dequeued(self, url: str) -> bool: + return url_matches_exclude(url, self.exclude_urls) diff --git a/src/website_profiling/crawl/page_record.py b/src/website_profiling/crawl/page_record.py new file mode 100644 index 00000000..b4c89791 --- /dev/null +++ b/src/website_profiling/crawl/page_record.py @@ -0,0 +1,226 @@ +"""Build crawl page records from fetched HTML.""" + +from __future__ import annotations + +import json +import re +from typing import Any, Optional + +from ..analysis.page import analyze_html +from ..common import ( + detect_tech_wappalyzer, + parse_content_text, + parse_link_edges, + parse_resources, + parse_seo, + parse_seo_extended, + parse_social_meta, + parse_tech_stack, +) +from .extraction import run_extractors +from .fetchers.base import FetchResult +from .fetchers.browser_diagnostics import merge_browser_into_page_analysis +from .fetchers.hybrid import HybridFetcher +from .fetchers.spa_heuristics import needs_js_render_after_parse +from .schema import empty_crawl_row, empty_crawl_row_ext + + +class PageRecordBuilder: + """Extract SEO/content fields and assemble crawl result rows.""" + + def __init__( + self, + *, + use_wappalyzer: bool = True, + store_content_excerpt: bool = False, + content_excerpt_max_chars: int = 4096, + custom_extraction_regex: str = "", + custom_extractors: Optional[list[dict]] = None, + ) -> None: + self.use_wappalyzer = use_wappalyzer + self.store_content_excerpt = store_content_excerpt + self.content_excerpt_max_chars = content_excerpt_max_chars + self.custom_extraction_regex = custom_extraction_regex + self.custom_extractors = list(custom_extractors or []) + self._wappalyzer_instance = None + + def empty_ext( + self, + url: str, + headers_dict: Optional[dict] = None, + redirect_chain_length: int = 0, + ) -> dict[str, Any]: + return empty_crawl_row_ext(url, headers_dict, redirect_chain_length) + + def parse_page_content( + self, + url: str, + text: str, + final_url: str, + headers_dict: dict, + redirect_chain_length: int, + ) -> dict[str, Any]: + """Extract title, links, and SEO/content fields from HTML.""" + ext = self.empty_ext(url, headers_dict, redirect_chain_length) + title, link_edge_rows = parse_link_edges(url, text) + links = {e["to_url"] for e in link_edge_rows} + meta_description, meta_description_len, h1_text, h1_count, canonical_url = parse_seo( + url, text + ) + seo_ext = parse_seo_extended(text, final_url or url) + ext["viewport_present"] = seo_ext.get("viewport_present", False) + ext["viewport_content"] = seo_ext.get("viewport_content", "") + ext["noindex"] = seo_ext.get("noindex", False) + if (headers_dict.get("X-Robots-Tag") or "").lower().find("noindex") >= 0: + ext["noindex"] = True + ext["has_schema"] = seo_ext.get("has_schema", False) + ext["heading_sequence"] = ",".join(seo_ext.get("heading_sequence") or []) + ext["heading_text"] = " | ".join(seo_ext.get("heading_text") or []) + ext["images_without_alt"] = seo_ext.get("images_without_alt", 0) + ext["images_total"] = seo_ext.get("images_total", 0) + ext["img_without_lazy"] = seo_ext.get("img_without_lazy", 0) + ext["img_without_dimensions"] = seo_ext.get("img_without_dimensions", 0) + ext["aria_count"] = seo_ext.get("aria_count", 0) + ext["mixed_content_count"] = seo_ext.get("mixed_content_count", 0) + res_res = parse_resources(text, final_url or url) + ext["script_count"] = res_res.get("script_count", 0) + ext["link_stylesheet_count"] = res_res.get("link_stylesheet_count", 0) + from bs4 import BeautifulSoup as _BS + + _soup = _BS(text, "lxml") + excerpt_max = self.content_excerpt_max_chars if self.store_content_excerpt else 0 + ct_data = parse_content_text(_soup, text, excerpt_max_chars=excerpt_max) + ext["word_count"] = ct_data.get("word_count", 0) + ext["reading_level"] = ct_data.get("reading_level", 0.0) + ext["content_html_ratio"] = ct_data.get("content_html_ratio", 0.0) + ext["top_keywords"] = ct_data.get("top_keywords", "[]") + ext["content_excerpt"] = ct_data.get("content_excerpt") or "" + social = parse_social_meta(_soup) + ext["og_title"] = social.get("og_title", "") + ext["og_description"] = social.get("og_description", "") + ext["og_image"] = social.get("og_image", "") + ext["og_type"] = social.get("og_type", "") + ext["twitter_card"] = social.get("twitter_card", "") + ext["twitter_title"] = social.get("twitter_title", "") + ext["twitter_image"] = social.get("twitter_image", "") + if self.use_wappalyzer: + ext["tech_stack"] = detect_tech_wappalyzer( + final_url or url, text, headers_dict, _soup, self._wappalyzer_instance + ) + else: + ext["tech_stack"] = parse_tech_stack(_soup, headers_dict, final_url or url) + ext["page_analysis"] = json.dumps( + analyze_html(text, final_url or url, final_url or url, canonical_url) + ) + return { + "title": title, + "links": links, + "link_edges": link_edge_rows, + "meta_description": meta_description, + "meta_description_len": meta_description_len, + "h1_text": h1_text, + "h1_count": h1_count, + "canonical_url": canonical_url, + "ext": ext, + } + + def apply_custom_extractions(self, ext: dict[str, Any], text: Optional[str]) -> None: + if self.custom_extraction_regex and text: + try: + match = re.search(self.custom_extraction_regex, text) + if match: + ext["custom_extract"] = match.group(1) if match.lastindex else match.group(0) + except re.error: + pass + if self.custom_extractors and text: + fields = run_extractors(text, self.custom_extractors) + if fields: + ext["custom_fields"] = json.dumps(fields) + + @staticmethod + def maybe_refetch_after_parse( + url: str, + result: FetchResult, + *, + render_mode: str, + hybrid_fetcher: Optional[HybridFetcher], + link_count: int, + same_domain_link_count: int, + ) -> FetchResult: + """Post-parse auto-mode fallback when static HTML has too few links.""" + if render_mode != "auto" or hybrid_fetcher is None: + return result + if result.fetch_method != "static": + return result + if not needs_js_render_after_parse( + result, + link_count=link_count, + same_domain_link_count=same_domain_link_count, + ): + return result + rendered = hybrid_fetcher.refetch_rendered(url) + if rendered.status == 200 and rendered.text: + return rendered + return result + + @staticmethod + def sync_from_fetch_result( + result: FetchResult, + url: str, + *, + content_length: int, + headers_dict: dict, + ) -> dict[str, Any]: + """Copy FetchResult fields after a post-parse browser refetch.""" + return { + "text": result.text, + "fetch_method": result.fetch_method, + "final_url": result.final_url or url, + "content_length": result.content_length or content_length, + "response_time_ms": result.response_time_ms, + "headers_dict": result.headers_dict or headers_dict, + "redirect_chain_length": result.redirect_chain_length, + "status": result.status, + "ct": result.content_type, + } + + @staticmethod + def build_robots_blocked_row(url: str, *, store_outlinks: bool) -> dict[str, Any]: + row = empty_crawl_row( + url=url, + status="blocked_by_robots", + fetch_method="static", + ) + if store_outlinks: + row["outlink_targets"] = "[]" + return row + + @staticmethod + def build_fetch_error_row( + url: str, + result: FetchResult, + *, + fetch_method: str, + store_outlinks: bool, + ) -> dict[str, Any]: + row = empty_crawl_row( + url=url, + status="error", + fetch_method=fetch_method, + headers_dict=result.headers_dict or {}, + redirect_chain_length=result.redirect_chain_length, + ) + if store_outlinks: + row["outlink_targets"] = "[]" + if result.browser_diagnostics: + row["page_analysis"] = merge_browser_into_page_analysis( + None, result.browser_diagnostics + ) + return row + + @staticmethod + def merge_browser_diagnostics(ext: dict[str, Any], result: FetchResult) -> None: + if result.browser_diagnostics: + ext["page_analysis"] = merge_browser_into_page_analysis( + ext.get("page_analysis"), result.browser_diagnostics + ) diff --git a/src/website_profiling/crawl/schema.py b/src/website_profiling/crawl/schema.py new file mode 100644 index 00000000..3d3cf077 --- /dev/null +++ b/src/website_profiling/crawl/schema.py @@ -0,0 +1,154 @@ +"""Crawl row schema: single source for DataFrame columns and default field values.""" + +from __future__ import annotations + +from typing import Any, Optional + +# Core crawl columns (excluding optional outlink_targets). +CRAWL_ROW_COLUMNS: list[str] = [ + "url", + "status", + "content_type", + "title", + "outlinks", + "response_time_ms", + "content_length", + "final_url", + "meta_description", + "meta_description_len", + "h1", + "h1_count", + "canonical_url", + "viewport_present", + "viewport_content", + "noindex", + "has_schema", + "heading_sequence", + "heading_text", + "images_without_alt", + "images_total", + "img_without_lazy", + "img_without_dimensions", + "aria_count", + "mixed_content_count", + "redirect_chain_length", + "cache_control", + "etag", + "x_robots_tag", + "strict_transport_security", + "x_content_type_options", + "x_frame_options", + "content_security_policy", + "script_count", + "link_stylesheet_count", + "total_js_bytes", + "total_css_bytes", + "word_count", + "reading_level", + "content_html_ratio", + "top_keywords", + "content_excerpt", + "og_title", + "og_description", + "og_image", + "og_type", + "twitter_card", + "twitter_title", + "twitter_image", + "tech_stack", + "depth", + "page_analysis", + "fetch_method", +] + + +def empty_crawl_row_ext( + url: str, + headers_dict: Optional[dict] = None, + redirect_chain_length: int = 0, +) -> dict[str, Any]: + """Default SEO/performance extension fields when no HTML or on error.""" + h = headers_dict or {} + return { + "response_time_ms": "", + "content_length": 0, + "final_url": url, + "meta_description": "", + "meta_description_len": 0, + "h1": "", + "h1_count": 0, + "canonical_url": "", + "viewport_present": False, + "viewport_content": "", + "noindex": False, + "has_schema": False, + "heading_sequence": "", + "heading_text": "", + "images_without_alt": 0, + "images_total": 0, + "img_without_lazy": 0, + "img_without_dimensions": 0, + "aria_count": 0, + "mixed_content_count": 0, + "redirect_chain_length": redirect_chain_length, + "cache_control": h.get("Cache-Control", ""), + "etag": h.get("ETag", ""), + "x_robots_tag": h.get("X-Robots-Tag", ""), + "strict_transport_security": h.get("Strict-Transport-Security", ""), + "x_content_type_options": h.get("X-Content-Type-Options", ""), + "x_frame_options": h.get("X-Frame-Options", ""), + "content_security_policy": h.get("Content-Security-Policy", ""), + "script_count": 0, + "link_stylesheet_count": 0, + "total_js_bytes": 0, + "total_css_bytes": 0, + "word_count": 0, + "reading_level": 0.0, + "content_html_ratio": 0.0, + "top_keywords": "[]", + "content_excerpt": "", + "og_title": "", + "og_description": "", + "og_image": "", + "og_type": "", + "twitter_card": "", + "twitter_title": "", + "twitter_image": "", + "tech_stack": "[]", + "depth": None, + "page_analysis": "{}", + } + + +def empty_crawl_row( + url: Optional[str] = None, + status: str | int = "error", + *, + content_type: str = "", + title: str = "", + outlinks: int = 0, + fetch_method: str = "static", + headers_dict: Optional[dict] = None, + redirect_chain_length: int = 0, + **overrides: Any, +) -> dict[str, Any]: + """Build a full crawl result row with defaults; overrides merge on top.""" + row: dict[str, Any] = { + "url": url, + "status": status, + "content_type": content_type, + "title": title, + "outlinks": outlinks, + "fetch_method": fetch_method, + **empty_crawl_row_ext(url or "", headers_dict, redirect_chain_length), + } + row.update(overrides) + return row + + +def crawl_dataframe_columns(*, store_outlinks: bool = False) -> list[str]: + """Column list for an empty crawl DataFrame.""" + cols = list(CRAWL_ROW_COLUMNS) + if store_outlinks: + cols.append("outlink_targets") + return cols diff --git a/src/website_profiling/lighthouse/config.py b/src/website_profiling/lighthouse/config.py new file mode 100644 index 00000000..1a6c7b0b --- /dev/null +++ b/src/website_profiling/lighthouse/config.py @@ -0,0 +1,166 @@ +"""Lighthouse CLI configuration and command helpers.""" +from __future__ import annotations + +import os +import shutil +import subprocess +import threading +from pathlib import Path + +# Lighthouse "good" thresholds for human summary +LCP_GOOD_MS = 2500 +CLS_GOOD = 0.1 +TBT_GOOD_MS = 200 +FCP_GOOD_MS = 1800 + +_LIGHTHOUSE_INSTALL_MSG = ( + "Lighthouse not found. Install Node/npm (https://nodejs.org), then run: npm install -g lighthouse. " + "Chrome or Chromium is also required for headless mode." +) + +_NPX_LIGHTHOUSE_LOCK = threading.Lock() +_LIGHTHOUSE_FLOW_MODES = frozenset({"snapshot", "timespan"}) + +def _repo_root() -> str: + explicit = (os.environ.get("WEBSITE_PROFILING_ROOT") or "").strip() + if explicit: + return explicit + return str(Path(__file__).resolve().parents[3]) + + +def _lighthouse_flow_script() -> str: + return os.path.join(_repo_root(), "scripts", "lighthouse_user_flow.mjs") + + +def _normalize_lighthouse_mode(mode: str | None) -> str: + m = (mode or "navigation").strip().lower() or "navigation" + if m not in ("navigation", "snapshot", "timespan"): + raise RuntimeError( + f"Invalid lighthouse_mode {m!r}; use navigation, snapshot, or timespan." + ) + return m + + +def _node_cmd() -> str: + node = shutil.which("node") + if node is None: + raise RuntimeError( + "Node.js not found. Install Node.js (https://nodejs.org) for Lighthouse user flows." + ) + return node + + +def _build_report_html_content(summary: dict[str, Any]) -> str: + """Build report.html content (for DB or file). Returns HTML string.""" + import html as html_module + mm = summary.get("median_metrics") or {} + cs = summary.get("category_scores") or {} + failures = summary.get("top_failures") or [] + raw_reports = summary.get("raw_reports") or [] + url = html_module.escape(summary.get("url", "")) + path_summary = "summary.json" + path_human = "human_summary.txt" + path_diag = "diagnostics.json" + raw_dir = "raw_runs" + rows_fail = "".join( + f"{html_module.escape(str(f.get('id', '')))}{html_module.escape(str(f.get('impact', '')))}{html_module.escape(str(f.get('helpText', ''))[:80])}..." + for f in failures[:10] + ) or "None" + raw_links = "".join(f"{os.path.basename(p)} " for p in raw_reports[:5]) + return f""" + +Lighthouse Report + +

Lighthouse Report

+

URL: {url}

+

Median metrics

+ + + + + + +
MetricValue
LCP (ms){mm.get('lcp_ms') or '—'}
CLS{mm.get('cls') or '—'}
TBT (ms){mm.get('tbt_ms') or '—'}
FCP (ms){mm.get('fcp_ms') or '—'}
+

Category scores (0–100)

+ + + + + + + +
CategoryScore
performance{cs.get('performance') or '—'}
accessibility{cs.get('accessibility') or '—'}
best-practices{cs.get('best-practices') or '—'}
seo{cs.get('seo') or '—'}
pwa{cs.get('pwa') or '—'}
+

Top failures

+{rows_fail}
AuditImpactHelp
+

Artifacts

+

summary.json | human_summary.txt | diagnostics.json

+

Raw runs: {raw_links or '—'}

+ + +""" + + +def _write_report_html(output_dir: str, summary: dict[str, Any]) -> None: + """Write report.html to output_dir (used when not using DB).""" + content = summary.get("report_html") or _build_report_html_content(summary) + report_path = os.path.join(output_dir, "report.html") + with open(report_path, "w", encoding="utf-8") as f: + f.write(content) + + +def _url_safe(s: str) -> str: + """Return a filesystem-safe slug from URL for filenames.""" + return re.sub(r"[^\w\-.]", "_", s.strip().rstrip("/"))[:80] + + +def _lighthouse_cmd() -> list[str]: + """Return argv prefix: [resolved lighthouse] or [resolved npx, -y, lighthouse]. Paths from shutil.which (portable).""" + explicit = (os.environ.get("LIGHTHOUSE_PATH") or os.environ.get("LIGHTHOUSE_BIN") or "").strip() + if explicit and os.path.isfile(explicit) and os.access(explicit, os.X_OK): + return [explicit] + lh = shutil.which("lighthouse") + if lh is not None: + return [lh] + npx = shutil.which("npx") + if npx is not None: + return [npx, "-y", "lighthouse"] + raise RuntimeError(_LIGHTHOUSE_INSTALL_MSG) + + +def _uses_npx(cmd: list[str]) -> bool: + base = os.path.basename(cmd[0]).lower() + return base in ("npx", "npx.cmd") + + +def is_lighthouse_available() -> bool: + """Return True if lighthouse or npx is on PATH (so we can run Lighthouse).""" + try: + _lighthouse_cmd() + return True + except RuntimeError: + return False + + +def _preset_for_strategy(strategy: str) -> str: + """Map user strategy 'mobile'|'desktop' to Lighthouse CLI preset. Newer Lighthouse only accepts perf, experimental, desktop.""" + s = (strategy or "mobile").lower() + if s == "desktop": + return "desktop" + return "perf" # mobile -> perf (mobile-like throttling in current Lighthouse) + + +# Valid Lighthouse category IDs for --only-categories +LIGHTHOUSE_CATEGORY_IDS = {"performance", "accessibility", "best-practices", "seo", "pwa"} + + +def _parse_categories(categories: str | list[str] | None) -> list[str] | None: + """Return list of valid category IDs, or None to run all categories.""" + if categories is None: + return None + if isinstance(categories, str): + categories = [c.strip().lower() for c in categories.split(",") if c.strip()] + if not categories: + return None + out = [c for c in categories if c in LIGHTHOUSE_CATEGORY_IDS] + return out if out else None + diff --git a/src/website_profiling/lighthouse/result_parser.py b/src/website_profiling/lighthouse/result_parser.py new file mode 100644 index 00000000..f3f195d5 --- /dev/null +++ b/src/website_profiling/lighthouse/result_parser.py @@ -0,0 +1,107 @@ +"""Parse Lighthouse JSON output into summary metrics.""" +from __future__ import annotations + +from typing import Any + +import statistics + +def _evidence_from_audit(audit: dict[str, Any]) -> list[str]: + """Extract resource URLs or selectors from audit details.""" + evidence: list[str] = [] + details = audit.get("details") + if not details or not isinstance(details, dict): + return evidence + items = details.get("items") or details.get("nodes") or [] + if not isinstance(items, list): + return evidence + for item in items[:5]: + if isinstance(item, dict): + url = item.get("url") + if url and isinstance(url, str) and not str(url).startswith("data:"): + evidence.append(str(url)[:500]) + node = item.get("node") + if isinstance(node, dict) and node.get("selector"): + evidence.append(str(node["selector"])[:200]) + if item.get("selector"): + evidence.append(str(item["selector"])[:200]) + return evidence[:15] + + +def extract_from_lighthouse_json(data: dict) -> dict[str, Any]: + """Extract LCP, CLS, TBT, FCP, Speed Index, category scores (all 5), and top 10 failing audits with impact and evidence.""" + out: dict[str, Any] = { + "lcp_ms": None, + "cls": None, + "tbt_ms": None, + "fcp_ms": None, + "speed_index_ms": None, + "performance_score": None, + "accessibility_score": None, + "seo_score": None, + "best_practices_score": None, + "pwa_score": None, + "category_scores": {}, + "top_failures": [], + } + lr = data.get("lighthouseResult") or data + audits = lr.get("audits") or {} + cats = lr.get("categories") or {} + + for audit_id, key in [ + ("largest-contentful-paint", "lcp_ms"), + ("cumulative-layout-shift", "cls"), + ("total-blocking-time", "tbt_ms"), + ("first-contentful-paint", "fcp_ms"), + ("speed-index", "speed_index_ms"), + ]: + a = audits.get(audit_id) + if a is not None and "numericValue" in a: + out[key] = a["numericValue"] + + for cat_id, key in [ + ("performance", "performance_score"), + ("accessibility", "accessibility_score"), + ("seo", "seo_score"), + ("best-practices", "best_practices_score"), + ("pwa", "pwa_score"), + ]: + c = cats.get(cat_id) + if c is not None and "score" in c: + s = c["score"] + out[key] = s + out["category_scores"][cat_id] = round((s * 100)) if s is not None else None + + # Resolve impact from warning_mapper for each failure + from ..tools.warnings import resolve_impact + failures = [] + for aid, a in audits.items(): + if a is None: + continue + score = a.get("score") + if score is None: + continue + if score < 1: + title = a.get("title") or aid + help_text = a.get("helpText") or "" + impact = resolve_impact(aid, title, help_text) + evidence = _evidence_from_audit(a) + failures.append({ + "id": aid, + "score": score, + "helpText": help_text, + "impact": impact, + "evidence": evidence, + }) + failures.sort(key=lambda x: (x["score"] or 0)) + out["top_failures"] = failures[:10] + + return out + + +def median_or_none(values: list[float]) -> float | None: + """Return median of list; None if empty or all None.""" + clean = [v for v in values if v is not None] + if not clean: + return None + return statistics.median(clean) + diff --git a/src/website_profiling/lighthouse/runner.py b/src/website_profiling/lighthouse/runner.py index ee79e736..3b0d0ca7 100644 --- a/src/website_profiling/lighthouse/runner.py +++ b/src/website_profiling/lighthouse/runner.py @@ -1,68 +1,38 @@ """ Run Lighthouse locally via CLI for a given URL; return machine-readable summary with median metrics. -Writes raw_runs/, summary.json, diagnostics.json, human_summary.txt, and optionally report.html. -Uses global lighthouse if on PATH (or LIGHTHOUSE_PATH), otherwise runs via npx (serialized to avoid cache races). -Requires: Node + npm, Chrome/Chromium. """ +from __future__ import annotations + import json import os import re import shutil -import statistics import subprocess import sys -import threading from datetime import datetime, timezone -from pathlib import Path from typing import Any from ..console_io import console_print - -# Lighthouse "good" thresholds for human summary -LCP_GOOD_MS = 2500 -CLS_GOOD = 0.1 -TBT_GOOD_MS = 200 -FCP_GOOD_MS = 1800 - -_LIGHTHOUSE_INSTALL_MSG = ( - "Lighthouse not found. Install Node/npm (https://nodejs.org), then run: npm install -g lighthouse. " - "Chrome or Chromium is also required for headless mode." +from .config import ( + CLS_GOOD, + FCP_GOOD_MS, + LCP_GOOD_MS, + TBT_GOOD_MS, + _LIGHTHOUSE_INSTALL_MSG, + _LIGHTHOUSE_FLOW_MODES, + _NPX_LIGHTHOUSE_LOCK, + _lighthouse_cmd, + _lighthouse_flow_script, + _node_cmd, + _normalize_lighthouse_mode, + _parse_categories, + _preset_for_strategy, + _repo_root, + _url_safe, + _uses_npx, + is_lighthouse_available, ) - -# Serialise npx-on-demand installs — parallel npx runs corrupt /root/.npm/_npx cache in Docker. -_NPX_LIGHTHOUSE_LOCK = threading.Lock() - -_LIGHTHOUSE_FLOW_MODES = frozenset({"snapshot", "timespan"}) - - -def _repo_root() -> str: - explicit = (os.environ.get("WEBSITE_PROFILING_ROOT") or "").strip() - if explicit: - return explicit - return str(Path(__file__).resolve().parents[3]) - - -def _lighthouse_flow_script() -> str: - return os.path.join(_repo_root(), "scripts", "lighthouse_user_flow.mjs") - - -def _normalize_lighthouse_mode(mode: str | None) -> str: - m = (mode or "navigation").strip().lower() or "navigation" - if m not in ("navigation", "snapshot", "timespan"): - raise RuntimeError( - f"Invalid lighthouse_mode {m!r}; use navigation, snapshot, or timespan." - ) - return m - - -def _node_cmd() -> str: - node = shutil.which("node") - if node is None: - raise RuntimeError( - "Node.js not found. Install Node.js (https://nodejs.org) for Lighthouse user flows." - ) - return node - +from .result_parser import _evidence_from_audit, extract_from_lighthouse_json, median_or_none def _build_report_html_content(summary: dict[str, Any]) -> str: """Build report.html content (for DB or file). Returns HTML string.""" @@ -272,108 +242,6 @@ def run_lighthouse_once( except FileNotFoundError as e: raise RuntimeError(_LIGHTHOUSE_INSTALL_MSG) from e - -def _evidence_from_audit(audit: dict[str, Any]) -> list[str]: - """Extract resource URLs or selectors from audit details.""" - evidence: list[str] = [] - details = audit.get("details") - if not details or not isinstance(details, dict): - return evidence - items = details.get("items") or details.get("nodes") or [] - if not isinstance(items, list): - return evidence - for item in items[:5]: - if isinstance(item, dict): - url = item.get("url") - if url and isinstance(url, str) and not str(url).startswith("data:"): - evidence.append(str(url)[:500]) - node = item.get("node") - if isinstance(node, dict) and node.get("selector"): - evidence.append(str(node["selector"])[:200]) - if item.get("selector"): - evidence.append(str(item["selector"])[:200]) - return evidence[:15] - - -def extract_from_lighthouse_json(data: dict) -> dict[str, Any]: - """Extract LCP, CLS, TBT, FCP, Speed Index, category scores (all 5), and top 10 failing audits with impact and evidence.""" - out: dict[str, Any] = { - "lcp_ms": None, - "cls": None, - "tbt_ms": None, - "fcp_ms": None, - "speed_index_ms": None, - "performance_score": None, - "accessibility_score": None, - "seo_score": None, - "best_practices_score": None, - "pwa_score": None, - "category_scores": {}, - "top_failures": [], - } - lr = data.get("lighthouseResult") or data - audits = lr.get("audits") or {} - cats = lr.get("categories") or {} - - for audit_id, key in [ - ("largest-contentful-paint", "lcp_ms"), - ("cumulative-layout-shift", "cls"), - ("total-blocking-time", "tbt_ms"), - ("first-contentful-paint", "fcp_ms"), - ("speed-index", "speed_index_ms"), - ]: - a = audits.get(audit_id) - if a is not None and "numericValue" in a: - out[key] = a["numericValue"] - - for cat_id, key in [ - ("performance", "performance_score"), - ("accessibility", "accessibility_score"), - ("seo", "seo_score"), - ("best-practices", "best_practices_score"), - ("pwa", "pwa_score"), - ]: - c = cats.get(cat_id) - if c is not None and "score" in c: - s = c["score"] - out[key] = s - out["category_scores"][cat_id] = round((s * 100)) if s is not None else None - - # Resolve impact from warning_mapper for each failure - from ..tools.warnings import resolve_impact - failures = [] - for aid, a in audits.items(): - if a is None: - continue - score = a.get("score") - if score is None: - continue - if score < 1: - title = a.get("title") or aid - help_text = a.get("helpText") or "" - impact = resolve_impact(aid, title, help_text) - evidence = _evidence_from_audit(a) - failures.append({ - "id": aid, - "score": score, - "helpText": help_text, - "impact": impact, - "evidence": evidence, - }) - failures.sort(key=lambda x: (x["score"] or 0)) - out["top_failures"] = failures[:10] - - return out - - -def median_or_none(values: list[float]) -> float | None: - """Return median of list; None if empty or all None.""" - clean = [v for v in values if v is not None] - if not clean: - return None - return statistics.median(clean) - - def run_lighthouse_audit( url: str, strategy: str = "mobile", diff --git a/src/website_profiling/parsing/__init__.py b/src/website_profiling/parsing/__init__.py new file mode 100644 index 00000000..6af83821 --- /dev/null +++ b/src/website_profiling/parsing/__init__.py @@ -0,0 +1 @@ +"""HTML/CSV parsing utilities.""" diff --git a/src/website_profiling/parsing/content.py b/src/website_profiling/parsing/content.py new file mode 100644 index 00000000..b61de2f3 --- /dev/null +++ b/src/website_profiling/parsing/content.py @@ -0,0 +1,108 @@ +"""Content text and social meta parsing.""" +from __future__ import annotations + +import json + +_STOP_WORDS = frozenset({ + "the", "and", "for", "that", "this", "with", "from", "your", "have", "are", + "was", "were", "been", "will", "would", "could", "should", "about", "which", + "their", "there", "what", "when", "where", "more", "some", "than", "them", + "other", "into", "over", "also", "just", "after", "before", "only", "then", + "very", "most", "each", "such", "like", "does", "here", "because", "being", + "well", "while", "these", "those", "both", "many", "much", "even", "back", + "through", "still", "between", "every", "under", "last", "long", "great", + "make", "same", "come", "take", "know", "they", "page", "site", "home", + "click", "read", "view", "next", "menu", "main", "skip", "content", "link", + "http", "https", "www", "html", "class", "none", "true", "false", "null", +}) + + +def _count_syllables(word: str) -> int: + word = word.lower().strip() + if len(word) <= 3: + return 1 + vowels = "aeiouy" + count = 0 + prev_vowel = False + for ch in word: + is_vowel = ch in vowels + if is_vowel and not prev_vowel: + count += 1 + prev_vowel = is_vowel + if word.endswith("e") and count > 1: + count -= 1 + return max(1, count) + + +def parse_content_text(soup, raw_html: str, excerpt_max_chars: int = 0) -> dict: + """Extract content analytics: word count, reading level, content-to-HTML ratio, top keywords. + + excerpt_max_chars: when > 0, strip script/style from body and store a whitespace-normalized + plain-text excerpt (truncated) in ``content_excerpt`` for analysis / AI / UI. + """ + import re + from collections import Counter + + body = soup.find("body") + if body: + for tag in body.find_all(["script", "style", "noscript"]): + tag.decompose() + body_text = body.get_text(separator=" ", strip=True) if body else "" + words = [w for w in re.findall(r"[a-zA-Z]+", body_text) if len(w) >= 2] + word_count = len(words) + + sentences = [s.strip() for s in re.split(r"[.!?]+", body_text) if len(s.strip()) > 5] + sentence_count = max(1, len(sentences)) + + total_syllables = sum(_count_syllables(w) for w in words) if words else 0 + + reading_level = 0.0 + if word_count > 30: + reading_level = ( + 0.39 * (word_count / sentence_count) + + 11.8 * (total_syllables / max(1, word_count)) + - 15.59 + ) + reading_level = max(0.0, min(18.0, round(reading_level, 1))) + + html_len = max(1, len(raw_html)) + content_html_ratio = round(len(body_text) / html_len * 100, 1) + + keyword_words = [w.lower() for w in words if len(w) >= 4 and w.lower() not in _STOP_WORDS] + top_keywords = Counter(keyword_words).most_common(10) + max_kw = top_keywords[0][1] if top_keywords else 0 + kw_rows = [] + for w, c in top_keywords: + score = round(100 * c / max_kw) if max_kw else 0 + kw_rows.append({"word": w, "count": c, "score": int(score)}) + + excerpt = "" + if excerpt_max_chars and excerpt_max_chars > 0 and body_text: + excerpt = re.sub(r"\s+", " ", body_text.strip()) + if len(excerpt) > excerpt_max_chars: + excerpt = excerpt[: excerpt_max_chars].rsplit(" ", 1)[0].strip() or excerpt[:excerpt_max_chars] + + return { + "word_count": word_count, + "reading_level": reading_level, + "content_html_ratio": content_html_ratio, + "top_keywords": json.dumps(kw_rows), + "content_excerpt": excerpt, + } + + +def parse_social_meta(soup) -> dict: + """Extract Open Graph and Twitter Card meta tags.""" + def _meta_content(attrs: dict) -> str: + tag = soup.find("meta", attrs=attrs) + return (tag.get("content") or "").strip() if tag else "" + + return { + "og_title": _meta_content({"property": "og:title"}), + "og_description": _meta_content({"property": "og:description"}), + "og_image": _meta_content({"property": "og:image"}), + "og_type": _meta_content({"property": "og:type"}), + "twitter_card": _meta_content({"name": "twitter:card"}), + "twitter_title": _meta_content({"name": "twitter:title"}), + "twitter_image": _meta_content({"name": "twitter:image"}), + } diff --git a/src/website_profiling/parsing/io.py b/src/website_profiling/parsing/io.py new file mode 100644 index 00000000..758e0191 --- /dev/null +++ b/src/website_profiling/parsing/io.py @@ -0,0 +1,72 @@ +"""DataFrame and edge list I/O.""" +from __future__ import annotations + +import json +import os + +import pandas as pd + +""" +Shared helpers for crawler and report/plot scripts. +""" +import json +import os +import warnings +from urllib.parse import urljoin, urldefrag, urlparse +import urllib.robotparser as robotparser +import ast +import math + +import pandas as pd +from bs4 import BeautifulSoup + + +def load_dataframe(path: str) -> pd.DataFrame: + """Load a DataFrame from CSV or JSON (by extension).""" + if not os.path.isfile(path): + raise FileNotFoundError(path) + path_lower = path.lower() + if path_lower.endswith(".json"): + return pd.read_json(path, orient="records") + return pd.read_csv(path) + + +def save_dataframe(df: pd.DataFrame, path: str) -> None: + """Save a DataFrame to CSV or JSON (by extension). Uses default_handler for JSON to avoid numpy types.""" + path_lower = path.lower() + if path_lower.endswith(".json"): + df.to_json(path, orient="records", indent=2, date_format="iso", default_handler=str) + else: + df.to_csv(path, index=False) + + +def load_edges(path: str) -> list[tuple[str, str]]: + """Load edge list from CSV or JSON (by extension). Returns list of (from_url, to_url).""" + if not os.path.isfile(path): + return [] + path_lower = path.lower() + try: + if path_lower.endswith(".json"): + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, list) and data and isinstance(data[0], dict): + return [(str(o.get("from", "")), str(o.get("to", ""))) for o in data if o.get("from") and o.get("to")] + return [] + edf = pd.read_csv(path) + if {"from", "to"}.issubset(edf.columns): + return [(str(a).rstrip("/"), str(b).rstrip("/")) for a, b in edf[["from", "to"]].values] + except Exception: + pass + return [] + + +def save_edges(edges: list[tuple[str, str]], path: str) -> None: + """Save edge list to CSV or JSON (by extension).""" + path_lower = path.lower() + if path_lower.endswith(".json"): + data = [{"from": a, "to": b} for a, b in edges] + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + else: + pd.DataFrame(edges, columns=["from", "to"]).to_csv(path, index=False) + diff --git a/src/website_profiling/parsing/links.py b/src/website_profiling/parsing/links.py new file mode 100644 index 00000000..6f78fb31 --- /dev/null +++ b/src/website_profiling/parsing/links.py @@ -0,0 +1,155 @@ +"""Link normalization and extraction.""" +from __future__ import annotations + +import ast +import math +from urllib.parse import urldefrag, urljoin, urlparse + +from bs4 import BeautifulSoup + +_TRACKING_PARAM_PREFIXES = ("utm_",) +_FACET_PARAM_NAMES = frozenset({"sort", "filter", "page", "offset", "limit"}) + + +def strip_crawl_query_params(url: str, ignore_params: list[str] | None = None) -> str: + """Remove tracking and facet query params for crawl deduplication.""" + parsed = urlparse(url) + if not parsed.query: + return url.rstrip("/") + ignore = {p.lower() for p in (ignore_params or [])} + parts = [] + for pair in parsed.query.split("&"): + if not pair: + continue + key = pair.split("=", 1)[0].lower() + if key in ignore: + continue + if any(key.startswith(p) for p in _TRACKING_PARAM_PREFIXES): + continue + if key in _FACET_PARAM_NAMES: + continue + parts.append(pair) + query = "&".join(parts) + rebuilt = parsed._replace(query=query).geturl() + return rebuilt.rstrip("/") + + +def normalize_link( + base: str, + href: str, + strip_params: bool = True, + ignore_params: list[str] | None = None, +) -> str | None: + if not href: + return None + href = href.strip() + if href.startswith(("mailto:", "javascript:", "tel:", "data:")): + return None + joined = urljoin(base, href) + joined, _ = urldefrag(joined) + parsed = urlparse(joined) + if parsed.scheme not in ("http", "https"): + return None + out = joined.rstrip("/") + if strip_params: + out = strip_crawl_query_params(out, ignore_params) + return out + + +def _parse_rel_flags(rel_raw: str) -> tuple[bool, bool, bool]: + parts = {p.strip().lower() for p in (rel_raw or "").split() if p.strip()} + return ("nofollow" in parts, "sponsored" in parts, "ugc" in parts) + + +def _anchor_text_from_tag(a) -> str: + parts: list[str] = [] + for child in a.children: + if getattr(child, "name", None) == "img": + parts.append("[image]") + elif isinstance(child, str): + t = child.strip() + if t: + parts.append(t) + text = " ".join(parts).strip() or a.get_text(separator=" ", strip=True) + return (text or "")[:500] + + +def parse_link_edges(base_url: str, html_text: str) -> tuple[str, list[dict]]: + """Extract title and rich outbound link records from HTML.""" + soup = BeautifulSoup(html_text, "lxml") + title_tag = ( + soup.title.string.strip() + if soup.title and soup.title.string + else "" + ) + start_netloc = urlparse(base_url).netloc + edges: list[dict] = [] + for a in soup.find_all("a", href=True): + ln = normalize_link(base_url, a["href"]) + if not ln: + continue + rel_raw = a.get("rel") or "" + if isinstance(rel_raw, list): + rel_str = " ".join(str(x) for x in rel_raw) + else: + rel_str = str(rel_raw) + nofollow, sponsored, ugc = _parse_rel_flags(rel_str) + link_type = "internal" if urlparse(ln).netloc == start_netloc else "external" + edges.append({ + "to_url": ln.rstrip("/"), + "anchor_text": _anchor_text_from_tag(a), + "rel": rel_str.strip(), + "is_nofollow": nofollow, + "is_sponsored": sponsored, + "is_ugc": ugc, + "link_type": link_type, + }) + return title_tag, edges + + +def parse_links(base_url: str, html_text: str) -> tuple[str, set[str]]: + """Extract page title and set of absolute links from HTML. Returns (title, links).""" + title, edges = parse_link_edges(base_url, html_text) + return title, {e["to_url"] for e in edges} + +def _is_empty(raw) -> bool: + if raw is None: + return True + if isinstance(raw, float) and math.isnan(raw): + return True + if raw == "": + return True + return False + + +def parse_links_serialized(raw) -> list[str]: + """ + Parse a serialized list of URLs from CSV/DataFrame (string list repr, comma-separated, or list). + """ + if _is_empty(raw): + return [] + if isinstance(raw, list): + return [str(x).strip().rstrip("/") for x in raw if x] + s = str(raw).strip() + if not s: + return [] + if s.startswith("[") and s.endswith("]"): + try: + v = ast.literal_eval(s) + if isinstance(v, (list, tuple)): + return [str(x).strip().rstrip("/") for x in v if x] + except Exception: + pass + return [t.strip().rstrip("/") for t in s.split(",") if t.strip()] + + +# Column names that may contain serialized outlink lists (for building edges from crawl CSV) +LINK_COLUMN_NAMES = ( + "links", + "edges", + "outlinks", + "outlink_targets", + "targets", + "link_targets", + "links_list", +) diff --git a/src/website_profiling/parsing/robots.py b/src/website_profiling/parsing/robots.py new file mode 100644 index 00000000..de523770 --- /dev/null +++ b/src/website_profiling/parsing/robots.py @@ -0,0 +1,17 @@ +"""robots.txt loading.""" +from __future__ import annotations + +from urllib.parse import urlparse +import urllib.robotparser as robotparser + +def load_robots(start_url: str): + """Load robots.txt for the given URL; returns RobotFileParser or None on error.""" + parsed = urlparse(start_url) + robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt" + rp = robotparser.RobotFileParser() + rp.set_url(robots_url) + try: + rp.read() + return rp + except Exception: + return None diff --git a/src/website_profiling/parsing/seo.py b/src/website_profiling/parsing/seo.py new file mode 100644 index 00000000..e64187d8 --- /dev/null +++ b/src/website_profiling/parsing/seo.py @@ -0,0 +1,138 @@ +"""SEO and resource parsing from HTML.""" +from __future__ import annotations + +from urllib.parse import urlparse + +from bs4 import BeautifulSoup + +from .links import normalize_link + +def parse_seo(base_url: str, html_text: str) -> tuple[str, int, str, int, str]: + """ + Extract SEO-related fields from HTML. + Returns (meta_description, meta_description_len, h1_text, h1_count, canonical_url). + """ + soup = BeautifulSoup(html_text, "lxml") + meta_desc = "" + meta = soup.find("meta", attrs={"name": "description"}) + if meta and meta.get("content"): + meta_desc = (meta["content"] or "").strip() + if not meta_desc: + og = soup.find("meta", attrs={"property": "og:description"}) + if og and og.get("content"): + meta_desc = (og["content"] or "").strip() + meta_desc_len = len(meta_desc) + + h1_tags = soup.find_all("h1") + h1_count = len(h1_tags) + h1_text = (h1_tags[0].get_text(separator=" ", strip=True) if h1_tags else "") or "" + + canonical_url = "" + link_canonical = soup.find("link", attrs={"rel": "canonical"}) + if link_canonical and link_canonical.get("href"): + canonical_url = normalize_link(base_url, link_canonical["href"]) or "" + + return meta_desc, meta_desc_len, h1_text, h1_count, canonical_url + + +def parse_seo_extended(html_text: str, base_url: str) -> dict: + """ + Extract extended SEO/accessibility/performance-related fields from HTML. + Returns a dict with: viewport_present, viewport_content, noindex, has_schema, + heading_sequence, images_without_alt, images_total, img_without_lazy, img_without_dimensions, + aria_count, mixed_content_count. + """ + soup = BeautifulSoup(html_text, "lxml") + out = { + "viewport_present": False, + "viewport_content": "", + "noindex": False, + "has_schema": False, + "heading_sequence": [], + "heading_text": [], + "images_without_alt": 0, + "images_total": 0, + "img_without_lazy": 0, + "img_without_dimensions": 0, + "aria_count": 0, + "mixed_content_count": 0, + } + # Viewport + viewport = soup.find("meta", attrs={"name": "viewport"}) + if viewport and viewport.get("content"): + out["viewport_present"] = True + out["viewport_content"] = (viewport["content"] or "").strip() + # noindex + robots = soup.find("meta", attrs={"name": "robots"}) + if robots and robots.get("content"): + content = (robots["content"] or "").lower() + out["noindex"] = "noindex" in content + # Structured data: JSON-LD or microdata + if soup.find("script", type="application/ld+json"): + out["has_schema"] = True + if soup.find(attrs={"itemscope": True}): + out["has_schema"] = True + # Heading order (h1..h6 tag names) and visible heading copy (for keywords / fingerprints) + for tag in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]): + if tag.name: + out["heading_sequence"].append(tag.name) + text = (tag.get_text(separator=" ", strip=True) or "").strip() + if text: + out["heading_text"].append(text) + # Images: alt, lazy, dimensions + base_scheme = urlparse(base_url).scheme.lower() + for img in soup.find_all("img"): + out["images_total"] += 1 + if not img.get("alt") and not img.get("aria-label"): + out["images_without_alt"] += 1 + loading = (img.get("loading") or "").strip().lower() + if loading != "lazy": + out["img_without_lazy"] += 1 + if not img.get("width") and not img.get("height"): + out["img_without_dimensions"] += 1 + src = img.get("src") or "" + if base_scheme == "https" and src.strip().lower().startswith("http://"): + out["mixed_content_count"] += 1 + # ARIA: count elements with any aria- attribute + for el in soup.find_all(True): + if getattr(el, "attrs", None) and any(k.startswith("aria-") for k in el.attrs): + out["aria_count"] += 1 + # Mixed content: links and other src/href + for tag in soup.find_all(True): + for attr in ("href", "src", "srcset"): + val = tag.get(attr) + if not val or base_scheme != "https": + continue + val = str(val).strip().lower() + if val.startswith("http://"): + out["mixed_content_count"] += 1 + elif attr == "srcset": + for part in val.split(","): + part = part.strip().split()[0] if part.strip() else "" + if part.startswith("http://"): + out["mixed_content_count"] += 1 + return out +def parse_resources(html_text: str, base_url: str) -> dict: + """ + Extract script/link resource counts and total sizes (same-origin only, no fetch). + Returns dict: script_count, link_stylesheet_count, script_urls, stylesheet_urls + (URLs for optional later HEAD/GET). Does not fetch; caller may fetch with limit. + """ + soup = BeautifulSoup(html_text, "lxml") + parsed_base = urlparse(base_url) + script_urls = [] + for s in soup.find_all("script", src=True): + url = normalize_link(base_url, s["src"]) + if url and urlparse(url).netloc == parsed_base.netloc: + script_urls.append(url) + stylesheet_urls = [] + for link in soup.find_all("link", rel=lambda r: r and "stylesheet" in (r.lower() if isinstance(r, str) else "")): + url = link.get("href") and normalize_link(base_url, link["href"]) + if url and urlparse(url).netloc == parsed_base.netloc: + stylesheet_urls.append(url) + return { + "script_count": len(script_urls), + "link_stylesheet_count": len(stylesheet_urls), + "script_urls": script_urls, + "stylesheet_urls": stylesheet_urls, + } diff --git a/src/website_profiling/parsing/tech.py b/src/website_profiling/parsing/tech.py new file mode 100644 index 00000000..587e22f5 --- /dev/null +++ b/src/website_profiling/parsing/tech.py @@ -0,0 +1,116 @@ +"""Technology stack detection.""" +from __future__ import annotations + +import json +import warnings + +_TECH_PATTERNS = [ + ("WordPress", "html", "/wp-content/"), + ("WordPress", "html", "/wp-includes/"), + ("Drupal", "meta_generator", "Drupal"), + ("Joomla", "meta_generator", "Joomla"), + ("Shopify", "html", "cdn.shopify.com"), + ("Squarespace", "html", "squarespace.com"), + ("Wix", "html", "wix.com"), + ("Next.js", "html", "__NEXT_DATA__"), + ("Next.js", "html", "_next/static"), + ("Nuxt.js", "html", "__NUXT__"), + ("Gatsby", "html", "gatsby-"), + ("React", "html", "data-reactroot"), + ("React", "html", "__REACT_DEVTOOLS"), + ("React", "html", "react.production.min"), + ("Vue.js", "html", "__vue"), + ("Vue.js", "html", "vue.min.js"), + ("Angular", "html", "ng-version"), + ("Angular", "html", "ng-app"), + ("Svelte", "html", "svelte"), + ("jQuery", "html", "jquery"), + ("Bootstrap", "html", "bootstrap"), + ("Tailwind CSS", "html", "tailwindcss"), + ("Google Analytics", "html", "google-analytics.com/analytics.js"), + ("Google Analytics", "html", "googletagmanager.com/gtag"), + ("Google Tag Manager", "html", "googletagmanager.com/gtm.js"), + ("Facebook Pixel", "html", "connect.facebook.net"), + ("Hotjar", "html", "hotjar.com"), + ("Google Fonts", "html", "fonts.googleapis.com"), + ("Font Awesome", "html", "fontawesome"), + ("Cloudflare", "header", "cf-ray"), + ("Nginx", "header_server", "nginx"), + ("Apache", "header_server", "apache"), + ("LiteSpeed", "header_server", "litespeed"), + ("Vercel", "header_server", "vercel"), + ("Netlify", "header_server", "netlify"), + ("Amazon CloudFront", "header", "x-amz-cf-id"), + ("AWS", "header_server", "amazons3"), +] + +# Module-level cache for Wappalyzer instance (avoids reloading technologies file per page). +_wappalyzer_instance = None +_wappalyzer_disabled = False + + +def _is_wappalyzer_regex_warning(msg: str) -> bool: + lower = msg.lower() + return "compiling regex" in lower and "unbalanced parenthesis" in lower + + +def detect_tech_wappalyzer( + url: str, + html: str, + headers: dict, + soup, + wappalyzer=None, +) -> str: + """ + Detect technologies using python-Wappalyzer from existing HTML and headers. + Returns JSON list of tech names. On any failure, falls back to parse_tech_stack(soup, headers, url). + """ + global _wappalyzer_instance, _wappalyzer_disabled + if _wappalyzer_disabled: + return parse_tech_stack(soup, headers, url) + try: + from Wappalyzer import Wappalyzer, WebPage + except ImportError: + return parse_tech_stack(soup, headers, url) + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + instance = wappalyzer if wappalyzer is not None else _wappalyzer_instance + if instance is None: + instance = Wappalyzer.latest() + if wappalyzer is None: + _wappalyzer_instance = instance + webpage = WebPage(url, html=html, headers=headers) + detected = instance.analyze(webpage) + if any(_is_wappalyzer_regex_warning(str(w.message)) for w in caught): + _wappalyzer_disabled = True + _wappalyzer_instance = None + return parse_tech_stack(soup, headers, url) + return json.dumps(sorted(detected)) + except Exception: + return parse_tech_stack(soup, headers, url) + + +def parse_tech_stack(soup, headers: dict, url: str) -> str: + """Detect technologies from HTML patterns and HTTP headers. Returns JSON list of tech names.""" + detected = set() + html_str = str(soup).lower() + meta_gen = soup.find("meta", attrs={"name": "generator"}) + generator = (meta_gen.get("content") or "").strip().lower() if meta_gen else "" + server_header = (headers.get("Server") or headers.get("server") or "").lower() + + for name, source, pattern in _TECH_PATTERNS: + pat = pattern.lower() + if source == "html" and pat in html_str: + detected.add(name) + elif source == "meta_generator" and pat in generator: + detected.add(name) + elif source == "header": + for v in headers.values(): + if isinstance(v, str) and pat in v.lower(): + detected.add(name) + break + elif source == "header_server" and pat in server_header: + detected.add(name) + + return json.dumps(sorted(detected)) diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py index c178dd16..baf28c2c 100644 --- a/src/website_profiling/reporting/builder.py +++ b/src/website_profiling/reporting/builder.py @@ -1,1181 +1,68 @@ """ Generate report data from crawl and write to PostgreSQL. The Next.js UI in web/ reads via /api/report/*. """ -import hashlib +from __future__ import annotations + import json import os -import socket -import ssl -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timezone from typing import Any, Optional from urllib.parse import urlparse import pandas as pd import requests -from bs4 import BeautifulSoup -from tqdm.auto import tqdm - -from ..common import ( - LINK_COLUMN_NAMES, - load_edges, - normalize_link, - parse_links_serialized, -) -from ..tools.keywords import cluster_keywords, extract_candidates_from_df, score_keywords -from ..config import get_bool, get_int + from ..analysis import merge_bundles, run_local_enrichment -from ..analysis.text_hygiene import filter_topic_clusters, is_junk_semantic_term -from ..llm.enrich import cluster_keywords_llm, run_llm_enrichment +from ..config import get_bool, get_int +from ..llm.enrich import run_llm_enrichment from ..llm_config import load_llm_config_from_db, llm_is_enabled -from .categories import build_categories from ..security_scanner import run_security_scan - -# SEO thresholds for recommendations -TITLE_LEN_MIN = 30 -TITLE_LEN_MAX = 60 -META_DESC_LEN_MIN = 70 -META_DESC_LEN_MAX = 160 -THIN_CONTENT_CHARS = 300 - - -def fetch_site_ssl_expires_iso(hostname: str, timeout: float = 5.0) -> Optional[str]: - """Return certificate notAfter as ISO 8601 UTC, or None on failure.""" - host = (hostname or "").strip().lower() - if not host: - return None - try: - ctx = ssl.create_default_context() - with socket.create_connection((host, 443), timeout=timeout) as sock: - with ctx.wrap_socket(sock, server_hostname=host) as ssock: - cert = ssock.getpeercert() - if not cert: - return None - na = cert.get("notAfter") - if not na: - return None - ts = ssl.cert_time_to_seconds(na) - return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() - except Exception: - return None - - -def _strip_www(host: str) -> str: - h = (host or "").strip().lower() - return h[4:] if h.startswith("www.") else h - - -def _url_hostname(url: str) -> str: - if not url: - return "" - try: - return (urlparse(str(url).strip()).hostname or "").lower() - except Exception: - return "" - - -def _hosts_match(a: str, b: str) -> bool: - if not a or not b: - return False - a, b = a.lower(), b.lower() - return a == b or _strip_www(a) == _strip_www(b) - - -def filter_lighthouse_by_host(by_url: dict[str, Any], expected_host: str) -> dict[str, Any]: - """Keep only Lighthouse entries whose URL hostname matches expected_host (www.-tolerant).""" - if not by_url or not expected_host: - return by_url or {} - return {u: v for u, v in by_url.items() if _hosts_match(_url_hostname(u), expected_host)} - - -def _derive_expected_host(start_url: str, df: pd.DataFrame) -> str: - host = _url_hostname(start_url) - if host: - return host - if df is not None and not df.empty and "url" in df.columns: - for u in df["url"]: - h = _url_hostname(str(u)) - if h: - return h - return "" - - -def _pick_lighthouse_summary( - lighthouse_by_url: dict[str, Any], - start_url: str, - global_summary: Optional[dict[str, Any]], - expected_host: str, -) -> Optional[dict[str, Any]]: - """Prefer per-URL summary for this crawl; only use global summary if hostname matches.""" - if lighthouse_by_url and start_url: - match = lighthouse_for_url(lighthouse_by_url, start_url) - if match: - return match - if lighthouse_by_url: - first_key = next(iter(lighthouse_by_url), None) - if first_key is not None: - return lighthouse_by_url[first_key] - if global_summary: - if not expected_host or _hosts_match(_url_hostname(str(global_summary.get("url") or "")), expected_host): - return global_summary - return None - - -def build_lighthouse_by_url_for_report(conn: Any) -> dict[str, Any]: - """ - Merge per-URL Lighthouse page summaries with latest lighthouse_runs row: full audits/items - from normalized tables, uncapped top_failures and diagnostics from stored LHR JSON. - """ - from ..db import ( - read_lh_audits_with_items, - read_lh_runs_by_url, - read_lighthouse_page_summaries, - read_lighthouse_run_json, - ) - from ..lighthouse.runner import _evidence_from_audit, extract_from_lighthouse_json - from ..tools.warnings import parse_lighthouse_to_diagnostics, resolve_impact - - summaries = read_lighthouse_page_summaries(conn) - runs_map = read_lh_runs_by_url(conn) - - summaries_norm: dict[str, Any] = {} - for k, v in summaries.items(): - nk = str(k).strip().rstrip("/") - summaries_norm[nk] = v - - all_urls = set(summaries_norm.keys()) | set(runs_map.keys()) - out: dict[str, Any] = {} - - for u in sorted(all_urls): - base: dict[str, Any] = dict(summaries_norm[u]) if u in summaries_norm else {} - run_ids = runs_map.get(u, []) - run_id = run_ids[-1] if run_ids else None - - if run_id is not None: - raw = read_lighthouse_run_json(conn, run_id) - if not base and raw: - ex = extract_from_lighthouse_json(raw) - lr = raw.get("lighthouseResult") or raw - final_u = lr.get("finalUrl") or lr.get("requestedUrl") or u - base = { - "url": str(final_u).strip().rstrip("/"), - "median_metrics": { - "lcp_ms": ex.get("lcp_ms"), - "cls": ex.get("cls"), - "tbt_ms": ex.get("tbt_ms"), - "fcp_ms": ex.get("fcp_ms"), - "speed_index_ms": ex.get("speed_index_ms"), - "performance_score": ex.get("performance_score"), - "accessibility_score": ex.get("accessibility_score"), - "seo_score": ex.get("seo_score"), - "best_practices_score": ex.get("best_practices_score"), - "pwa_score": ex.get("pwa_score"), - }, - "category_scores": dict(ex.get("category_scores") or {}), - "strategy": "mobile", - "device": "mobile", - "mode": "navigation", - } - base["audits"] = read_lh_audits_with_items(conn, run_id) - if raw: - lr = raw.get("lighthouseResult") or raw - audits_map = lr.get("audits") or {} - failures: list[dict[str, Any]] = [] - for aid, a in audits_map.items(): - if not isinstance(a, dict): - continue - score = a.get("score") - if score is None or score >= 1: - continue - title = a.get("title") or aid - help_text = a.get("helpText") or "" - failures.append( - { - "id": aid, - "score": score, - "helpText": help_text, - "impact": resolve_impact(aid, title, help_text), - "evidence": _evidence_from_audit(a), - } - ) - failures.sort(key=lambda x: (x["score"] or 0)) - base["top_failures"] = failures - base["diagnostics"] = parse_lighthouse_to_diagnostics(raw, max_nodes_in_refs=None) - elif not base: - continue - - if not base.get("url"): - base["url"] = u - out[u] = base - - return out - - -def lighthouse_for_url(lighthouse_by_url: dict[str, Any], url: str) -> Optional[dict[str, Any]]: - """Resolve Lighthouse summary for a crawled URL (trailing-slash tolerant).""" - if not lighthouse_by_url or not url: - return None - u = str(url).strip().rstrip("/") - if u in lighthouse_by_url: - return lighthouse_by_url[u] - for k, v in lighthouse_by_url.items(): - if str(k).strip().rstrip("/") == u: - return v - return None - - -def build_edges_from_df( - df: pd.DataFrame, - edges_csv: str, - same_domain_only: bool, - max_fetch_for_edges: int, - concurrency: int, - timeout: int, - polite_delay: float, - render_mode: str = "static", - js_timeout: int = 30, - js_concurrency: int = 3, - js_wait_until: str = "domcontentloaded", - js_extra_wait_ms: int = 1500, - js_block_resources: bool = True, -) -> list[tuple[str, str]]: - """Build or load edges; return list of (from, to) tuples.""" - edges = load_edges(edges_csv) if (edges_csv or "").strip() else [] - if edges: - return edges - - # Prefer columns that hold URL lists (e.g. outlink_targets); skip "outlinks" (numeric count) - candidate_cols = [ - c for c in df.columns - if c.lower() in LINK_COLUMN_NAMES and c.lower() != "outlinks" - ] - if candidate_cols: - for col in candidate_cols: - if df[col].notna().sum() == 0: - continue - for src, raw in zip(df["url"], df[col].fillna("")): - for t in parse_links_serialized(raw): - if not t: - continue - if same_domain_only and urlparse(src).netloc != urlparse(t).netloc: - continue - edges.append((src, t)) - if edges: - return edges - - session = requests.Session() - session.headers.update({"User-Agent": "WebsiteProfiling/1.0"}) - urls = df["url"].tolist()[:max_fetch_for_edges] - mode = (render_mode or "static").strip().lower() - use_js = mode in ("javascript", "auto") - fetcher = None - if use_js: - from ..crawl.fetchers import build_fetcher - - fetcher = build_fetcher( - render_mode="javascript" if mode == "javascript" else "auto", - timeout=timeout, - user_agent="WebsiteProfiling/1.0", - session=session, - js_timeout=js_timeout, - js_concurrency=js_concurrency, - js_wait_until=js_wait_until, - js_extra_wait_ms=js_extra_wait_ms, - js_block_resources=js_block_resources, - ) - - def fetch(src): - try: - if fetcher is not None: - r = fetcher.fetch(src) - if r.status != 200 or not r.text: - return [] - html = r.text - else: - resp = session.get(src, timeout=timeout, allow_redirects=True) - if resp.status_code != 200 or not resp.headers.get("Content-Type", "").lower().startswith("text/html"): - return [] - html = resp.text - soup = BeautifulSoup(html, "lxml") - out = set() - for a in soup.find_all("a", href=True): - ln = normalize_link(src, a["href"]) - if not ln or (same_domain_only and urlparse(src).netloc != urlparse(ln).netloc): - continue - out.add(ln) - if polite_delay: - time.sleep(polite_delay) - return list(out) - except Exception: - return [] - - try: - with ThreadPoolExecutor(max_workers=concurrency) as ex: - futures = {ex.submit(fetch, u): u for u in urls} - for f in tqdm(as_completed(futures), total=len(futures), desc="Extracting links"): - src = futures[f] - try: - outs = f.result() - except Exception: - outs = [] - for t in outs: - edges.append((src, t)) - finally: - if fetcher is not None: - fetcher.close() - return edges - - -def _fetch_site_level(start_url: str, timeout: int = 8) -> dict: - """Fetch robots.txt, sitemap.xml, ads.txt, and security.txt from start_url origin.""" - from .site_files import fetch_ads_txt, fetch_security_txt, merge_site_file_fields - - parsed = urlparse(start_url) - if not parsed.scheme or not parsed.netloc: - return { - "robots_present": False, - "sitemap_present": False, - "sitemap_valid": False, - "ads_txt_present": False, - "security_txt_present": False, - } - base = f"{parsed.scheme}://{parsed.netloc}" - session = requests.Session() - session.headers.update({"User-Agent": "WebsiteProfiling/1.0"}) - out: dict[str, Any] = { - "robots_present": False, - "sitemap_present": False, - "sitemap_valid": False, - } - try: - r = session.get(f"{base}/robots.txt", timeout=timeout) - if r.status_code == 200 and r.text: - out["robots_present"] = True - for line in r.text.splitlines(): - line = line.strip() - if line.lower().startswith("sitemap:"): - break - except Exception: - pass - try: - r = session.get(f"{base}/sitemap.xml", timeout=timeout) - if r.status_code == 200 and r.text: - out["sitemap_present"] = True - out["sitemap_valid"] = "<" in r.text and ">" in r.text and ("urlset" in r.text or "sitemapindex" in r.text) - except Exception: - pass - merge_site_file_fields(out, fetch_ads_txt(session, base, timeout=timeout)) - merge_site_file_fields(out, fetch_security_txt(session, base, timeout=timeout)) - return out - - -def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: - """Compute crawl summary, SEO health metrics, issues list, and recommendations from crawl DataFrame.""" - total = len(df) - status_str = df["status"].astype(str) if "status" in df.columns else pd.Series(["unknown"] * len(df)) - count_2xx = int((status_str.str.match(r"2\d{2}").fillna(False)).sum()) - count_3xx = int((status_str.str.match(r"3\d{2}").fillna(False)).sum()) - count_4xx = int((status_str.str.match(r"4\d{2}").fillna(False)).sum()) - count_5xx = int((status_str.str.match(r"5\d{2}").fillna(False)).sum()) - count_error = int((status_str.isin(["error", "blocked_by_robots"])).sum()) - success_rate = round(100 * count_2xx / total, 1) if total else 0 - - outlinks = ( - pd.to_numeric(df["outlinks"], errors="coerce").fillna(0).astype(int) - if "outlinks" in df.columns - else pd.Series([0] * len(df)) - ) - title_len = ( - df["title"].fillna("").astype(str).apply(len) - if "title" in df.columns - else pd.Series([0] * len(df)) - ) - crawl_time_s = float(df["crawl_time_s"].iloc[0]) if "crawl_time_s" in df.columns and len(df) else None - - summary = { - "total_urls": total, - "count_2xx": count_2xx, - "count_3xx": count_3xx, - "count_4xx": count_4xx, - "count_5xx": count_5xx, - "count_error": count_error, - "success_rate": success_rate, - "avg_outlinks": round(float(outlinks.mean()), 1) if total else 0, - "avg_title_len": round(float(title_len.mean()), 1) if total else 0, - "crawl_time_s": round(crawl_time_s, 1) if crawl_time_s is not None else None, - } - - # SEO health (when columns exist) - seo_health = {} - if "title" in df.columns: - titles = df["title"].fillna("").astype(str) - seo_health["missing_title"] = int((titles.str.len() == 0).sum()) - seo_health["title_short"] = int(((title_len > 0) & (title_len < TITLE_LEN_MIN)).sum()) - seo_health["title_long"] = int((title_len > TITLE_LEN_MAX).sum()) - seo_health["title_ok"] = int(((title_len >= TITLE_LEN_MIN) & (title_len <= TITLE_LEN_MAX)).sum()) - if "meta_description_len" in df.columns: - md_len = pd.to_numeric(df["meta_description_len"], errors="coerce").fillna(0).astype(int) - seo_health["missing_meta_desc"] = int((md_len == 0).sum()) - seo_health["meta_desc_short"] = int(((md_len > 0) & (md_len < META_DESC_LEN_MIN)).sum()) - seo_health["meta_desc_long"] = int((md_len > META_DESC_LEN_MAX).sum()) - seo_health["meta_desc_ok"] = int(((md_len >= META_DESC_LEN_MIN) & (md_len <= META_DESC_LEN_MAX)).sum()) - if "h1_count" in df.columns: - h1c = pd.to_numeric(df["h1_count"], errors="coerce").fillna(-1).astype(int) - seo_health["h1_zero"] = int((h1c == 0).sum()) - seo_health["h1_one"] = int((h1c == 1).sum()) - seo_health["h1_multi"] = int((h1c > 1).sum()) - if "content_length" in df.columns: - cl = pd.to_numeric(df["content_length"], errors="coerce").fillna(0).astype(int) - seo_health["thin_content"] = int(((cl > 0) & (cl < THIN_CONTENT_CHARS)).sum()) - - # Issues: broken, redirects, SEO - issues = {"broken": [], "redirects": [], "seo": []} - for _, row in df.iterrows(): - u = row.get("url") - if pd.isna(u) or not u: - continue - u = str(u).strip() - st = str(row.get("status", "")).strip() - if st.startswith("4") or st.startswith("5") or st in ("error", "blocked_by_robots"): - issues["broken"].append({"url": u, "status": st}) - elif st.startswith("3"): - final = row.get("final_url") or "" - issues["redirects"].append({"url": u, "status": st, "final_url": str(final) if pd.notna(final) else ""}) - - if "title" in df.columns: - for _, row in df.iterrows(): - u = row.get("url") - if pd.isna(u): - continue - u = str(u).strip() - t = row.get("title") or "" - tl = len(str(t).strip()) - if tl == 0: - issues["seo"].append({"type": "missing_title", "url": u, "message": "Missing title"}) - elif tl < TITLE_LEN_MIN: - issues["seo"].append({"type": "title_short", "url": u, "message": f"Title too short ({tl} chars)"}) - elif tl > TITLE_LEN_MAX: - issues["seo"].append({"type": "title_long", "url": u, "message": f"Title too long ({tl} chars)"}) - if "meta_description_len" in df.columns: - for _, row in df.iterrows(): - md_len = pd.to_numeric(row.get("meta_description_len"), errors="coerce") - if pd.isna(md_len) or md_len == 0: - continue - u = row.get("url") - if pd.isna(u): - continue - u = str(u).strip() - ml = int(md_len) - if ml < META_DESC_LEN_MIN: - issues["seo"].append({"type": "meta_desc_short", "url": u, "message": f"Meta description too short ({ml} chars)"}) - elif ml > META_DESC_LEN_MAX: - issues["seo"].append({"type": "meta_desc_long", "url": u, "message": f"Meta description too long ({ml} chars)"}) - if "h1_count" in df.columns: - for _, row in df.iterrows(): - h1c = pd.to_numeric(row.get("h1_count"), errors="coerce") - if pd.isna(h1c) or h1c == 1: - continue - u = row.get("url") - if pd.isna(u): - continue - u = str(u).strip() - if int(h1c) == 0: - issues["seo"].append({"type": "h1_missing", "url": u, "message": "Missing H1"}) - else: - issues["seo"].append({"type": "h1_multi", "url": u, "message": f"Multiple H1s ({int(h1c)})"}) - if "content_length" in df.columns: - for _, row in df.iterrows(): - cl = pd.to_numeric(row.get("content_length"), errors="coerce") - cl = 0 if pd.isna(cl) else int(cl) - if cl >= THIN_CONTENT_CHARS or cl == 0: - continue - u = row.get("url") - if pd.isna(u): - continue - issues["seo"].append({"type": "thin_content", "url": str(u).strip(), "message": f"Thin content ({int(cl)} chars)"}) - - # Recommendations (actionable bullets) - recommendations = [] - if issues["broken"]: - recommendations.append(f"Fix {len(issues['broken'])} broken or error URL(s).") - if issues["redirects"]: - recommendations.append(f"Review {len(issues['redirects'])} redirect(s); consolidate if possible.") - if seo_health.get("missing_title", 0) > 0: - recommendations.append(f"Add titles to {seo_health['missing_title']} page(s).") - if seo_health.get("title_short", 0) + seo_health.get("title_long", 0) > 0: - n = seo_health.get("title_short", 0) + seo_health.get("title_long", 0) - recommendations.append(f"Optimize title length on {n} page(s) (aim 30–60 chars).") - if seo_health.get("missing_meta_desc", 0) > 0: - recommendations.append(f"Add meta descriptions to {seo_health['missing_meta_desc']} page(s).") - if seo_health.get("meta_desc_short", 0) + seo_health.get("meta_desc_long", 0) > 0: - n = seo_health.get("meta_desc_short", 0) + seo_health.get("meta_desc_long", 0) - recommendations.append(f"Optimize meta description length on {n} page(s) (aim 70–160 chars).") - if seo_health.get("h1_zero", 0) > 0: - recommendations.append(f"Add one H1 per page on {seo_health['h1_zero']} page(s).") - if seo_health.get("h1_multi", 0) > 0: - recommendations.append(f"Use a single H1 per page on {seo_health['h1_multi']} page(s).") - if seo_health.get("thin_content", 0) > 0: - recommendations.append(f"Expand thin content on {seo_health['thin_content']} page(s) (under {THIN_CONTENT_CHARS} chars).") - - return { - "summary": summary, - "seo_health": seo_health, - "issues": issues, - "recommendations": recommendations, - } - - -def _build_content_analytics(df: pd.DataFrame) -> dict: - """Build content analytics: word count stats, reading level distribution, content ratio, top keywords.""" - from collections import Counter - - result = { - "word_count_stats": {"mean": 0, "median": 0, "p25": 0, "p75": 0, "min": 0, "max": 0}, - "word_count_distribution": {}, - "reading_level_distribution": {}, - "content_ratio_distribution": {}, - "top_keywords_site": [], - "thin_pages": [], - } - if "word_count" not in df.columns or df.empty: - return result - - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - if success_df.empty: - return result - - wc = pd.to_numeric(success_df["word_count"], errors="coerce").fillna(0).astype(int) - result["word_count_stats"] = { - "mean": round(float(wc.mean()), 1), - "median": round(float(wc.median()), 1), - "p25": round(float(wc.quantile(0.25)), 1), - "p75": round(float(wc.quantile(0.75)), 1), - "min": int(wc.min()), - "max": int(wc.max()), - } - - wc_bins = [(0, 100), (101, 300), (301, 600), (601, 1000), (1001, 2000), (2001, 999999)] - wc_labels = ["0-100", "101-300", "301-600", "601-1000", "1001-2000", "2001+"] - result["word_count_distribution"] = { - lbl: int(((wc >= lo) & (wc <= hi)).sum()) for (lo, hi), lbl in zip(wc_bins, wc_labels) - } - - if "reading_level" in success_df.columns: - rl = pd.to_numeric(success_df["reading_level"], errors="coerce").fillna(0) - rl_bins = [(0, 5), (6, 8), (9, 12), (13, 99)] - rl_labels = ["Elementary (0-5)", "Middle School (6-8)", "High School (9-12)", "College (13+)"] - result["reading_level_distribution"] = { - lbl: int(((rl >= lo) & (rl <= hi)).sum()) for (lo, hi), lbl in zip(rl_bins, rl_labels) - } - - if "content_html_ratio" in success_df.columns: - cr = pd.to_numeric(success_df["content_html_ratio"], errors="coerce").fillna(0) - cr_bins = [(0, 10), (10.01, 20), (20.01, 40), (40.01, 100)] - cr_labels = ["<10%", "10-20%", "20-40%", ">40%"] - result["content_ratio_distribution"] = { - lbl: int(((cr >= lo) & (cr <= hi)).sum()) for (lo, hi), lbl in zip(cr_bins, cr_labels) - } - - if "top_keywords" in success_df.columns: - kw_counter = Counter() - for raw in success_df["top_keywords"].fillna("[]"): - try: - items = json.loads(str(raw)) if isinstance(raw, str) else raw - if isinstance(items, list): - for item in items: - if isinstance(item, dict): - kw_counter[item.get("word", "")] += item.get("count", 0) - except (json.JSONDecodeError, TypeError): - pass - result["top_keywords_site"] = [ - {"word": w, "count": c} - for w, c in kw_counter.most_common(50) - if w and not is_junk_semantic_term(str(w)) - ][:30] - - for _, row in success_df.iterrows(): - u = row.get("url") - if pd.isna(u) or not u: - continue - w = int(pd.to_numeric(row.get("word_count"), errors="coerce") or 0) - if 0 < w < 300: - result["thin_pages"].append({"url": str(u).strip(), "word_count": w}) - - return result - - -def _parse_top_keywords_items(raw: Any) -> list[dict[str, Any]]: - """Parse per-page top_keywords JSON into dict items with word/count.""" - if raw is None or (isinstance(raw, float) and pd.isna(raw)): - return [] - try: - items = json.loads(str(raw)) if isinstance(raw, str) else raw - except (json.JSONDecodeError, TypeError, ValueError): - return [] - if not isinstance(items, list): - return [] - out: list[dict[str, Any]] = [] - for item in items: - if isinstance(item, dict): - word = str(item.get("word") or "").strip() - if word: - out.append({"word": word, "count": int(item.get("count") or 1)}) - return out - - -def _build_text_content_analysis(df: pd.DataFrame) -> dict: - """Cross-page keyword aggregates for the text content analysis view.""" - empty = { - "vocabulary_stats": { - "unique_terms": 0, - "pages_with_keywords": 0, - "avg_terms_per_page": 0.0, - "total_term_occurrences": 0, - }, - "keyword_index": [], - "keyword_frequency_histogram": {"1": 0, "2-5": 0, "6-20": 0, "21+": 0}, - } - if df.empty or "top_keywords" not in df.columns: - return empty - - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - if success_df.empty: - return empty - - # word -> { total_count, pages: { url -> count } } - index: dict[str, dict[str, Any]] = {} - pages_with_keywords = 0 - total_occurrences = 0 - - for _, row in success_df.iterrows(): - url = row.get("url") - if pd.isna(url) or not url: - continue - url_str = str(url).strip() - items = _parse_top_keywords_items(row.get("top_keywords")) - page_had_kw = False - for item in items: - word = item["word"].lower() - if is_junk_semantic_term(word): - continue - count = max(1, int(item.get("count") or 1)) - if word not in index: - index[word] = {"total_count": 0, "pages": {}} - index[word]["total_count"] += count - index[word]["pages"][url_str] = index[word]["pages"].get(url_str, 0) + count - total_occurrences += count - page_had_kw = True - if page_had_kw: - pages_with_keywords += 1 - - unique_terms = len(index) - avg_terms = round(total_occurrences / pages_with_keywords, 1) if pages_with_keywords else 0.0 - - histogram = {"1": 0, "2-5": 0, "6-20": 0, "21+": 0} - for data in index.values(): - pc = len(data["pages"]) - if pc == 1: - histogram["1"] += 1 - elif pc <= 5: - histogram["2-5"] += 1 - elif pc <= 20: - histogram["6-20"] += 1 - else: - histogram["21+"] += 1 - - sorted_words = sorted(index.items(), key=lambda x: x[1]["total_count"], reverse=True) - keyword_index: list[dict[str, Any]] = [] - for word, data in sorted_words: - top_pages = sorted(data["pages"].items(), key=lambda x: x[1], reverse=True)[:5] - keyword_index.append( - { - "word": word, - "total_count": data["total_count"], - "page_count": len(data["pages"]), - "top_pages": [{"url": u, "count": c} for u, c in top_pages], - } - ) - - return { - "vocabulary_stats": { - "unique_terms": unique_terms, - "pages_with_keywords": pages_with_keywords, - "avg_terms_per_page": avg_terms, - "total_term_occurrences": total_occurrences, - }, - "keyword_index": keyword_index, - "keyword_frequency_histogram": histogram, - } - - -def _build_social_coverage(df: pd.DataFrame) -> dict: - """Build social meta coverage stats: OG and Twitter Card presence percentages.""" - result = { - "og_coverage_pct": 0, - "twitter_coverage_pct": 0, - "og_image_coverage_pct": 0, - "missing_og": [], - "missing_twitter": [], - "og_image_missing": [], - } - if df.empty: - return result - - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - html_df = success_df - if "content_type" in success_df.columns: - html_df = success_df[success_df["content_type"].fillna("").str.contains("text/html", case=False, na=False)] - if html_df.empty: - return result - - total = len(html_df) - - if "og_title" in html_df.columns: - has_og = (html_df["og_title"].fillna("").astype(str).str.strip() != "").sum() - result["og_coverage_pct"] = round(100 * int(has_og) / total, 1) - for _, row in html_df.iterrows(): - u = row.get("url") - if pd.isna(u): - continue - u = str(u).strip() - og = str(row.get("og_title") or "").strip() - if not og: - result["missing_og"].append(u) - - if "twitter_card" in html_df.columns: - has_tw = (html_df["twitter_card"].fillna("").astype(str).str.strip() != "").sum() - result["twitter_coverage_pct"] = round(100 * int(has_tw) / total, 1) - for _, row in html_df.iterrows(): - u = row.get("url") - if pd.isna(u): - continue - u = str(u).strip() - tw = str(row.get("twitter_card") or "").strip() - if not tw: - result["missing_twitter"].append(u) - - if "og_image" in html_df.columns: - has_og_img = (html_df["og_image"].fillna("").astype(str).str.strip() != "").sum() - result["og_image_coverage_pct"] = round(100 * int(has_og_img) / total, 1) - for _, row in html_df.iterrows(): - u = row.get("url") - if pd.isna(u): - continue - u = str(u).strip() - img = str(row.get("og_image") or "").strip() - if not img: - result["og_image_missing"].append(u) - - result["missing_og"] = result["missing_og"][:100] - result["missing_twitter"] = result["missing_twitter"][:100] - result["og_image_missing"] = result["og_image_missing"][:100] - return result - - -def _build_tech_stack_summary(df: pd.DataFrame) -> dict: - """Build tech stack summary: detected technologies with counts and sample URLs.""" - from collections import defaultdict - - result = {"technologies": [], "total_pages_analyzed": 0} - if "tech_stack" not in df.columns or df.empty: - return result - - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - html_df = success_df - if "content_type" in success_df.columns: - html_df = success_df[success_df["content_type"].fillna("").str.contains("text/html", case=False, na=False)] - if html_df.empty: - return result - - result["total_pages_analyzed"] = len(html_df) - tech_urls = defaultdict(list) - - for _, row in html_df.iterrows(): - u = str(row.get("url", "")).strip() - raw = row.get("tech_stack") or "[]" - try: - techs = json.loads(str(raw)) if isinstance(raw, str) else raw - if isinstance(techs, list): - for t in techs: - if isinstance(t, str) and t: - tech_urls[t].append(u) - except (json.JSONDecodeError, TypeError): - pass - - result["technologies"] = sorted( - [{"name": name, "count": len(urls), "sample_urls": urls[:3]} for name, urls in tech_urls.items()], - key=lambda x: x["count"], - reverse=True, - ) - return result - - -def _build_response_time_stats(df: pd.DataFrame) -> dict: - """Build response time statistics and distribution.""" - result = { - "p25": 0, "p50": 0, "p75": 0, "p95": 0, "p99": 0, - "slow_pages": [], - "distribution": {}, - } - if "response_time_ms" not in df.columns or df.empty: - return result - - rt = pd.to_numeric(df["response_time_ms"], errors="coerce").dropna() - if rt.empty: - return result - - result["p25"] = round(float(rt.quantile(0.25)), 0) - result["p50"] = round(float(rt.quantile(0.50)), 0) - result["p75"] = round(float(rt.quantile(0.75)), 0) - result["p95"] = round(float(rt.quantile(0.95)), 0) - result["p99"] = round(float(rt.quantile(0.99)), 0) - - rt_bins = [(0, 200), (200, 500), (500, 1000), (1000, 2000), (2000, 999999)] - rt_labels = ["<200ms", "200-500ms", "500ms-1s", "1-2s", ">2s"] - rt_full = pd.to_numeric(df["response_time_ms"], errors="coerce").fillna(0) - result["distribution"] = { - lbl: int(((rt_full >= lo) & (rt_full < hi)).sum()) for (lo, hi), lbl in zip(rt_bins, rt_labels) - } - - for _, row in df.iterrows(): - u = row.get("url") - ms = pd.to_numeric(row.get("response_time_ms"), errors="coerce") - if pd.isna(u) or pd.isna(ms) or ms <= 2000: - continue - result["slow_pages"].append({"url": str(u).strip(), "response_time_ms": int(ms)}) - result["slow_pages"] = sorted(result["slow_pages"], key=lambda x: x["response_time_ms"], reverse=True)[:50] - return result - - -def _build_depth_distribution(df: pd.DataFrame) -> dict: - """Build crawl depth distribution.""" - result = {"by_depth": {}, "max_depth": 0, "avg_depth": 0} - if "depth" not in df.columns or df.empty: - return result - - depths = pd.to_numeric(df["depth"], errors="coerce").dropna().astype(int) - if depths.empty: - return result - - result["max_depth"] = int(depths.max()) - result["avg_depth"] = round(float(depths.mean()), 1) - counts = depths.value_counts().sort_index() - result["by_depth"] = {str(int(k)): int(v) for k, v in counts.items()} - return result - - -def _parse_page_analysis_cell(raw: object) -> dict[str, Any]: - if raw is None or (isinstance(raw, float) and pd.isna(raw)): - return {} - s = str(raw).strip() - if not s or s == "{}": - return {} - try: - o = json.loads(s) - return o if isinstance(o, dict) else {} - except json.JSONDecodeError: - return {} - - -def _build_outbound_link_domains( - df: pd.DataFrame, - start_url: str, - max_rows: int, -) -> list[dict[str, Any]]: - """Aggregate external hosts linked from crawled pages (outbound), not referring domains.""" - site_host = urlparse((start_url or "").strip()).netloc.lower() - host_pages: dict[str, set[str]] = {} - host_link_count: dict[str, int] = {} - for _, row in df.iterrows(): - st = str(row.get("status", "")).strip() - if st.startswith(("4", "5")): - continue - u = str(row.get("url") or "").strip().rstrip("/") - if not u: - continue - seen_on_page: set[str] = set() - pa = _parse_page_analysis_cell(row.get("page_analysis")) if "page_analysis" in df.columns else {} - for link in pa.get("external_links") or []: - if not isinstance(link, str): - continue - h = urlparse(link).netloc.lower() - if not h or h == site_host: - continue - host_pages.setdefault(h, set()).add(u) - host_link_count[h] = host_link_count.get(h, 0) + 1 - seen_on_page.add(link) - if "outlink_targets" in df.columns: - for link in parse_links_serialized(row.get("outlink_targets")): - h = urlparse(link).netloc.lower() - if not h or h == site_host: - continue - host_pages.setdefault(h, set()).add(u) - if link not in seen_on_page: - host_link_count[h] = host_link_count.get(h, 0) + 1 - seen_on_page.add(link) - rows: list[dict[str, Any]] = [] - for h in host_pages: - rows.append({ - "host": h, - "page_count": len(host_pages[h]), - "link_count": host_link_count.get(h, 0), - }) - rows.sort(key=lambda x: (-x["link_count"], -x["page_count"], x["host"])) - return rows[:max_rows] - - -def _build_url_fingerprints(df: pd.DataFrame) -> list[dict[str, Any]]: - """Stable fingerprints for comparing page content/structure between report runs (no raw HTML stored).""" - out: list[dict[str, Any]] = [] - for _, row in df.iterrows(): - u = str(row.get("url") or "").strip().rstrip("/") - if not u: - continue - title = str(row.get("title") or "") - meta = str(row.get("meta_description") or "") - h1 = str(row.get("h1") or "") - headings = str(row.get("heading_sequence") or "") - wc = int(pd.to_numeric(row.get("word_count"), errors="coerce") or 0) - cl = int(pd.to_numeric(row.get("content_length"), errors="coerce") or 0) - h1c = int(pd.to_numeric(row.get("h1_count"), errors="coerce") or 0) - sc = int(pd.to_numeric(row.get("script_count"), errors="coerce") or 0) - lc = int(pd.to_numeric(row.get("link_stylesheet_count"), errors="coerce") or 0) - # heading_sequence is structural (h1,h2,...) — keep it in structure fingerprint only. - raw_c = "|".join([title, meta, h1, str(wc), str(cl)]).encode("utf-8") - content_fp = hashlib.sha256(raw_c).hexdigest() - raw_s = "|".join([str(cl), str(sc), str(lc), str(h1c), headings]).encode("utf-8") - structure_fp = hashlib.sha256(raw_s).hexdigest() - out.append({ - "url": u, - "content_fingerprint": content_fp, - "structure_fingerprint": structure_fp, - }) - return out - - -def _build_hreflang_summary(df: pd.DataFrame) -> dict[str, Any]: - total = 0 - missing_lang = 0 - with_hreflang = 0 - for _, row in df.iterrows(): - st = str(row.get("status", "")).strip() - if not st.startswith("2"): - continue - total += 1 - pa = _parse_page_analysis_cell(row.get("page_analysis")) if "page_analysis" in df.columns else {} - if not (pa.get("html_lang") or "").strip(): - missing_lang += 1 - if pa.get("hreflang_alternates"): - with_hreflang += 1 - return { - "pages_200": total, - "pages_missing_html_lang": missing_lang, - "pages_with_hreflang_links": with_hreflang, - } - - -def _validate_report_url_counts(report_data: dict[str, Any], df_row_count: int) -> None: - """Ensure crawled URL counts are consistent across report payload fields.""" - links = report_data.get("links") or [] - summary = report_data.get("summary") or {} - scope = (report_data.get("report_meta") or {}).get("crawl_scope") or {} - link_count = len(links) if isinstance(links, list) else 0 - total_urls = int(summary.get("total_urls") or 0) - pages_crawled = int(scope.get("pages_crawled") or 0) - counts = {link_count, total_urls, pages_crawled, df_row_count} - if len(counts) > 1: - msg = ( - f"report count mismatch: links={link_count}, " - f"summary.total_urls={total_urls}, " - f"pages_crawled={pages_crawled}, df_rows={df_row_count}" - ) - print(f" WARNING: {msg}", flush=True) - report_data.setdefault("ml_errors", []).append(msg) - - -def _build_report_metadata( - df: pd.DataFrame, - config: Optional[dict[str, str]], - lighthouse_summary: Optional[dict[str, Any]], - google_data: Optional[dict[str, Any]], - keywords_data: Optional[dict[str, Any]], - ml_bundle: dict[str, Any], - run_id: Optional[int], - crawl_run_created_at: Optional[str], - gsc_links_data: Optional[dict[str, Any]] = None, -) -> dict[str, Any]: - """Provenance and crawl scope for agency-facing audits.""" - sources: list[str] = ["crawl"] - if lighthouse_summary: - sources.append("lighthouse") - if google_data: - if google_data.get("gsc") or google_data.get("gsc_summary"): - sources.append("search_console") - if google_data.get("ga4") or google_data.get("ga4_summary"): - sources.append("analytics") - if gsc_links_data and "search_console" not in sources: - sources.append("search_console") - llm_meta = ml_bundle.get("llm_meta") - if isinstance(llm_meta, dict) and llm_meta.get("model"): - sources.append("ai") - kw_rows = (keywords_data or {}).get("rows") or [] - has_gsc_kw = any( - (r.get("gsc_impressions") or r.get("gsc_clicks")) and r.get("source") in ("gsc", "site+gsc", None) - for r in kw_rows[:500] - if isinstance(r, dict) - ) - if kw_rows and not has_gsc_kw and "estimated" not in sources: - sources.append("estimated") - - max_pages_cfg = get_int(config or {}, "max_pages", 0) or 0 - pages_crawled = len(df) - blocked = 0 - if not df.empty and "status" in df.columns: - blocked = int((df["status"].astype(str) == "blocked_by_robots").sum()) - - render_mode = (str((config or {}).get("crawl_render_mode") or "static")).strip().lower() - js_concurrency = get_int(config or {}, "crawl_js_concurrency", 3) or 3 - static_html_only = render_mode == "static" - - crawl_scope: dict[str, Any] = { - "pages_crawled": pages_crawled, - "max_pages_configured": max_pages_cfg or pages_crawled, - "robots_blocked_count": blocked, - "static_html_only": static_html_only, - "render_mode": render_mode, - "js_concurrency": js_concurrency if not static_html_only else None, - "crawl_limited": bool(max_pages_cfg and pages_crawled >= max_pages_cfg), - } - if not df.empty and "fetch_method" in df.columns: - fm = df["fetch_method"].astype(str).str.strip().str.lower() - pages_static = int((fm == "static").sum()) - pages_rendered = int((fm == "rendered").sum()) - if render_mode == "auto" or pages_rendered > 0: - crawl_scope["pages_static"] = pages_static - crawl_scope["pages_rendered"] = pages_rendered - - from ..crawl.fetchers.browser_diagnostics import aggregate_browser_diagnostics_df - - browser_agg = aggregate_browser_diagnostics_df(df) - if browser_agg and (render_mode != "static" or browser_agg.get("total_console_errors", 0) > 0): - crawl_scope["browser_diagnostics"] = browser_agg - - meta: dict[str, Any] = { - "data_sources": sources, - "generated_at": datetime.now(timezone.utc).isoformat(), - "crawl_scope": crawl_scope, - } - if run_id is not None: - meta["crawl_run_id"] = run_id - if crawl_run_created_at: - meta["crawl_run_created_at"] = crawl_run_created_at - if google_data: - meta["google_fetched_at"] = google_data.get("fetched_at") - meta["google_date_range_days"] = google_data.get("date_range_days") - gsc = google_data.get("gsc") or {} - if isinstance(gsc, dict) and gsc.get("row_count") is not None: - meta["gsc_row_count"] = gsc.get("row_count") - if keywords_data: - meta["keywords_enriched_at"] = keywords_data.get("enriched_at") or keywords_data.get("fetched_at") - if gsc_links_data: - meta["gsc_links_imported_at"] = gsc_links_data.get("imported_at") - meta["gsc_links_referring_domains"] = len(gsc_links_data.get("top_linking_sites") or []) - sample_n = len(gsc_links_data.get("sample_links") or []) - latest_n = len(gsc_links_data.get("latest_links") or []) - meta["gsc_links_sample_count"] = sample_n + latest_n - if isinstance(llm_meta, dict): - meta["llm"] = llm_meta - logo_url = (str((config or {}).get("export_logo_url") or "")).strip() - if logo_url: - meta["export_logo_url"] = logo_url - return meta - - -def _build_keyword_opportunities(df: pd.DataFrame, config: dict[str, str] | None) -> dict[str, Any]: - if not get_bool(config or {}, "include_keyword_opportunities", True): - return {} - if "status" not in df.columns or df.empty: - return {"quick_wins": [], "high_value": [], "token_topic_clusters": []} - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] - if success_df.empty: - return {"quick_wins": [], "high_value": [], "token_topic_clusters": []} - candidates = extract_candidates_from_df(success_df) - if not candidates: - return {"quick_wins": [], "high_value": [], "token_topic_clusters": []} - corpus_size = len(success_df) - scored = score_keywords(candidates, corpus_size=corpus_size) - clusters = cluster_keywords(scored) - quick_wins = [s for s in scored if s.get("difficulty", 100) < 60][:10] - high_value = [s for s in scored if (s.get("volume") or 0) >= 0.5][:10] - if not high_value: - high_value = scored[:10] - return { - "quick_wins": quick_wins[:10], - "high_value": high_value[:10], - "token_topic_clusters": filter_topic_clusters(clusters)[:50], - } - - -def _build_image_inventory( - links: list[dict[str, Any]], - config: Optional[dict[str, str]], -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - from ..analysis.image_probe import collect_image_refs_from_links, probe_image_urls - - refs = collect_image_refs_from_links(links) - unoptimized_min_kb = get_int(config or {}, "image_unoptimized_min_kb", 200) or 200 - summary: dict[str, Any] = { - "probed": 0, - "failed": 0, - "total_bytes": 0, - "over_threshold_count": 0, - "unoptimized_min_kb": unoptimized_min_kb, - "inventory_available": False, - } - if not get_bool(config or {}, "probe_image_inventory", False): - return [], summary - - max_urls = get_int(config or {}, "max_image_probe_urls", 500) or 500 - concurrency = get_int(config or {}, "image_probe_concurrency", 6) or 6 - probe_timeout = get_int(config or {}, "image_probe_timeout", 8) or 8 - url_list = list(refs.keys())[:max_urls] - if not url_list: - return [], summary - - print(f" Probing up to {len(url_list)} image URL(s)...", flush=True) - probed = probe_image_urls( - url_list, - concurrency=concurrency, - timeout=probe_timeout, - ) - threshold_bytes = unoptimized_min_kb * 1024 - inventory: list[dict[str, Any]] = [] - for row in probed: - url = row.get("url") - meta = refs.get(str(url or ""), {"source_pages": set(), "kinds": set()}) - size = row.get("size_bytes") - entry = { - "url": url, - "status": row.get("status"), - "content_type": row.get("content_type"), - "size_bytes": size, - "error": row.get("error"), - "source_pages": sorted(meta.get("source_pages") or []), - "kinds": sorted(meta.get("kinds") or []), - } - inventory.append(entry) - summary["probed"] += 1 - if row.get("error") or row.get("status") is None: - summary["failed"] += 1 - if size is not None: - summary["total_bytes"] += int(size) - if int(size) >= threshold_bytes: - summary["over_threshold_count"] += 1 - summary["inventory_available"] = True - print(f" Image probe complete ({summary['probed']} URLs, {summary['failed']} failed).", flush=True) - return inventory, summary - +from .categories import build_categories +from .content_analytics import ( + _build_content_analytics, + _build_depth_distribution, + _build_image_inventory, + _build_keyword_opportunities, + _build_response_time_stats, + _build_social_coverage, + _build_tech_stack_summary, + _build_text_content_analysis, + _parse_top_keywords_items, +) +from .edges_report import build_edges_from_df +from .lighthouse_report import ( + _derive_expected_host, + _pick_lighthouse_summary, + build_lighthouse_by_url_for_report, + fetch_site_ssl_expires_iso, + filter_lighthouse_by_host, + lighthouse_for_url, +) +from .report_metadata import ( + _build_hreflang_summary, + _build_outbound_link_domains, + _build_report_metadata, + _build_url_fingerprints, + _parse_page_analysis_cell, + _validate_report_url_counts, +) +from .seo_summary import _compute_summary_seo_issues +from .site_level import _fetch_site_level + +# Backward-compatible re-exports for tests and external imports. +__all__ = [ + "run_simple_report", + "build_edges_from_df", + "build_lighthouse_by_url_for_report", + "fetch_site_ssl_expires_iso", + "filter_lighthouse_by_host", + "lighthouse_for_url", + "_fetch_site_level", + "_compute_summary_seo_issues", + "_build_content_analytics", + "_build_text_content_analysis", + "_build_image_inventory", + "_build_report_metadata", +] def run_simple_report( max_fetch_for_edges: int = 300, diff --git a/src/website_profiling/reporting/categories.py b/src/website_profiling/reporting/categories.py deleted file mode 100644 index 15014e7d..00000000 --- a/src/website_profiling/reporting/categories.py +++ /dev/null @@ -1,1053 +0,0 @@ -""" -Report categories for site audits: Technical SEO, Core Web Vitals, Performance, -Accessibility & markup, Links, Mobile SEO, Security, Content quality. -""" -import json -from typing import Any, Optional -from urllib.parse import urlparse - -import pandas as pd - -from .terminology import ( - CATEGORY_ACCESSIBILITY, - CATEGORY_CONTENT_QUALITY, - CATEGORY_CORE_WEB_VITALS, - CATEGORY_LINKS, - CATEGORY_MOBILE, - CATEGORY_PERFORMANCE, - CATEGORY_SECURITY, - CATEGORY_TECHNICAL_SEO, -) - -# Priority order for sorting -PRIORITY_ORDER = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3} - -# Thresholds -RESPONSE_TIME_SLOW_MS = 2000 -THIN_CONTENT_CHARS = 300 -TITLE_LEN_MIN = 30 -TITLE_LEN_MAX = 60 -META_DESC_LEN_MIN = 70 -META_DESC_LEN_MAX = 160 -REDIRECT_CHAIN_LONG = 2 - - -def _issue(message: str, url: Optional[str] = None, priority: str = "Medium", recommendation: str = "") -> dict: - return {"message": message, "url": url or "", "priority": priority, "recommendation": recommendation} - - -def _sort_issues(issues: list[dict]) -> list[dict]: - return sorted(issues, key=lambda x: PRIORITY_ORDER.get(x.get("priority", "Low"), 99)) - - -def _page_analysis_dict(row: pd.Series) -> dict: - """Parse page_analysis JSON cell from a crawl row.""" - import json - - raw = row.get("page_analysis") - if raw is None or (isinstance(raw, float) and pd.isna(raw)): - return {} - s = str(raw).strip() - if not s or s == "{}": - return {} - try: - o = json.loads(s) - return o if isinstance(o, dict) else {} - except json.JSONDecodeError: - return {} - - -def _score_deductions(max_score: int, deductions: list[tuple[int, bool]]) -> int: - """Return max(0, max_score - sum of deduction for each True).""" - total = sum(d for d, apply in deductions if apply) - return max(0, max_score - total) - - -def _hreflang_issues(success_df: pd.DataFrame) -> list[dict]: - """Hreflang cluster consistency (return tags, self-reference).""" - issues: list[dict] = [] - if "page_analysis" not in success_df.columns: - return issues - for _, row in success_df.iterrows(): - pa = _page_analysis_dict(row) - alts = pa.get("hreflang_alternates") or [] - if not alts: - continue - url = str(row.get("url") or "").strip() - langs = [str(a.get("hreflang") or a.get("lang") or "").strip().lower() for a in alts if isinstance(a, dict)] - hrefs = [str(a.get("href") or "").strip() for a in alts if isinstance(a, dict)] - if langs and len(set(langs)) < len(langs): - issues.append(_issue( - "Duplicate hreflang language codes on page.", - url=url, - priority="High", - recommendation="Each hreflang alternate should use a unique language/region code.", - )) - break - if url and hrefs and url.rstrip("/") not in [h.rstrip("/") for h in hrefs]: - issues.append(_issue( - "Hreflang cluster missing self-referencing alternate.", - url=url, - priority="Medium", - recommendation="Include a hreflang link pointing to this page URL.", - )) - break - return issues - - -def _schema_issues(success_df: pd.DataFrame) -> list[dict]: - issues: list[dict] = [] - invalid = 0 - for _, row in success_df.iterrows(): - pa = _page_analysis_dict(row) - schemas = pa.get("json_ld_types") or pa.get("schema_types") or [] - if isinstance(schemas, str): - schemas = [schemas] - url = str(row.get("url") or "").strip() - has_schema = str(row.get("has_schema", "")).lower() in ("true", "1", "yes") - if has_schema and not schemas: - invalid += 1 - if invalid == 1: - issues.append(_issue( - "Structured data present but could not parse JSON-LD @type.", - url=url, - priority="Low", - recommendation="Validate JSON-LD with Google Rich Results Test.", - )) - return issues - - -def _soft_404_issues(success_df: pd.DataFrame) -> list[dict]: - issues: list[dict] = [] - markers = ("not found", "404", "page not found", "doesn't exist", "does not exist") - for _, row in success_df.iterrows(): - title = str(row.get("title") or "").lower() - if any(m in title for m in markers): - url = str(row.get("url") or "").strip() - issues.append(_issue( - "Possible soft 404: page returns 200 but title suggests not found.", - url=url, - priority="High", - recommendation="Return 404 status or redirect to a relevant page.", - )) - if len(issues) >= 10: - break - return issues - - -def _broken_link_sources(edges: list[tuple[str, str]], broken_urls: set[str]) -> list[dict]: - """Issues listing which pages link to broken URLs.""" - issues: list[dict] = [] - if not broken_urls: - return issues - sources: dict[str, list[str]] = {} - for src, tgt in edges: - if tgt in broken_urls: - sources.setdefault(tgt, []).append(src) - for tgt, srcs in list(sources.items())[:15]: - sample = ", ".join(srcs[:3]) - more = f" (+{len(srcs) - 3} more)" if len(srcs) > 3 else "" - issues.append(_issue( - f"Broken URL linked from {len(srcs)} page(s): {sample}{more}", - url=tgt, - priority="High", - recommendation="Fix or remove links pointing to this URL.", - )) - return issues - - -def _indexation_coverage_issues( - df: pd.DataFrame, - indexation: dict | None, -) -> list[dict]: - """Sitemap vs crawl mismatches and noindex URLs listed in sitemap.""" - issues: list[dict] = [] - if not indexation: - return issues - lists = indexation.get("lists") if isinstance(indexation.get("lists"), dict) else {} - sitemap_only = lists.get("sitemap_only") or [] - for url in sitemap_only[:15]: - issues.append(_issue( - f"URL in sitemap but not crawled: {url}", - url=str(url), - priority="High", - recommendation="Verify the URL is linked internally, not blocked by robots, and within crawl scope.", - )) - sitemap_urls = indexation.get("sitemap_urls") or [] - if sitemap_urls and "noindex" in df.columns: - from ..integrations.google.normalize import normalize_url - - sitemap_norm = {normalize_url(u) for u in sitemap_urls} - success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df - for _, row in success.iterrows(): - url = str(row.get("url") or "").strip() - if not url: - continue - noindex = str(row.get("noindex") or "").lower() in ("true", "1", "yes") - if noindex and normalize_url(url) in sitemap_norm: - issues.append(_issue( - "Page has noindex but is listed in XML sitemap.", - url=url, - priority="Critical", - recommendation="Remove the URL from the sitemap or remove noindex if the page should be indexed.", - )) - break - return issues - - -def merge_indexation_issues(categories: list[dict], df: pd.DataFrame, indexation: dict | None) -> None: - """Append indexation coverage issues to the technical SEO category.""" - extra = _indexation_coverage_issues(df, indexation) - if not extra: - return - for cat in categories: - if cat.get("id") == "technical_seo": - cat["issues"] = _sort_issues((cat.get("issues") or []) + extra) - recs = {i["recommendation"] for i in cat["issues"] if i.get("recommendation")} - cat["recommendations"] = list(recs) - break - - -def merge_subdomain_issues(categories: list[dict], subdomains: dict | None) -> None: - """Append GSC subdomain gap summary to technical SEO.""" - if not subdomains or subdomains.get("disabled"): - return - hosts = subdomains.get("gsc_hosts_not_crawled") or [] - if not hosts: - return - preview = ", ".join(hosts[:5]) - suffix = f" (+{len(hosts) - 5} more)" if len(hosts) > 5 else "" - msg = f"GSC shows URLs on subdomain(s) not reached by crawl: {preview}{suffix}." - issue = _issue( - msg, - priority="Medium", - recommendation="Include these hosts in crawl scope or verify they are intentional separate properties.", - ) - for cat in categories: - if cat.get("id") == "technical_seo": - cat["issues"] = _sort_issues((cat.get("issues") or []) + [issue]) - recs = {i["recommendation"] for i in cat["issues"] if i.get("recommendation")} - cat["recommendations"] = list(recs) - break - - -def _orphan_hub_suggestions(edges: list[tuple[str, str]], orphan_urls: list[str]) -> list[dict]: - issues: list[dict] = [] - if not edges or not orphan_urls: - return issues - in_deg: dict[str, int] = {} - out_from: dict[str, list[str]] = {} - for src, tgt in edges: - in_deg[tgt] = in_deg.get(tgt, 0) + 1 - out_from.setdefault(src, []).append(tgt) - hubs = sorted(in_deg.keys(), key=lambda u: -in_deg.get(u, 0))[:5] - hub_label = hubs[0] if hubs else "" - for orphan in orphan_urls[:10]: - issues.append(_issue( - f"Orphan page (no inlinks). Consider linking from hub page: {hub_label}" if hub_label else "Orphan page (no inlinks).", - url=orphan, - priority="Medium", - recommendation="Add internal links from category or hub pages to this URL.", - )) - return issues - - -def category_technical_seo( - df: pd.DataFrame, - site_level: dict, -) -> dict: - """Technical SEO: robots, sitemap, canonical, duplicate content, noindex, schema.""" - issues = [] - deductions = [] - total = len(df) - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() - - if not site_level.get("robots_present", True): - issues.append(_issue( - "robots.txt is missing or unreachable.", - priority="High", - recommendation="Add a robots.txt at the site root to control crawler access.", - )) - deductions.append((15, True)) - if not site_level.get("sitemap_present", True): - issues.append(_issue( - "sitemap.xml (or sitemap index) is missing or unreachable.", - priority="High", - recommendation="Add a sitemap at /sitemap.xml or link it in robots.txt.", - )) - deductions.append((10, True)) - if site_level.get("sitemap_present") and not site_level.get("sitemap_valid", True): - issues.append(_issue( - "sitemap.xml could not be parsed as valid XML.", - priority="Medium", - recommendation="Ensure sitemap is valid XML and follows sitemaps.org format.", - )) - deductions.append((5, True)) - if site_level.get("ads_txt_present") is False: - issues.append(_issue( - "ads.txt is missing or unreachable.", - priority="Low", - recommendation="Add an ads.txt file at the site root if you run programmatic advertising.", - )) - if site_level.get("security_txt_present") is False: - issues.append(_issue( - "security.txt is missing or unreachable.", - priority="Low", - recommendation="Publish security.txt at /.well-known/security.txt with a Contact field for security reporting.", - )) - - # Canonical: missing or self-mismatch - if "canonical_url" in df.columns and len(success_df) > 0: - for _, row in success_df.iterrows(): - url = row.get("url") - canon = row.get("canonical_url") - if pd.isna(url): - continue - url = str(url).strip() - canon = "" if pd.isna(canon) else str(canon).strip() - if not canon: - issues.append(_issue("Missing canonical URL.", url=url, priority="Medium", recommendation="Add a canonical link tag pointing to the preferred URL.")) - break - missing_canon = success_df["canonical_url"].fillna("").astype(str).str.strip().eq("").sum() - if missing_canon > 0: - deductions.append((min(15, missing_canon * 2), True)) - # Self-canonical mismatch: canonical points to different URL - for _, row in success_df.iterrows(): - url = row.get("url") - canon = row.get("canonical_url") - if pd.isna(url) or pd.isna(canon) or not str(canon).strip(): - continue - url = str(url).rstrip("/") - canon = str(canon).strip().rstrip("/") - if url != canon: - issues.append(_issue(f"Canonical points to different URL: {canon}", url=url, priority="High", recommendation="Set canonical to this page URL or the preferred duplicate.")) - deductions.append((10, True)) - break - - # Noindex on important pages (CSV may store True/False as strings) - if "noindex" in df.columns and len(success_df) > 0: - noindex_ser = success_df["noindex"].astype(str).str.lower().isin(("true", "1", "yes")) - noindex_count = int(noindex_ser.sum()) - if noindex_count > 0: - issues.append(_issue( - f"{int(noindex_count)} page(s) have noindex.", - priority="High" if noindex_count > 5 else "Medium", - recommendation="Remove noindex from pages that should be indexed, or keep for intentional no-index pages.", - )) - deductions.append((min(15, noindex_count * 3), True)) - - # Duplicate content heuristic: same title + meta description - if "title" in df.columns and "meta_description" in df.columns and len(success_df) > 1: - key = success_df["title"].fillna("").astype(str) + "|" + success_df["meta_description"].fillna("").astype(str) - dupes = key.value_counts() - dupes = dupes[dupes > 1] - if len(dupes) > 0: - issues.append(_issue( - f"Possible duplicate content: {len(dupes)} group(s) of pages share same title and meta description.", - priority="Medium", - recommendation="Differentiate titles and meta descriptions, or use canonicals to designate the preferred URL.", - )) - deductions.append((10, True)) - - # Social meta tags - if "og_title" in df.columns and len(success_df) > 0: - og_present = (success_df["og_title"].fillna("").astype(str).str.strip() != "").sum() - og_pct = og_present / len(success_df) if len(success_df) > 0 else 1 - if og_pct < 0.5: - issues.append(_issue( - f"Open Graph tags missing on {int((1 - og_pct) * 100)}% of pages.", - priority="Medium", - recommendation="Add og:title, og:description, and og:image meta tags for social sharing.", - )) - deductions.append((5, True)) - - if "twitter_card" in df.columns and len(success_df) > 0: - tw_present = (success_df["twitter_card"].fillna("").astype(str).str.strip() != "").sum() - tw_pct = tw_present / len(success_df) if len(success_df) > 0 else 1 - if tw_pct < 0.2: - issues.append(_issue( - f"Twitter Card tags missing on {int((1 - tw_pct) * 100)}% of pages.", - priority="Low", - recommendation="Add twitter:card meta tags for better Twitter/X sharing previews.", - )) - deductions.append((3, True)) - - # Structured data - if "has_schema" in df.columns and len(success_df) > 0: - with_schema = int(success_df["has_schema"].astype(str).str.lower().isin(("true", "1", "yes")).sum()) - if with_schema == 0: - issues.append(_issue( - "No structured data (JSON-LD or microdata) detected.", - priority="Low", - recommendation="Add schema.org markup (e.g. Organization, Article) for rich results.", - )) - deductions.append((5, True)) - - # Internationalization: from page_analysis (re-crawl to populate) - if "page_analysis" in df.columns and len(success_df) > 0: - missing_lang = 0 - for _, row in success_df.iterrows(): - pa = _page_analysis_dict(row) - if not (pa.get("html_lang") or "").strip(): - missing_lang += 1 - if missing_lang > 0 and len(success_df) >= 3: - ratio = missing_lang / len(success_df) - if ratio > 0.1: - issues.append(_issue( - f"{missing_lang} page(s) missing (of {len(success_df)} OK responses).", - priority="Medium" if ratio > 0.5 else "Low", - recommendation="Add matching the primary language of each page.", - )) - deductions.append((min(10, max(2, missing_lang // 5)), True)) - - issues.extend(_hreflang_issues(success_df)) - issues.extend(_schema_issues(success_df)) - issues.extend(_soft_404_issues(success_df)) - - if "page_analysis" in df.columns and len(success_df) > 0: - from ..crawl.fetchers.browser_diagnostics import browser_summary_from_page_analysis - - pages_with_console = 0 - for _, row in success_df.iterrows(): - pa = _page_analysis_dict(row) - counts = browser_summary_from_page_analysis(pa) - url = str(row.get("url") or "").strip() - if counts["console_error_count"] > 0: - pages_with_console += 1 - if counts["page_error_count"] > 0 and url: - issues.append(_issue( - "Uncaught JavaScript error during browser render.", - url=url, - priority="High", - recommendation="Fix runtime JS errors that may break page functionality or SEO signals.", - )) - deductions.append((5, True)) - if pages_with_console > 0: - issues.append(_issue( - f"{pages_with_console} page(s) logged console errors during JavaScript rendering.", - priority="High" if pages_with_console > 3 else "Medium", - recommendation="Inspect browser console errors on affected URLs; fix broken scripts or API calls.", - )) - deductions.append((min(15, pages_with_console * 2), True)) - - score = _score_deductions(100, deductions) - return { - "id": "technical_seo", - "name": CATEGORY_TECHNICAL_SEO, - "score": score, - "issues": _sort_issues(issues), - "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), - } - - -def category_core_web_vitals() -> dict: - """Core Web Vitals: not measured; recommend Lighthouse.""" - return { - "id": "core_web_vitals", - "name": CATEGORY_CORE_WEB_VITALS, - "score": None, - "issues": [_issue( - "LCP, INP, and CLS are not measured by this crawl.", - priority="Medium", - recommendation="Run Lighthouse (PageSpeed Insights) from Run audit to measure Core Web Vitals.", - )], - "recommendations": ["Run Lighthouse from Run audit to measure LCP, INP, and CLS."], - } - - -def category_core_web_vitals_from_lighthouse( - lighthouse_summary: dict, - crux_summary: Optional[dict] = None, -) -> dict: - """Core Web Vitals from Lighthouse summary: score 0–100 from performance score, issues from top_failures.""" - issues = [] - recommendations = [] - perf_score = None - mm = lighthouse_summary.get("median_metrics") or {} - if isinstance(mm.get("performance_score"), (int, float)): - perf_score = max(0, min(100, int(round(mm["performance_score"] * 100)))) - for f in lighthouse_summary.get("top_failures") or []: - aid = f.get("id") or "" - help_text = (f.get("helpText") or "")[:200] - msg = f"{aid}: {help_text}" if aid else help_text or "Audit failed" - issues.append(_issue( - msg, - priority="High" if (f.get("score") or 0) < 0.5 else "Medium", - recommendation="See Performance (Core Web Vitals) in this audit, or re-run Lighthouse from Run audit.", - )) - if not issues and perf_score is not None and perf_score < 80: - recommendations.append("Improve Core Web Vitals (LCP, CLS, TBT) per Lighthouse recommendations.") - if crux_summary and crux_summary.get("ok"): - pw = crux_summary.get("pass") or {} - for metric, label, rec in ( - ("lcp", "LCP", "Improve largest contentful paint (field data)."), - ("inp", "INP", "Reduce interaction to next paint (field data)."), - ("cls", "CLS", "Reduce cumulative layout shift (field data)."), - ): - if pw.get(metric) is False: - issues.append(_issue( - f"CrUX field data: {label} does not pass Core Web Vitals threshold.", - priority="High", - recommendation=rec, - )) - return { - "id": "core_web_vitals", - "name": CATEGORY_CORE_WEB_VITALS, - "score": perf_score, - "issues": _sort_issues(issues), - "recommendations": recommendations or ["Core Web Vitals measured by Lighthouse; see median_metrics in lighthouse_summary.json."], - } - - -def category_performance(df: pd.DataFrame) -> dict: - """Performance: response time, JS/CSS size, images, lazy loading, caching.""" - issues = [] - deductions = [] - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() - if len(success_df) == 0: - return {"id": "performance", "name": CATEGORY_PERFORMANCE, "score": 0, "issues": [], "recommendations": []} - - if "response_time_ms" in success_df.columns: - rt = pd.to_numeric(success_df["response_time_ms"], errors="coerce").fillna(0) - slow = (rt > RESPONSE_TIME_SLOW_MS).sum() - if slow > 0: - issues.append(_issue( - f"{int(slow)} page(s) have server response time > {RESPONSE_TIME_SLOW_MS // 1000}s.", - priority="High" if slow > 5 else "Medium", - recommendation="Optimize server response time (TTFB): caching, CDN, or backend tuning.", - )) - deductions.append((min(20, int(slow) * 2), True)) - valid_rt = rt[rt > 0] - if len(valid_rt) > 5: - p95 = float(valid_rt.quantile(0.95)) - if p95 > 3000: - issues.append(_issue( - f"95th percentile response time is {int(p95)}ms (over 3s).", - priority="High", - recommendation="Investigate slowest pages; consider CDN, server-side caching, or database optimization.", - )) - deductions.append((10, True)) - - if "images_total" in success_df.columns: - total_imgs = success_df["images_total"].fillna(0).astype(int).sum() - if total_imgs > 0 and "img_without_lazy" in success_df.columns: - no_lazy = success_df["img_without_lazy"].fillna(0).astype(int).sum() - if no_lazy > total_imgs * 0.5: - issues.append(_issue( - "Many images without lazy loading.", - priority="Medium", - recommendation="Add loading='lazy' to off-screen images.", - )) - deductions.append((10, True)) - if total_imgs > 0 and "img_without_dimensions" in success_df.columns: - no_dims = success_df["img_without_dimensions"].fillna(0).astype(int).sum() - if no_dims > 0: - issues.append(_issue( - f"{int(no_dims)} image(s) without width/height (can cause CLS).", - priority="High", - recommendation="Set width and height attributes on img tags to avoid layout shift.", - )) - deductions.append((10, True)) - - if "cache_control" in success_df.columns: - cache = success_df["cache_control"].fillna("").astype(str) - no_cache = (cache.str.strip() == "").sum() - if no_cache > len(success_df) * 0.5: - issues.append(_issue( - "Many pages without Cache-Control header.", - priority="Medium", - recommendation="Set Cache-Control (and optionally ETag) for static and cacheable pages.", - )) - deductions.append((10, True)) - - if "script_count" in success_df.columns: - scripts = success_df["script_count"].fillna(0).astype(int) - if scripts.sum() > len(success_df) * 10: - issues.append(_issue( - "High number of script tags across pages.", - priority="Low", - recommendation="Consider bundling and code-splitting to reduce JS payload.", - )) - deductions.append((5, True)) - - score = _score_deductions(100, deductions) - return { - "id": "performance", - "name": CATEGORY_PERFORMANCE, - "score": score, - "issues": _sort_issues(issues), - "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), - } - - -def _parse_page_analysis_cell(raw: object) -> dict[str, Any]: - if isinstance(raw, dict): - return raw - if not raw or not isinstance(raw, str): - return {} - try: - parsed = json.loads(raw) - return parsed if isinstance(parsed, dict) else {} - except Exception: - return {} - - -def contrast_issues_from_sources( - df: pd.DataFrame, - lighthouse_by_url: Optional[dict[str, Any]] = None, -) -> list[dict]: - """Contrast issues from axe crawl data and per-URL Lighthouse failures.""" - issues: list[dict] = [] - seen_urls: set[str] = set() - - if df is not None and not df.empty and "page_analysis" in df.columns: - for _, row in df.iterrows(): - url = str(row.get("url") or "").strip() - if not url: - continue - pa = _parse_page_analysis_cell(row.get("page_analysis")) - axe = pa.get("axe_violations") - if not isinstance(axe, list): - continue - contrast_hits = [ - v for v in axe - if isinstance(v, dict) and "color-contrast" in str(v.get("id") or "") - ] - if not contrast_hits: - continue - seen_urls.add(url.rstrip("/")) - first = contrast_hits[0] - msg = str(first.get("description") or first.get("help") or "Color contrast violation") - issues.append(_issue( - f"axe: {msg}", - url=url, - priority="Medium", - recommendation=str( - first.get("help") - or "Fix text/background contrast to meet WCAG AA (axe-core)." - ), - )) - - lh_map = lighthouse_by_url or {} - for url, summary in lh_map.items(): - if not isinstance(summary, dict): - continue - u = str(url or summary.get("url") or "").strip().rstrip("/") - if not u or u in seen_urls: - continue - for fail in summary.get("top_failures") or []: - if not isinstance(fail, dict): - continue - if str(fail.get("id") or "") != "color-contrast": - continue - seen_urls.add(u) - help_text = str(fail.get("helpText") or "Low color contrast") - issues.append(_issue( - f"Lighthouse: {help_text}", - url=u, - priority="Medium", - recommendation="Increase contrast ratio between text and background to meet WCAG AA.", - )) - break - - return issues[:40] - - -def category_html_accessibility( - df: pd.DataFrame, - lighthouse_by_url: Optional[dict[str, Any]] = None, -) -> dict: - """HTML and Accessibility: semantic HTML, heading structure, alt, ARIA, contrast.""" - issues = [] - deductions = [] - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() - if len(success_df) == 0: - return {"id": "html_accessibility", "name": CATEGORY_ACCESSIBILITY, "score": 0, "issues": [], "recommendations": []} - - if "h1_count" in df.columns: - h1c = pd.to_numeric(success_df["h1_count"], errors="coerce").fillna(-1).astype(int) - zero_h1 = (h1c == 0).sum() - multi_h1 = (h1c > 1).sum() - if zero_h1 > 0: - issues.append(_issue( - f"{int(zero_h1)} page(s) missing H1.", - priority="High", - recommendation="Add exactly one H1 per page describing the main content.", - )) - deductions.append((min(20, int(zero_h1) * 3), True)) - if multi_h1 > 0: - issues.append(_issue( - f"{int(multi_h1)} page(s) have multiple H1s.", - priority="Medium", - recommendation="Use a single H1 per page; use H2–H6 for subsections.", - )) - deductions.append((min(10, int(multi_h1) * 2), True)) - - if "heading_sequence" in df.columns: - pages_with_skipped_heading = 0 - for _, row in success_df.iterrows(): - seq = row.get("heading_sequence") - if pd.isna(seq) or not str(seq).strip(): - continue - parts = [p.strip() for p in str(seq).split(",") if p.strip()] - if not parts: - continue - levels = [int(h[1]) for h in parts if len(h) == 2 and h[0] == "h" and h[1] in "123456"] - for i in range(1, len(levels)): - if levels[i] > levels[i - 1] + 1: - if pages_with_skipped_heading == 0: - issues.append(_issue( - "Skipped heading level (e.g. H1 then H3).", - url=str(row.get("url", "")), - priority="Medium", - recommendation="Use heading levels in order (H1, H2, H3) without skipping.", - )) - pages_with_skipped_heading += 1 - break - if pages_with_skipped_heading > 0: - deductions.append((min(15, pages_with_skipped_heading * 5), True)) - - if "images_total" in df.columns and "images_without_alt" in df.columns: - total = success_df["images_total"].fillna(0).astype(int).sum() - missing_alt = success_df["images_without_alt"].fillna(0).astype(int).sum() - if total > 0 and missing_alt > 0: - issues.append(_issue( - f"{int(missing_alt)} image(s) without alt (or aria-label).", - priority="High", - recommendation="Add meaningful alt text to all images; use alt='' for decorative images.", - )) - deductions.append((min(15, int(missing_alt) * 2), True)) - - if "word_count" in success_df.columns: - wc = pd.to_numeric(success_df["word_count"], errors="coerce").fillna(0).astype(int) - very_thin = int(((wc > 0) & (wc < 100)).sum()) - if very_thin > 0: - issues.append(_issue( - f"{very_thin} page(s) with very thin content (under 100 words).", - priority="High", - recommendation="Expand thin pages with meaningful content (aim for 300+ words).", - )) - deductions.append((min(15, very_thin * 3), True)) - - if "reading_level" in success_df.columns: - rl = pd.to_numeric(success_df["reading_level"], errors="coerce").fillna(0) - complex_pages = int((rl > 14).sum()) - if complex_pages > 0: - issues.append(_issue( - f"{complex_pages} page(s) have very complex content (reading level > 14).", - priority="Medium", - recommendation="Simplify language for broader audience accessibility (aim for grade 8-10).", - )) - deductions.append((min(10, complex_pages * 2), True)) - - contrast_issues = contrast_issues_from_sources(df, lighthouse_by_url) - if contrast_issues: - issues.extend(contrast_issues) - deductions.append((min(25, len(contrast_issues) * 4), True)) - else: - issues.append(_issue( - "Color contrast is not measured by this tool.", - priority="Low", - recommendation="Enable axe (browser crawl) or Lighthouse to check contrast.", - )) - - score = _score_deductions(100, deductions) - if len(success_df) > 0 and score == 0: - score = 5 - score = min(100, max(0, score)) - return { - "id": "html_accessibility", - "name": CATEGORY_ACCESSIBILITY, - "score": score, - "issues": _sort_issues(issues), - "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), - } - - -def category_link_health( - df: pd.DataFrame, - edges: list[tuple[str, str]], - issues_broken: list[dict], - issues_redirects: list[dict], -) -> dict: - """Link Health: broken links, redirect chains, internal linking.""" - issues = [] - deductions = [] - - for b in issues_broken[:30]: - status = str(b.get("status", "")) - priority = "Critical" if status.startswith("5") else "High" - issues.append(_issue( - f"Broken URL: {status}", - url=b.get("url", ""), - priority=priority, - recommendation="Fix or remove the link; return 200 or redirect to a valid URL.", - )) - broken_url_set = {str(b.get("url") or "").strip() for b in issues_broken if b.get("url")} - issues.extend(_broken_link_sources(edges, broken_url_set)) - if issues_broken: - deductions.append((min(30, len(issues_broken) * 2), True)) - - for r in issues_redirects[:20]: - issues.append(_issue( - f"Redirect: {r.get('status', '')} to {r.get('final_url', '')}", - url=r.get("url", ""), - priority="Medium", - recommendation="Prefer direct URLs or shorten redirect chains.", - )) - if issues_redirects: - deductions.append((min(15, len(issues_redirects)), True)) - - if "redirect_chain_length" in df.columns and len(df) > 0: - rcl = pd.to_numeric(df["redirect_chain_length"], errors="coerce").fillna(0).astype(int) - long_chains = (rcl >= REDIRECT_CHAIN_LONG).sum() - if long_chains > 0: - issues.append(_issue( - f"{int(long_chains)} URL(s) have redirect chains (2+ hops).", - priority="Medium", - recommendation="Consolidate redirects to a single hop where possible.", - )) - deductions.append((min(10, int(long_chains)), True)) - - if edges: - import networkx as nx - G = nx.DiGraph() - G.add_edges_from(edges) - in_deg = dict(G.in_degree()) - orphans = [n for n in G.nodes() if in_deg.get(n, 0) == 0] - if len(orphans) > len(G.nodes()) * 0.3: - issues.append(_issue( - f"Many pages have no internal links pointing to them ({len(orphans)}).", - priority="Low", - recommendation="Add internal links to important pages to improve crawlability and internal link equity.", - )) - deductions.append((5, True)) - issues.extend(_orphan_hub_suggestions(edges, orphans[:15])) - - score = _score_deductions(100, deductions) - return { - "id": "link_health", - "name": CATEGORY_LINKS, - "score": score, - "issues": _sort_issues(issues), - "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), - } - - -def category_mobile(df: pd.DataFrame) -> dict: - """Mobile: viewport, responsive heuristic.""" - issues = [] - deductions = [] - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() - if len(success_df) == 0: - return {"id": "mobile", "name": CATEGORY_MOBILE, "score": 0, "issues": [], "recommendations": []} - - if "viewport_present" in df.columns: - viewport_ok = success_df["viewport_present"].astype(str).str.lower().isin(("true", "1", "yes")) - no_viewport = int((~viewport_ok).sum()) - if no_viewport > 0: - issues.append(_issue( - f"{int(no_viewport)} page(s) missing viewport meta tag.", - priority="Critical", - recommendation="Add .", - )) - deductions.append((min(25, int(no_viewport) * 5), True)) - viewport_content = success_df["viewport_content"].fillna("").astype(str) - viewport_ok = success_df["viewport_present"].astype(str).str.lower().isin(("true", "1", "yes")) - invalid = (viewport_content.str.strip().eq("") | (~viewport_content.str.contains("width|device-width", case=False, na=False))) & viewport_ok - if invalid.sum() > 0: - issues.append(_issue( - "Some pages have viewport without width or device-width.", - priority="High", - recommendation="Use content='width=device-width, initial-scale=1' (or similar).", - )) - deductions.append((10, True)) - - score = _score_deductions(100, deductions) - return { - "id": "mobile", - "name": CATEGORY_MOBILE, - "score": score, - "issues": _sort_issues(issues), - "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), - } - - -def category_security( - df: pd.DataFrame, - site_level: dict, - start_url: str, - security_findings: Optional[list[dict]] = None, -) -> dict: - """Security: HTTPS, security headers, mixed content, and optional vulnerability scan findings.""" - issues = [] - deductions = [] - parsed = urlparse(start_url) - if parsed.scheme and parsed.scheme.lower() != "https": - issues.append(_issue( - "Site is not using HTTPS.", - url=start_url, - priority="Critical", - recommendation="Serve the site over HTTPS and redirect HTTP to HTTPS.", - )) - deductions.append((30, True)) - - if "final_url" in df.columns and len(df) > 0: - final_urls = df["final_url"].fillna("").astype(str) - http_finals = final_urls.str.strip().str.lower().str.startswith("http://") - if http_finals.sum() > 0: - issues.append(_issue( - f"{int(http_finals.sum())} URL(s) resolve to HTTP.", - priority="Critical", - recommendation="Ensure all pages redirect to HTTPS.", - )) - deductions.append((20, True)) - - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() - if len(success_df) > 0: - # Security headers: sample from first row or aggregate (optional columns) - missing_hsts = (success_df["strict_transport_security"].fillna("").astype(str).str.strip() == "").sum() if "strict_transport_security" in success_df.columns else len(success_df) - missing_xcto = (success_df["x_content_type_options"].fillna("").astype(str).str.strip() == "").sum() if "x_content_type_options" in success_df.columns else len(success_df) - missing_xfo = (success_df["x_frame_options"].fillna("").astype(str).str.strip() == "").sum() if "x_frame_options" in success_df.columns else len(success_df) - if missing_hsts >= len(success_df) * 0.5: - issues.append(_issue( - "Strict-Transport-Security header not set.", - priority="High", - recommendation="Add Strict-Transport-Security to enforce HTTPS.", - )) - deductions.append((15, True)) - if missing_xcto >= len(success_df) * 0.5: - issues.append(_issue( - "X-Content-Type-Options header not set.", - priority="Medium", - recommendation="Add X-Content-Type-Options: nosniff.", - )) - deductions.append((5, True)) - if missing_xfo >= len(success_df) * 0.5: - issues.append(_issue( - "X-Frame-Options header not set.", - priority="Medium", - recommendation="Add X-Frame-Options: DENY or SAMEORIGIN.", - )) - deductions.append((5, True)) - - if "mixed_content_count" in success_df.columns: - mixed = success_df["mixed_content_count"].fillna(0).astype(int).sum() - scheme = (parsed.scheme or "").lower() - if mixed > 0 and scheme == "https": - issues.append(_issue( - f"Mixed content: {int(mixed)} HTTP resource(s) on HTTPS pages.", - priority="High", - recommendation="Load all resources over HTTPS to avoid mixed content.", - )) - deductions.append((15, True)) - - # Merge vulnerability scan findings (same format as issues: message, url, priority, recommendation) - if security_findings: - for f in security_findings: - severity = f.get("severity", "Medium") - issues.append(_issue( - f.get("message", ""), - url=f.get("url", ""), - priority=severity, - recommendation=f.get("recommendation", ""), - )) - # Deduct by severity: Critical 15, High 10, Medium 5, Low 2 - ded = {"Critical": 15, "High": 10, "Medium": 5, "Low": 2}.get(severity, 2) - deductions.append((min(ded, 15), True)) - - score = _score_deductions(100, deductions) - return { - "id": "security", - "name": CATEGORY_SECURITY, - "score": score, - "issues": _sort_issues(issues), - "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), - } - - -def category_intelligence(ml_bundle: Optional[dict] = None) -> dict: - """Content quality: duplicate clusters and language mix from crawl analysis and optional AI insights.""" - issues: list[dict] = [] - deductions: list[tuple[int, bool]] = [] - ml_bundle = ml_bundle or {} - - dups = ml_bundle.get("content_duplicates") or [] - if dups: - big = [g for g in dups if (g.get("member_count") or len(g.get("member_urls") or [])) >= 3] - if big: - issues.append(_issue( - f"Near-duplicate content: {len(big)} group(s) with 3+ URLs.", - priority="High", - recommendation="Consolidate or canonicalize duplicate pages; differentiate thin similar URLs.", - )) - deductions.append((min(20, 5 + len(big)), True)) - elif dups: - issues.append(_issue( - f"Possible duplicate content: {len(dups)} pair/group(s) detected.", - priority="Medium", - recommendation="Review clusters and add canonicals or noindex where appropriate.", - )) - deductions.append((8, True)) - - lang = ml_bundle.get("language_summary") or {} - if lang.get("mixed_site") and (lang.get("detected_pages") or 0) >= 10: - counts = lang.get("counts") or {} - top = sorted(counts.items(), key=lambda x: -x[1])[:3] - desc = ", ".join(f"{k}:{v}" for k, v in top) if top else "multiple" - issues.append(_issue( - f"Mixed languages detected across pages ({desc}).", - priority="Medium", - recommendation="Ensure hreflang and localized URLs match user intent; split sitemaps if needed.", - )) - deductions.append((5, True)) - - score = _score_deductions(100, deductions) - return { - "id": "intelligence", - "name": CATEGORY_CONTENT_QUALITY, - "score": score, - "issues": _sort_issues(issues), - "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), - } - - -def build_categories( - df: pd.DataFrame, - edges: list[tuple[str, str]], - summary_seo: dict, - site_level: dict, - start_url: str, - security_findings: Optional[list[dict]] = None, - lighthouse_summary: Optional[dict] = None, - ml_bundle: Optional[dict] = None, - crux_summary: Optional[dict] = None, - lighthouse_by_url: Optional[dict[str, Any]] = None, -) -> list[dict]: - """ - Build all category dicts with score, issues (with priority and recommendation), and recommendations. - site_level should have: robots_present, sitemap_present, sitemap_valid (all optional). - summary_seo should have: issues["broken"], issues["redirects"]. - security_findings: optional list from security scanner (finding_type, severity, url, message, recommendation). - lighthouse_summary: optional dict from lighthouse_runner (median_metrics, top_failures); when set, Core Web Vitals uses real data. - ml_bundle: optional dict from analysis + AI insights (duplicates, language_summary, etc.) for Content quality category. - """ - issues_broken = summary_seo.get("issues", {}).get("broken", []) - issues_redirects = summary_seo.get("issues", {}).get("redirects", []) - - cwv = ( - category_core_web_vitals_from_lighthouse(lighthouse_summary, crux_summary) - if lighthouse_summary - else category_core_web_vitals() - ) - categories = [ - category_technical_seo(df, site_level), - cwv, - category_performance(df), - category_html_accessibility(df, lighthouse_by_url=lighthouse_by_url), - category_link_health(df, edges, issues_broken, issues_redirects), - category_mobile(df), - category_security(df, site_level, start_url or "", security_findings=security_findings), - category_intelligence(ml_bundle), - ] - return categories diff --git a/src/website_profiling/reporting/categories/__init__.py b/src/website_profiling/reporting/categories/__init__.py new file mode 100644 index 00000000..e9921746 --- /dev/null +++ b/src/website_profiling/reporting/categories/__init__.py @@ -0,0 +1,113 @@ +"""Report categories for site audits.""" +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd + +from .accessibility import category_html_accessibility, contrast_issues_from_sources +from .intelligence import category_intelligence +from .link_health import category_link_health +from .mobile import category_mobile +from .performance import ( + category_core_web_vitals, + category_core_web_vitals_from_lighthouse, + category_performance, +) +from .security import category_security +from .technical_seo import category_technical_seo +from ._helpers import ( + META_DESC_LEN_MAX, + META_DESC_LEN_MIN, + PRIORITY_ORDER, + REDIRECT_CHAIN_LONG, + RESPONSE_TIME_SLOW_MS, + THIN_CONTENT_CHARS, + TITLE_LEN_MAX, + TITLE_LEN_MIN, + _broken_link_sources, + _hreflang_issues, + _indexation_coverage_issues, + _issue, + _orphan_hub_suggestions, + _page_analysis_dict, + _schema_issues, + _score_deductions, + _soft_404_issues, + _sort_issues, + merge_indexation_issues, + merge_subdomain_issues, +) + +__all__ = [ + "build_categories", + "merge_indexation_issues", + "merge_subdomain_issues", + "category_technical_seo", + "category_core_web_vitals", + "category_core_web_vitals_from_lighthouse", + "category_performance", + "category_html_accessibility", + "contrast_issues_from_sources", + "category_link_health", + "category_mobile", + "category_security", + "category_intelligence", + "_issue", + "_sort_issues", + "_page_analysis_dict", + "_broken_link_sources", + "_hreflang_issues", + "_schema_issues", + "_soft_404_issues", + "_indexation_coverage_issues", + "_orphan_hub_suggestions", + "REDIRECT_CHAIN_LONG", + "PRIORITY_ORDER", + "RESPONSE_TIME_SLOW_MS", + "THIN_CONTENT_CHARS", + "TITLE_LEN_MIN", + "TITLE_LEN_MAX", + "META_DESC_LEN_MIN", + "META_DESC_LEN_MAX", +] + +def build_categories( + df: pd.DataFrame, + edges: list[tuple[str, str]], + summary_seo: dict, + site_level: dict, + start_url: str, + security_findings: Optional[list[dict]] = None, + lighthouse_summary: Optional[dict] = None, + ml_bundle: Optional[dict] = None, + crux_summary: Optional[dict] = None, + lighthouse_by_url: Optional[dict[str, Any]] = None, +) -> list[dict]: + """ + Build all category dicts with score, issues (with priority and recommendation), and recommendations. + site_level should have: robots_present, sitemap_present, sitemap_valid (all optional). + summary_seo should have: issues["broken"], issues["redirects"]. + security_findings: optional list from security scanner (finding_type, severity, url, message, recommendation). + lighthouse_summary: optional dict from lighthouse_runner (median_metrics, top_failures); when set, Core Web Vitals uses real data. + ml_bundle: optional dict from analysis + AI insights (duplicates, language_summary, etc.) for Content quality category. + """ + issues_broken = summary_seo.get("issues", {}).get("broken", []) + issues_redirects = summary_seo.get("issues", {}).get("redirects", []) + + cwv = ( + category_core_web_vitals_from_lighthouse(lighthouse_summary, crux_summary) + if lighthouse_summary + else category_core_web_vitals() + ) + categories = [ + category_technical_seo(df, site_level), + cwv, + category_performance(df), + category_html_accessibility(df, lighthouse_by_url=lighthouse_by_url), + category_link_health(df, edges, issues_broken, issues_redirects), + category_mobile(df), + category_security(df, site_level, start_url or "", security_findings=security_findings), + category_intelligence(ml_bundle), + ] + return categories diff --git a/src/website_profiling/reporting/categories/_helpers.py b/src/website_profiling/reporting/categories/_helpers.py new file mode 100644 index 00000000..8b5d9c87 --- /dev/null +++ b/src/website_profiling/reporting/categories/_helpers.py @@ -0,0 +1,252 @@ +"""Shared helpers for report category builders.""" +from __future__ import annotations + +import json +from typing import Any, Optional + +import pandas as pd + +# Priority order for sorting +PRIORITY_ORDER = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3} + +# Thresholds +RESPONSE_TIME_SLOW_MS = 2000 +THIN_CONTENT_CHARS = 300 +TITLE_LEN_MIN = 30 +TITLE_LEN_MAX = 60 +META_DESC_LEN_MIN = 70 +META_DESC_LEN_MAX = 160 +REDIRECT_CHAIN_LONG = 2 + + +def _issue(message: str, url: Optional[str] = None, priority: str = "Medium", recommendation: str = "") -> dict: + return {"message": message, "url": url or "", "priority": priority, "recommendation": recommendation} + + +def _sort_issues(issues: list[dict]) -> list[dict]: + return sorted(issues, key=lambda x: PRIORITY_ORDER.get(x.get("priority", "Low"), 99)) + + +def _page_analysis_dict(row: pd.Series) -> dict: + """Parse page_analysis JSON cell from a crawl row.""" + import json + + raw = row.get("page_analysis") + if raw is None or (isinstance(raw, float) and pd.isna(raw)): + return {} + s = str(raw).strip() + if not s or s == "{}": + return {} + try: + o = json.loads(s) + return o if isinstance(o, dict) else {} + except json.JSONDecodeError: + return {} + + +def _score_deductions(max_score: int, deductions: list[tuple[int, bool]]) -> int: + """Return max(0, max_score - sum of deduction for each True).""" + total = sum(d for d, apply in deductions if apply) + return max(0, max_score - total) + + +def _hreflang_issues(success_df: pd.DataFrame) -> list[dict]: + """Hreflang cluster consistency (return tags, self-reference).""" + issues: list[dict] = [] + if "page_analysis" not in success_df.columns: + return issues + for _, row in success_df.iterrows(): + pa = _page_analysis_dict(row) + alts = pa.get("hreflang_alternates") or [] + if not alts: + continue + url = str(row.get("url") or "").strip() + langs = [str(a.get("hreflang") or a.get("lang") or "").strip().lower() for a in alts if isinstance(a, dict)] + hrefs = [str(a.get("href") or "").strip() for a in alts if isinstance(a, dict)] + if langs and len(set(langs)) < len(langs): + issues.append(_issue( + "Duplicate hreflang language codes on page.", + url=url, + priority="High", + recommendation="Each hreflang alternate should use a unique language/region code.", + )) + break + if url and hrefs and url.rstrip("/") not in [h.rstrip("/") for h in hrefs]: + issues.append(_issue( + "Hreflang cluster missing self-referencing alternate.", + url=url, + priority="Medium", + recommendation="Include a hreflang link pointing to this page URL.", + )) + break + return issues + + +def _schema_issues(success_df: pd.DataFrame) -> list[dict]: + issues: list[dict] = [] + invalid = 0 + for _, row in success_df.iterrows(): + pa = _page_analysis_dict(row) + schemas = pa.get("json_ld_types") or pa.get("schema_types") or [] + if isinstance(schemas, str): + schemas = [schemas] + url = str(row.get("url") or "").strip() + has_schema = str(row.get("has_schema", "")).lower() in ("true", "1", "yes") + if has_schema and not schemas: + invalid += 1 + if invalid == 1: + issues.append(_issue( + "Structured data present but could not parse JSON-LD @type.", + url=url, + priority="Low", + recommendation="Validate JSON-LD with Google Rich Results Test.", + )) + return issues + + +def _soft_404_issues(success_df: pd.DataFrame) -> list[dict]: + issues: list[dict] = [] + markers = ("not found", "404", "page not found", "doesn't exist", "does not exist") + for _, row in success_df.iterrows(): + title = str(row.get("title") or "").lower() + if any(m in title for m in markers): + url = str(row.get("url") or "").strip() + issues.append(_issue( + "Possible soft 404: page returns 200 but title suggests not found.", + url=url, + priority="High", + recommendation="Return 404 status or redirect to a relevant page.", + )) + if len(issues) >= 10: + break + return issues + + +def _broken_link_sources(edges: list[tuple[str, str]], broken_urls: set[str]) -> list[dict]: + """Issues listing which pages link to broken URLs.""" + issues: list[dict] = [] + if not broken_urls: + return issues + sources: dict[str, list[str]] = {} + for src, tgt in edges: + if tgt in broken_urls: + sources.setdefault(tgt, []).append(src) + for tgt, srcs in list(sources.items())[:15]: + sample = ", ".join(srcs[:3]) + more = f" (+{len(srcs) - 3} more)" if len(srcs) > 3 else "" + issues.append(_issue( + f"Broken URL linked from {len(srcs)} page(s): {sample}{more}", + url=tgt, + priority="High", + recommendation="Fix or remove links pointing to this URL.", + )) + return issues + + +def _indexation_coverage_issues( + df: pd.DataFrame, + indexation: dict | None, +) -> list[dict]: + """Sitemap vs crawl mismatches and noindex URLs listed in sitemap.""" + issues: list[dict] = [] + if not indexation: + return issues + lists = indexation.get("lists") if isinstance(indexation.get("lists"), dict) else {} + sitemap_only = lists.get("sitemap_only") or [] + for url in sitemap_only[:15]: + issues.append(_issue( + f"URL in sitemap but not crawled: {url}", + url=str(url), + priority="High", + recommendation="Verify the URL is linked internally, not blocked by robots, and within crawl scope.", + )) + sitemap_urls = indexation.get("sitemap_urls") or [] + if sitemap_urls and "noindex" in df.columns: + from ...integrations.google.normalize import normalize_url + + sitemap_norm = {normalize_url(u) for u in sitemap_urls} + success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + for _, row in success.iterrows(): + url = str(row.get("url") or "").strip() + if not url: + continue + noindex = str(row.get("noindex") or "").lower() in ("true", "1", "yes") + if noindex and normalize_url(url) in sitemap_norm: + issues.append(_issue( + "Page has noindex but is listed in XML sitemap.", + url=url, + priority="Critical", + recommendation="Remove the URL from the sitemap or remove noindex if the page should be indexed.", + )) + break + return issues + + +def merge_indexation_issues(categories: list[dict], df: pd.DataFrame, indexation: dict | None) -> None: + """Append indexation coverage issues to the technical SEO category.""" + extra = _indexation_coverage_issues(df, indexation) + if not extra: + return + for cat in categories: + if cat.get("id") == "technical_seo": + cat["issues"] = _sort_issues((cat.get("issues") or []) + extra) + recs = {i["recommendation"] for i in cat["issues"] if i.get("recommendation")} + cat["recommendations"] = list(recs) + break + + +def merge_subdomain_issues(categories: list[dict], subdomains: dict | None) -> None: + """Append GSC subdomain gap summary to technical SEO.""" + if not subdomains or subdomains.get("disabled"): + return + hosts = subdomains.get("gsc_hosts_not_crawled") or [] + if not hosts: + return + preview = ", ".join(hosts[:5]) + suffix = f" (+{len(hosts) - 5} more)" if len(hosts) > 5 else "" + msg = f"GSC shows URLs on subdomain(s) not reached by crawl: {preview}{suffix}." + issue = _issue( + msg, + priority="Medium", + recommendation="Include these hosts in crawl scope or verify they are intentional separate properties.", + ) + for cat in categories: + if cat.get("id") == "technical_seo": + cat["issues"] = _sort_issues((cat.get("issues") or []) + [issue]) + recs = {i["recommendation"] for i in cat["issues"] if i.get("recommendation")} + cat["recommendations"] = list(recs) + break + + +def _orphan_hub_suggestions(edges: list[tuple[str, str]], orphan_urls: list[str]) -> list[dict]: + issues: list[dict] = [] + if not edges or not orphan_urls: + return issues + in_deg: dict[str, int] = {} + out_from: dict[str, list[str]] = {} + for src, tgt in edges: + in_deg[tgt] = in_deg.get(tgt, 0) + 1 + out_from.setdefault(src, []).append(tgt) + hubs = sorted(in_deg.keys(), key=lambda u: -in_deg.get(u, 0))[:5] + hub_label = hubs[0] if hubs else "" + for orphan in orphan_urls[:10]: + issues.append(_issue( + f"Orphan page (no inlinks). Consider linking from hub page: {hub_label}" if hub_label else "Orphan page (no inlinks).", + url=orphan, + priority="Medium", + recommendation="Add internal links from category or hub pages to this URL.", + )) + return issues + + +def _parse_page_analysis_cell(raw: object) -> dict[str, Any]: + if isinstance(raw, dict): + return raw + if not raw or not isinstance(raw, str): + return {} + try: + parsed = json.loads(raw) + return parsed if isinstance(parsed, dict) else {} + except Exception: + return {} + diff --git a/src/website_profiling/reporting/categories/accessibility.py b/src/website_profiling/reporting/categories/accessibility.py new file mode 100644 index 00000000..695c3022 --- /dev/null +++ b/src/website_profiling/reporting/categories/accessibility.py @@ -0,0 +1,188 @@ +"""Report category: accessibility.""" +from __future__ import annotations + +import json +from typing import Any, Optional + +import pandas as pd + +from ..terminology import CATEGORY_ACCESSIBILITY +from ._helpers import ( + _issue, + _page_analysis_dict, + _parse_page_analysis_cell, + _score_deductions, + _sort_issues, +) + +def contrast_issues_from_sources( + df: pd.DataFrame, + lighthouse_by_url: Optional[dict[str, Any]] = None, +) -> list[dict]: + """Contrast issues from axe crawl data and per-URL Lighthouse failures.""" + issues: list[dict] = [] + seen_urls: set[str] = set() + + if df is not None and not df.empty and "page_analysis" in df.columns: + for _, row in df.iterrows(): + url = str(row.get("url") or "").strip() + if not url: + continue + pa = _parse_page_analysis_cell(row.get("page_analysis")) + axe = pa.get("axe_violations") + if not isinstance(axe, list): + continue + contrast_hits = [ + v for v in axe + if isinstance(v, dict) and "color-contrast" in str(v.get("id") or "") + ] + if not contrast_hits: + continue + seen_urls.add(url.rstrip("/")) + first = contrast_hits[0] + msg = str(first.get("description") or first.get("help") or "Color contrast violation") + issues.append(_issue( + f"axe: {msg}", + url=url, + priority="Medium", + recommendation=str( + first.get("help") + or "Fix text/background contrast to meet WCAG AA (axe-core)." + ), + )) + + lh_map = lighthouse_by_url or {} + for url, summary in lh_map.items(): + if not isinstance(summary, dict): + continue + u = str(url or summary.get("url") or "").strip().rstrip("/") + if not u or u in seen_urls: + continue + for fail in summary.get("top_failures") or []: + if not isinstance(fail, dict): + continue + if str(fail.get("id") or "") != "color-contrast": + continue + seen_urls.add(u) + help_text = str(fail.get("helpText") or "Low color contrast") + issues.append(_issue( + f"Lighthouse: {help_text}", + url=u, + priority="Medium", + recommendation="Increase contrast ratio between text and background to meet WCAG AA.", + )) + break + + return issues[:40] + + +def category_html_accessibility( + df: pd.DataFrame, + lighthouse_by_url: Optional[dict[str, Any]] = None, +) -> dict: + """HTML and Accessibility: semantic HTML, heading structure, alt, ARIA, contrast.""" + issues = [] + deductions = [] + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() + if len(success_df) == 0: + return {"id": "html_accessibility", "name": CATEGORY_ACCESSIBILITY, "score": 0, "issues": [], "recommendations": []} + + if "h1_count" in df.columns: + h1c = pd.to_numeric(success_df["h1_count"], errors="coerce").fillna(-1).astype(int) + zero_h1 = (h1c == 0).sum() + multi_h1 = (h1c > 1).sum() + if zero_h1 > 0: + issues.append(_issue( + f"{int(zero_h1)} page(s) missing H1.", + priority="High", + recommendation="Add exactly one H1 per page describing the main content.", + )) + deductions.append((min(20, int(zero_h1) * 3), True)) + if multi_h1 > 0: + issues.append(_issue( + f"{int(multi_h1)} page(s) have multiple H1s.", + priority="Medium", + recommendation="Use a single H1 per page; use H2–H6 for subsections.", + )) + deductions.append((min(10, int(multi_h1) * 2), True)) + + if "heading_sequence" in df.columns: + pages_with_skipped_heading = 0 + for _, row in success_df.iterrows(): + seq = row.get("heading_sequence") + if pd.isna(seq) or not str(seq).strip(): + continue + parts = [p.strip() for p in str(seq).split(",") if p.strip()] + if not parts: + continue + levels = [int(h[1]) for h in parts if len(h) == 2 and h[0] == "h" and h[1] in "123456"] + for i in range(1, len(levels)): + if levels[i] > levels[i - 1] + 1: + if pages_with_skipped_heading == 0: + issues.append(_issue( + "Skipped heading level (e.g. H1 then H3).", + url=str(row.get("url", "")), + priority="Medium", + recommendation="Use heading levels in order (H1, H2, H3) without skipping.", + )) + pages_with_skipped_heading += 1 + break + if pages_with_skipped_heading > 0: + deductions.append((min(15, pages_with_skipped_heading * 5), True)) + + if "images_total" in df.columns and "images_without_alt" in df.columns: + total = success_df["images_total"].fillna(0).astype(int).sum() + missing_alt = success_df["images_without_alt"].fillna(0).astype(int).sum() + if total > 0 and missing_alt > 0: + issues.append(_issue( + f"{int(missing_alt)} image(s) without alt (or aria-label).", + priority="High", + recommendation="Add meaningful alt text to all images; use alt='' for decorative images.", + )) + deductions.append((min(15, int(missing_alt) * 2), True)) + + if "word_count" in success_df.columns: + wc = pd.to_numeric(success_df["word_count"], errors="coerce").fillna(0).astype(int) + very_thin = int(((wc > 0) & (wc < 100)).sum()) + if very_thin > 0: + issues.append(_issue( + f"{very_thin} page(s) with very thin content (under 100 words).", + priority="High", + recommendation="Expand thin pages with meaningful content (aim for 300+ words).", + )) + deductions.append((min(15, very_thin * 3), True)) + + if "reading_level" in success_df.columns: + rl = pd.to_numeric(success_df["reading_level"], errors="coerce").fillna(0) + complex_pages = int((rl > 14).sum()) + if complex_pages > 0: + issues.append(_issue( + f"{complex_pages} page(s) have very complex content (reading level > 14).", + priority="Medium", + recommendation="Simplify language for broader audience accessibility (aim for grade 8-10).", + )) + deductions.append((min(10, complex_pages * 2), True)) + + contrast_issues = contrast_issues_from_sources(df, lighthouse_by_url) + if contrast_issues: + issues.extend(contrast_issues) + deductions.append((min(25, len(contrast_issues) * 4), True)) + else: + issues.append(_issue( + "Color contrast is not measured by this tool.", + priority="Low", + recommendation="Enable axe (browser crawl) or Lighthouse to check contrast.", + )) + + score = _score_deductions(100, deductions) + if len(success_df) > 0 and score == 0: + score = 5 + score = min(100, max(0, score)) + return { + "id": "html_accessibility", + "name": CATEGORY_ACCESSIBILITY, + "score": score, + "issues": _sort_issues(issues), + "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), + } + diff --git a/src/website_profiling/reporting/categories/intelligence.py b/src/website_profiling/reporting/categories/intelligence.py new file mode 100644 index 00000000..4abb1b7d --- /dev/null +++ b/src/website_profiling/reporting/categories/intelligence.py @@ -0,0 +1,68 @@ +"""Report category: intelligence.""" +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd + +from ._helpers import ( + PRIORITY_ORDER, + _broken_link_sources, + _hreflang_issues, + _indexation_coverage_issues, + _issue, + _orphan_hub_suggestions, + _page_analysis_dict, + _schema_issues, + _score_deductions, + _soft_404_issues, + _sort_issues, +) +from ..terminology import ( + CATEGORY_CONTENT_QUALITY, +) + +def category_intelligence(ml_bundle: Optional[dict] = None) -> dict: + """Content quality: duplicate clusters and language mix from crawl analysis and optional AI insights.""" + issues: list[dict] = [] + deductions: list[tuple[int, bool]] = [] + ml_bundle = ml_bundle or {} + + dups = ml_bundle.get("content_duplicates") or [] + if dups: + big = [g for g in dups if (g.get("member_count") or len(g.get("member_urls") or [])) >= 3] + if big: + issues.append(_issue( + f"Near-duplicate content: {len(big)} group(s) with 3+ URLs.", + priority="High", + recommendation="Consolidate or canonicalize duplicate pages; differentiate thin similar URLs.", + )) + deductions.append((min(20, 5 + len(big)), True)) + elif dups: + issues.append(_issue( + f"Possible duplicate content: {len(dups)} pair/group(s) detected.", + priority="Medium", + recommendation="Review clusters and add canonicals or noindex where appropriate.", + )) + deductions.append((8, True)) + + lang = ml_bundle.get("language_summary") or {} + if lang.get("mixed_site") and (lang.get("detected_pages") or 0) >= 10: + counts = lang.get("counts") or {} + top = sorted(counts.items(), key=lambda x: -x[1])[:3] + desc = ", ".join(f"{k}:{v}" for k, v in top) if top else "multiple" + issues.append(_issue( + f"Mixed languages detected across pages ({desc}).", + priority="Medium", + recommendation="Ensure hreflang and localized URLs match user intent; split sitemaps if needed.", + )) + deductions.append((5, True)) + + score = _score_deductions(100, deductions) + return { + "id": "intelligence", + "name": CATEGORY_CONTENT_QUALITY, + "score": score, + "issues": _sort_issues(issues), + "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), + } diff --git a/src/website_profiling/reporting/categories/link_health.py b/src/website_profiling/reporting/categories/link_health.py new file mode 100644 index 00000000..e4af71d4 --- /dev/null +++ b/src/website_profiling/reporting/categories/link_health.py @@ -0,0 +1,89 @@ +"""Report category: link_health.""" +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd + +from ._helpers import ( + PRIORITY_ORDER, + REDIRECT_CHAIN_LONG, + _broken_link_sources, + _issue, + _orphan_hub_suggestions, + _score_deductions, + _sort_issues, +) +from ..terminology import ( + CATEGORY_LINKS, +) + +def category_link_health( + df: pd.DataFrame, + edges: list[tuple[str, str]], + issues_broken: list[dict], + issues_redirects: list[dict], +) -> dict: + """Link Health: broken links, redirect chains, internal linking.""" + issues = [] + deductions = [] + + for b in issues_broken[:30]: + status = str(b.get("status", "")) + priority = "Critical" if status.startswith("5") else "High" + issues.append(_issue( + f"Broken URL: {status}", + url=b.get("url", ""), + priority=priority, + recommendation="Fix or remove the link; return 200 or redirect to a valid URL.", + )) + broken_url_set = {str(b.get("url") or "").strip() for b in issues_broken if b.get("url")} + issues.extend(_broken_link_sources(edges, broken_url_set)) + if issues_broken: + deductions.append((min(30, len(issues_broken) * 2), True)) + + for r in issues_redirects[:20]: + issues.append(_issue( + f"Redirect: {r.get('status', '')} to {r.get('final_url', '')}", + url=r.get("url", ""), + priority="Medium", + recommendation="Prefer direct URLs or shorten redirect chains.", + )) + if issues_redirects: + deductions.append((min(15, len(issues_redirects)), True)) + + if "redirect_chain_length" in df.columns and len(df) > 0: + rcl = pd.to_numeric(df["redirect_chain_length"], errors="coerce").fillna(0).astype(int) + long_chains = (rcl >= REDIRECT_CHAIN_LONG).sum() + if long_chains > 0: + issues.append(_issue( + f"{int(long_chains)} URL(s) have redirect chains (2+ hops).", + priority="Medium", + recommendation="Consolidate redirects to a single hop where possible.", + )) + deductions.append((min(10, int(long_chains)), True)) + + if edges: + import networkx as nx + G = nx.DiGraph() + G.add_edges_from(edges) + in_deg = dict(G.in_degree()) + orphans = [n for n in G.nodes() if in_deg.get(n, 0) == 0] + if len(orphans) > len(G.nodes()) * 0.3: + issues.append(_issue( + f"Many pages have no internal links pointing to them ({len(orphans)}).", + priority="Low", + recommendation="Add internal links to important pages to improve crawlability and internal link equity.", + )) + deductions.append((5, True)) + issues.extend(_orphan_hub_suggestions(edges, orphans[:15])) + + score = _score_deductions(100, deductions) + return { + "id": "link_health", + "name": CATEGORY_LINKS, + "score": score, + "issues": _sort_issues(issues), + "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), + } + diff --git a/src/website_profiling/reporting/categories/mobile.py b/src/website_profiling/reporting/categories/mobile.py new file mode 100644 index 00000000..065dc97f --- /dev/null +++ b/src/website_profiling/reporting/categories/mobile.py @@ -0,0 +1,62 @@ +"""Report category: mobile.""" +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd + +from ._helpers import ( + PRIORITY_ORDER, + _broken_link_sources, + _hreflang_issues, + _indexation_coverage_issues, + _issue, + _orphan_hub_suggestions, + _page_analysis_dict, + _schema_issues, + _score_deductions, + _soft_404_issues, + _sort_issues, +) +from ..terminology import ( + CATEGORY_MOBILE, +) + +def category_mobile(df: pd.DataFrame) -> dict: + """Mobile: viewport, responsive heuristic.""" + issues = [] + deductions = [] + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() + if len(success_df) == 0: + return {"id": "mobile", "name": CATEGORY_MOBILE, "score": 0, "issues": [], "recommendations": []} + + if "viewport_present" in df.columns: + viewport_ok = success_df["viewport_present"].astype(str).str.lower().isin(("true", "1", "yes")) + no_viewport = int((~viewport_ok).sum()) + if no_viewport > 0: + issues.append(_issue( + f"{int(no_viewport)} page(s) missing viewport meta tag.", + priority="Critical", + recommendation="Add .", + )) + deductions.append((min(25, int(no_viewport) * 5), True)) + viewport_content = success_df["viewport_content"].fillna("").astype(str) + viewport_ok = success_df["viewport_present"].astype(str).str.lower().isin(("true", "1", "yes")) + invalid = (viewport_content.str.strip().eq("") | (~viewport_content.str.contains("width|device-width", case=False, na=False))) & viewport_ok + if invalid.sum() > 0: + issues.append(_issue( + "Some pages have viewport without width or device-width.", + priority="High", + recommendation="Use content='width=device-width, initial-scale=1' (or similar).", + )) + deductions.append((10, True)) + + score = _score_deductions(100, deductions) + return { + "id": "mobile", + "name": CATEGORY_MOBILE, + "score": score, + "issues": _sort_issues(issues), + "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), + } + diff --git a/src/website_profiling/reporting/categories/performance.py b/src/website_profiling/reporting/categories/performance.py new file mode 100644 index 00000000..b4e7ffa6 --- /dev/null +++ b/src/website_profiling/reporting/categories/performance.py @@ -0,0 +1,160 @@ +"""Report category: performance.""" +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd + +from urllib.parse import urlparse + +from ._helpers import ( + PRIORITY_ORDER, + RESPONSE_TIME_SLOW_MS, + _issue, + _score_deductions, + _sort_issues, +) +from ..terminology import ( + CATEGORY_CORE_WEB_VITALS, + CATEGORY_PERFORMANCE, +) + +def category_core_web_vitals() -> dict: + """Core Web Vitals: not measured; recommend Lighthouse.""" + return { + "id": "core_web_vitals", + "name": CATEGORY_CORE_WEB_VITALS, + "score": None, + "issues": [_issue( + "LCP, INP, and CLS are not measured by this crawl.", + priority="Medium", + recommendation="Run Lighthouse (PageSpeed Insights) from Run audit to measure Core Web Vitals.", + )], + "recommendations": ["Run Lighthouse from Run audit to measure LCP, INP, and CLS."], + } + + +def category_core_web_vitals_from_lighthouse( + lighthouse_summary: dict, + crux_summary: Optional[dict] = None, +) -> dict: + """Core Web Vitals from Lighthouse summary: score 0–100 from performance score, issues from top_failures.""" + issues = [] + recommendations = [] + perf_score = None + mm = lighthouse_summary.get("median_metrics") or {} + if isinstance(mm.get("performance_score"), (int, float)): + perf_score = max(0, min(100, int(round(mm["performance_score"] * 100)))) + for f in lighthouse_summary.get("top_failures") or []: + aid = f.get("id") or "" + help_text = (f.get("helpText") or "")[:200] + msg = f"{aid}: {help_text}" if aid else help_text or "Audit failed" + issues.append(_issue( + msg, + priority="High" if (f.get("score") or 0) < 0.5 else "Medium", + recommendation="See Performance (Core Web Vitals) in this audit, or re-run Lighthouse from Run audit.", + )) + if not issues and perf_score is not None and perf_score < 80: + recommendations.append("Improve Core Web Vitals (LCP, CLS, TBT) per Lighthouse recommendations.") + if crux_summary and crux_summary.get("ok"): + pw = crux_summary.get("pass") or {} + for metric, label, rec in ( + ("lcp", "LCP", "Improve largest contentful paint (field data)."), + ("inp", "INP", "Reduce interaction to next paint (field data)."), + ("cls", "CLS", "Reduce cumulative layout shift (field data)."), + ): + if pw.get(metric) is False: + issues.append(_issue( + f"CrUX field data: {label} does not pass Core Web Vitals threshold.", + priority="High", + recommendation=rec, + )) + return { + "id": "core_web_vitals", + "name": CATEGORY_CORE_WEB_VITALS, + "score": perf_score, + "issues": _sort_issues(issues), + "recommendations": recommendations or ["Core Web Vitals measured by Lighthouse; see median_metrics in lighthouse_summary.json."], + } + + +def category_performance(df: pd.DataFrame) -> dict: + """Performance: response time, JS/CSS size, images, lazy loading, caching.""" + issues = [] + deductions = [] + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() + if len(success_df) == 0: + return {"id": "performance", "name": CATEGORY_PERFORMANCE, "score": 0, "issues": [], "recommendations": []} + + if "response_time_ms" in success_df.columns: + rt = pd.to_numeric(success_df["response_time_ms"], errors="coerce").fillna(0) + slow = (rt > RESPONSE_TIME_SLOW_MS).sum() + if slow > 0: + issues.append(_issue( + f"{int(slow)} page(s) have server response time > {RESPONSE_TIME_SLOW_MS // 1000}s.", + priority="High" if slow > 5 else "Medium", + recommendation="Optimize server response time (TTFB): caching, CDN, or backend tuning.", + )) + deductions.append((min(20, int(slow) * 2), True)) + valid_rt = rt[rt > 0] + if len(valid_rt) > 5: + p95 = float(valid_rt.quantile(0.95)) + if p95 > 3000: + issues.append(_issue( + f"95th percentile response time is {int(p95)}ms (over 3s).", + priority="High", + recommendation="Investigate slowest pages; consider CDN, server-side caching, or database optimization.", + )) + deductions.append((10, True)) + + if "images_total" in success_df.columns: + total_imgs = success_df["images_total"].fillna(0).astype(int).sum() + if total_imgs > 0 and "img_without_lazy" in success_df.columns: + no_lazy = success_df["img_without_lazy"].fillna(0).astype(int).sum() + if no_lazy > total_imgs * 0.5: + issues.append(_issue( + "Many images without lazy loading.", + priority="Medium", + recommendation="Add loading='lazy' to off-screen images.", + )) + deductions.append((10, True)) + if total_imgs > 0 and "img_without_dimensions" in success_df.columns: + no_dims = success_df["img_without_dimensions"].fillna(0).astype(int).sum() + if no_dims > 0: + issues.append(_issue( + f"{int(no_dims)} image(s) without width/height (can cause CLS).", + priority="High", + recommendation="Set width and height attributes on img tags to avoid layout shift.", + )) + deductions.append((10, True)) + + if "cache_control" in success_df.columns: + cache = success_df["cache_control"].fillna("").astype(str) + no_cache = (cache.str.strip() == "").sum() + if no_cache > len(success_df) * 0.5: + issues.append(_issue( + "Many pages without Cache-Control header.", + priority="Medium", + recommendation="Set Cache-Control (and optionally ETag) for static and cacheable pages.", + )) + deductions.append((10, True)) + + if "script_count" in success_df.columns: + scripts = success_df["script_count"].fillna(0).astype(int) + if scripts.sum() > len(success_df) * 10: + issues.append(_issue( + "High number of script tags across pages.", + priority="Low", + recommendation="Consider bundling and code-splitting to reduce JS payload.", + )) + deductions.append((5, True)) + + score = _score_deductions(100, deductions) + return { + "id": "performance", + "name": CATEGORY_PERFORMANCE, + "score": score, + "issues": _sort_issues(issues), + "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), + } + diff --git a/src/website_profiling/reporting/categories/security.py b/src/website_profiling/reporting/categories/security.py new file mode 100644 index 00000000..31ff13ba --- /dev/null +++ b/src/website_profiling/reporting/categories/security.py @@ -0,0 +1,117 @@ +"""Report category: security.""" +from __future__ import annotations + +from typing import Any, Optional +from urllib.parse import urlparse + +import pandas as pd + +from ._helpers import ( + PRIORITY_ORDER, + _broken_link_sources, + _hreflang_issues, + _indexation_coverage_issues, + _issue, + _orphan_hub_suggestions, + _page_analysis_dict, + _schema_issues, + _score_deductions, + _soft_404_issues, + _sort_issues, +) +from ..terminology import ( + CATEGORY_SECURITY, +) + +def category_security( + df: pd.DataFrame, + site_level: dict, + start_url: str, + security_findings: Optional[list[dict]] = None, +) -> dict: + """Security: HTTPS, security headers, mixed content, and optional vulnerability scan findings.""" + issues = [] + deductions = [] + parsed = urlparse(start_url) + if parsed.scheme and parsed.scheme.lower() != "https": + issues.append(_issue( + "Site is not using HTTPS.", + url=start_url, + priority="Critical", + recommendation="Serve the site over HTTPS and redirect HTTP to HTTPS.", + )) + deductions.append((30, True)) + + if "final_url" in df.columns and len(df) > 0: + final_urls = df["final_url"].fillna("").astype(str) + http_finals = final_urls.str.strip().str.lower().str.startswith("http://") + if http_finals.sum() > 0: + issues.append(_issue( + f"{int(http_finals.sum())} URL(s) resolve to HTTP.", + priority="Critical", + recommendation="Ensure all pages redirect to HTTPS.", + )) + deductions.append((20, True)) + + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() + if len(success_df) > 0: + # Security headers: sample from first row or aggregate (optional columns) + missing_hsts = (success_df["strict_transport_security"].fillna("").astype(str).str.strip() == "").sum() if "strict_transport_security" in success_df.columns else len(success_df) + missing_xcto = (success_df["x_content_type_options"].fillna("").astype(str).str.strip() == "").sum() if "x_content_type_options" in success_df.columns else len(success_df) + missing_xfo = (success_df["x_frame_options"].fillna("").astype(str).str.strip() == "").sum() if "x_frame_options" in success_df.columns else len(success_df) + if missing_hsts >= len(success_df) * 0.5: + issues.append(_issue( + "Strict-Transport-Security header not set.", + priority="High", + recommendation="Add Strict-Transport-Security to enforce HTTPS.", + )) + deductions.append((15, True)) + if missing_xcto >= len(success_df) * 0.5: + issues.append(_issue( + "X-Content-Type-Options header not set.", + priority="Medium", + recommendation="Add X-Content-Type-Options: nosniff.", + )) + deductions.append((5, True)) + if missing_xfo >= len(success_df) * 0.5: + issues.append(_issue( + "X-Frame-Options header not set.", + priority="Medium", + recommendation="Add X-Frame-Options: DENY or SAMEORIGIN.", + )) + deductions.append((5, True)) + + if "mixed_content_count" in success_df.columns: + mixed = success_df["mixed_content_count"].fillna(0).astype(int).sum() + scheme = (parsed.scheme or "").lower() + if mixed > 0 and scheme == "https": + issues.append(_issue( + f"Mixed content: {int(mixed)} HTTP resource(s) on HTTPS pages.", + priority="High", + recommendation="Load all resources over HTTPS to avoid mixed content.", + )) + deductions.append((15, True)) + + # Merge vulnerability scan findings (same format as issues: message, url, priority, recommendation) + if security_findings: + for f in security_findings: + severity = f.get("severity", "Medium") + issues.append(_issue( + f.get("message", ""), + url=f.get("url", ""), + priority=severity, + recommendation=f.get("recommendation", ""), + )) + # Deduct by severity: Critical 15, High 10, Medium 5, Low 2 + ded = {"Critical": 15, "High": 10, "Medium": 5, "Low": 2}.get(severity, 2) + deductions.append((min(ded, 15), True)) + + score = _score_deductions(100, deductions) + return { + "id": "security", + "name": CATEGORY_SECURITY, + "score": score, + "issues": _sort_issues(issues), + "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), + } + diff --git a/src/website_profiling/reporting/categories/technical_seo.py b/src/website_profiling/reporting/categories/technical_seo.py new file mode 100644 index 00000000..725122e9 --- /dev/null +++ b/src/website_profiling/reporting/categories/technical_seo.py @@ -0,0 +1,211 @@ +"""Report category: technical_seo.""" +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd + +from ._helpers import ( + PRIORITY_ORDER, + _broken_link_sources, + _hreflang_issues, + _indexation_coverage_issues, + _issue, + _orphan_hub_suggestions, + _page_analysis_dict, + _schema_issues, + _score_deductions, + _soft_404_issues, + _sort_issues, +) +from ..terminology import ( + CATEGORY_TECHNICAL_SEO, +) + +def category_technical_seo( + df: pd.DataFrame, + site_level: dict, +) -> dict: + """Technical SEO: robots, sitemap, canonical, duplicate content, noindex, schema.""" + issues = [] + deductions = [] + total = len(df) + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() + + if not site_level.get("robots_present", True): + issues.append(_issue( + "robots.txt is missing or unreachable.", + priority="High", + recommendation="Add a robots.txt at the site root to control crawler access.", + )) + deductions.append((15, True)) + if not site_level.get("sitemap_present", True): + issues.append(_issue( + "sitemap.xml (or sitemap index) is missing or unreachable.", + priority="High", + recommendation="Add a sitemap at /sitemap.xml or link it in robots.txt.", + )) + deductions.append((10, True)) + if site_level.get("sitemap_present") and not site_level.get("sitemap_valid", True): + issues.append(_issue( + "sitemap.xml could not be parsed as valid XML.", + priority="Medium", + recommendation="Ensure sitemap is valid XML and follows sitemaps.org format.", + )) + deductions.append((5, True)) + if site_level.get("ads_txt_present") is False: + issues.append(_issue( + "ads.txt is missing or unreachable.", + priority="Low", + recommendation="Add an ads.txt file at the site root if you run programmatic advertising.", + )) + if site_level.get("security_txt_present") is False: + issues.append(_issue( + "security.txt is missing or unreachable.", + priority="Low", + recommendation="Publish security.txt at /.well-known/security.txt with a Contact field for security reporting.", + )) + + # Canonical: missing or self-mismatch + if "canonical_url" in df.columns and len(success_df) > 0: + for _, row in success_df.iterrows(): + url = row.get("url") + canon = row.get("canonical_url") + if pd.isna(url): + continue + url = str(url).strip() + canon = "" if pd.isna(canon) else str(canon).strip() + if not canon: + issues.append(_issue("Missing canonical URL.", url=url, priority="Medium", recommendation="Add a canonical link tag pointing to the preferred URL.")) + break + missing_canon = success_df["canonical_url"].fillna("").astype(str).str.strip().eq("").sum() + if missing_canon > 0: + deductions.append((min(15, missing_canon * 2), True)) + # Self-canonical mismatch: canonical points to different URL + for _, row in success_df.iterrows(): + url = row.get("url") + canon = row.get("canonical_url") + if pd.isna(url) or pd.isna(canon) or not str(canon).strip(): + continue + url = str(url).rstrip("/") + canon = str(canon).strip().rstrip("/") + if url != canon: + issues.append(_issue(f"Canonical points to different URL: {canon}", url=url, priority="High", recommendation="Set canonical to this page URL or the preferred duplicate.")) + deductions.append((10, True)) + break + + # Noindex on important pages (CSV may store True/False as strings) + if "noindex" in df.columns and len(success_df) > 0: + noindex_ser = success_df["noindex"].astype(str).str.lower().isin(("true", "1", "yes")) + noindex_count = int(noindex_ser.sum()) + if noindex_count > 0: + issues.append(_issue( + f"{int(noindex_count)} page(s) have noindex.", + priority="High" if noindex_count > 5 else "Medium", + recommendation="Remove noindex from pages that should be indexed, or keep for intentional no-index pages.", + )) + deductions.append((min(15, noindex_count * 3), True)) + + # Duplicate content heuristic: same title + meta description + if "title" in df.columns and "meta_description" in df.columns and len(success_df) > 1: + key = success_df["title"].fillna("").astype(str) + "|" + success_df["meta_description"].fillna("").astype(str) + dupes = key.value_counts() + dupes = dupes[dupes > 1] + if len(dupes) > 0: + issues.append(_issue( + f"Possible duplicate content: {len(dupes)} group(s) of pages share same title and meta description.", + priority="Medium", + recommendation="Differentiate titles and meta descriptions, or use canonicals to designate the preferred URL.", + )) + deductions.append((10, True)) + + # Social meta tags + if "og_title" in df.columns and len(success_df) > 0: + og_present = (success_df["og_title"].fillna("").astype(str).str.strip() != "").sum() + og_pct = og_present / len(success_df) if len(success_df) > 0 else 1 + if og_pct < 0.5: + issues.append(_issue( + f"Open Graph tags missing on {int((1 - og_pct) * 100)}% of pages.", + priority="Medium", + recommendation="Add og:title, og:description, and og:image meta tags for social sharing.", + )) + deductions.append((5, True)) + + if "twitter_card" in df.columns and len(success_df) > 0: + tw_present = (success_df["twitter_card"].fillna("").astype(str).str.strip() != "").sum() + tw_pct = tw_present / len(success_df) if len(success_df) > 0 else 1 + if tw_pct < 0.2: + issues.append(_issue( + f"Twitter Card tags missing on {int((1 - tw_pct) * 100)}% of pages.", + priority="Low", + recommendation="Add twitter:card meta tags for better Twitter/X sharing previews.", + )) + deductions.append((3, True)) + + # Structured data + if "has_schema" in df.columns and len(success_df) > 0: + with_schema = int(success_df["has_schema"].astype(str).str.lower().isin(("true", "1", "yes")).sum()) + if with_schema == 0: + issues.append(_issue( + "No structured data (JSON-LD or microdata) detected.", + priority="Low", + recommendation="Add schema.org markup (e.g. Organization, Article) for rich results.", + )) + deductions.append((5, True)) + + # Internationalization: from page_analysis (re-crawl to populate) + if "page_analysis" in df.columns and len(success_df) > 0: + missing_lang = 0 + for _, row in success_df.iterrows(): + pa = _page_analysis_dict(row) + if not (pa.get("html_lang") or "").strip(): + missing_lang += 1 + if missing_lang > 0 and len(success_df) >= 3: + ratio = missing_lang / len(success_df) + if ratio > 0.1: + issues.append(_issue( + f"{missing_lang} page(s) missing (of {len(success_df)} OK responses).", + priority="Medium" if ratio > 0.5 else "Low", + recommendation="Add matching the primary language of each page.", + )) + deductions.append((min(10, max(2, missing_lang // 5)), True)) + + issues.extend(_hreflang_issues(success_df)) + issues.extend(_schema_issues(success_df)) + issues.extend(_soft_404_issues(success_df)) + + if "page_analysis" in df.columns and len(success_df) > 0: + from ...crawl.fetchers.browser_diagnostics import browser_summary_from_page_analysis + + pages_with_console = 0 + for _, row in success_df.iterrows(): + pa = _page_analysis_dict(row) + counts = browser_summary_from_page_analysis(pa) + url = str(row.get("url") or "").strip() + if counts["console_error_count"] > 0: + pages_with_console += 1 + if counts["page_error_count"] > 0 and url: + issues.append(_issue( + "Uncaught JavaScript error during browser render.", + url=url, + priority="High", + recommendation="Fix runtime JS errors that may break page functionality or SEO signals.", + )) + deductions.append((5, True)) + if pages_with_console > 0: + issues.append(_issue( + f"{pages_with_console} page(s) logged console errors during JavaScript rendering.", + priority="High" if pages_with_console > 3 else "Medium", + recommendation="Inspect browser console errors on affected URLs; fix broken scripts or API calls.", + )) + deductions.append((min(15, pages_with_console * 2), True)) + + score = _score_deductions(100, deductions) + return { + "id": "technical_seo", + "name": CATEGORY_TECHNICAL_SEO, + "score": score, + "issues": _sort_issues(issues), + "recommendations": list({i["recommendation"] for i in issues if i["recommendation"]}), + } + diff --git a/src/website_profiling/reporting/content_analytics.py b/src/website_profiling/reporting/content_analytics.py new file mode 100644 index 00000000..3018e131 --- /dev/null +++ b/src/website_profiling/reporting/content_analytics.py @@ -0,0 +1,434 @@ +"""Content and crawl analytics for report payloads.""" +from __future__ import annotations + +import json +from typing import Any, Optional + +import pandas as pd + +from ..analysis.text_hygiene import filter_topic_clusters, is_junk_semantic_term +from ..config import get_bool, get_int +from ..tools.keywords import cluster_keywords, extract_candidates_from_df, score_keywords + +def _build_content_analytics(df: pd.DataFrame) -> dict: + """Build content analytics: word count stats, reading level distribution, content ratio, top keywords.""" + from collections import Counter + + result = { + "word_count_stats": {"mean": 0, "median": 0, "p25": 0, "p75": 0, "min": 0, "max": 0}, + "word_count_distribution": {}, + "reading_level_distribution": {}, + "content_ratio_distribution": {}, + "top_keywords_site": [], + "thin_pages": [], + } + if "word_count" not in df.columns or df.empty: + return result + + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + if success_df.empty: + return result + + wc = pd.to_numeric(success_df["word_count"], errors="coerce").fillna(0).astype(int) + result["word_count_stats"] = { + "mean": round(float(wc.mean()), 1), + "median": round(float(wc.median()), 1), + "p25": round(float(wc.quantile(0.25)), 1), + "p75": round(float(wc.quantile(0.75)), 1), + "min": int(wc.min()), + "max": int(wc.max()), + } + + wc_bins = [(0, 100), (101, 300), (301, 600), (601, 1000), (1001, 2000), (2001, 999999)] + wc_labels = ["0-100", "101-300", "301-600", "601-1000", "1001-2000", "2001+"] + result["word_count_distribution"] = { + lbl: int(((wc >= lo) & (wc <= hi)).sum()) for (lo, hi), lbl in zip(wc_bins, wc_labels) + } + + if "reading_level" in success_df.columns: + rl = pd.to_numeric(success_df["reading_level"], errors="coerce").fillna(0) + rl_bins = [(0, 5), (6, 8), (9, 12), (13, 99)] + rl_labels = ["Elementary (0-5)", "Middle School (6-8)", "High School (9-12)", "College (13+)"] + result["reading_level_distribution"] = { + lbl: int(((rl >= lo) & (rl <= hi)).sum()) for (lo, hi), lbl in zip(rl_bins, rl_labels) + } + + if "content_html_ratio" in success_df.columns: + cr = pd.to_numeric(success_df["content_html_ratio"], errors="coerce").fillna(0) + cr_bins = [(0, 10), (10.01, 20), (20.01, 40), (40.01, 100)] + cr_labels = ["<10%", "10-20%", "20-40%", ">40%"] + result["content_ratio_distribution"] = { + lbl: int(((cr >= lo) & (cr <= hi)).sum()) for (lo, hi), lbl in zip(cr_bins, cr_labels) + } + + if "top_keywords" in success_df.columns: + kw_counter = Counter() + for raw in success_df["top_keywords"].fillna("[]"): + try: + items = json.loads(str(raw)) if isinstance(raw, str) else raw + if isinstance(items, list): + for item in items: + if isinstance(item, dict): + kw_counter[item.get("word", "")] += item.get("count", 0) + except (json.JSONDecodeError, TypeError): + pass + result["top_keywords_site"] = [ + {"word": w, "count": c} + for w, c in kw_counter.most_common(50) + if w and not is_junk_semantic_term(str(w)) + ][:30] + + for _, row in success_df.iterrows(): + u = row.get("url") + if pd.isna(u) or not u: + continue + w = int(pd.to_numeric(row.get("word_count"), errors="coerce") or 0) + if 0 < w < 300: + result["thin_pages"].append({"url": str(u).strip(), "word_count": w}) + + return result + + +def _parse_top_keywords_items(raw: Any) -> list[dict[str, Any]]: + """Parse per-page top_keywords JSON into dict items with word/count.""" + if raw is None or (isinstance(raw, float) and pd.isna(raw)): + return [] + try: + items = json.loads(str(raw)) if isinstance(raw, str) else raw + except (json.JSONDecodeError, TypeError, ValueError): + return [] + if not isinstance(items, list): + return [] + out: list[dict[str, Any]] = [] + for item in items: + if isinstance(item, dict): + word = str(item.get("word") or "").strip() + if word: + out.append({"word": word, "count": int(item.get("count") or 1)}) + return out + + +def _build_text_content_analysis(df: pd.DataFrame) -> dict: + """Cross-page keyword aggregates for the text content analysis view.""" + empty = { + "vocabulary_stats": { + "unique_terms": 0, + "pages_with_keywords": 0, + "avg_terms_per_page": 0.0, + "total_term_occurrences": 0, + }, + "keyword_index": [], + "keyword_frequency_histogram": {"1": 0, "2-5": 0, "6-20": 0, "21+": 0}, + } + if df.empty or "top_keywords" not in df.columns: + return empty + + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + if success_df.empty: + return empty + + # word -> { total_count, pages: { url -> count } } + index: dict[str, dict[str, Any]] = {} + pages_with_keywords = 0 + total_occurrences = 0 + + for _, row in success_df.iterrows(): + url = row.get("url") + if pd.isna(url) or not url: + continue + url_str = str(url).strip() + items = _parse_top_keywords_items(row.get("top_keywords")) + page_had_kw = False + for item in items: + word = item["word"].lower() + if is_junk_semantic_term(word): + continue + count = max(1, int(item.get("count") or 1)) + if word not in index: + index[word] = {"total_count": 0, "pages": {}} + index[word]["total_count"] += count + index[word]["pages"][url_str] = index[word]["pages"].get(url_str, 0) + count + total_occurrences += count + page_had_kw = True + if page_had_kw: + pages_with_keywords += 1 + + unique_terms = len(index) + avg_terms = round(total_occurrences / pages_with_keywords, 1) if pages_with_keywords else 0.0 + + histogram = {"1": 0, "2-5": 0, "6-20": 0, "21+": 0} + for data in index.values(): + pc = len(data["pages"]) + if pc == 1: + histogram["1"] += 1 + elif pc <= 5: + histogram["2-5"] += 1 + elif pc <= 20: + histogram["6-20"] += 1 + else: + histogram["21+"] += 1 + + sorted_words = sorted(index.items(), key=lambda x: x[1]["total_count"], reverse=True) + keyword_index: list[dict[str, Any]] = [] + for word, data in sorted_words: + top_pages = sorted(data["pages"].items(), key=lambda x: x[1], reverse=True)[:5] + keyword_index.append( + { + "word": word, + "total_count": data["total_count"], + "page_count": len(data["pages"]), + "top_pages": [{"url": u, "count": c} for u, c in top_pages], + } + ) + + return { + "vocabulary_stats": { + "unique_terms": unique_terms, + "pages_with_keywords": pages_with_keywords, + "avg_terms_per_page": avg_terms, + "total_term_occurrences": total_occurrences, + }, + "keyword_index": keyword_index, + "keyword_frequency_histogram": histogram, + } + + +def _build_social_coverage(df: pd.DataFrame) -> dict: + """Build social meta coverage stats: OG and Twitter Card presence percentages.""" + result = { + "og_coverage_pct": 0, + "twitter_coverage_pct": 0, + "og_image_coverage_pct": 0, + "missing_og": [], + "missing_twitter": [], + "og_image_missing": [], + } + if df.empty: + return result + + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + html_df = success_df + if "content_type" in success_df.columns: + html_df = success_df[success_df["content_type"].fillna("").str.contains("text/html", case=False, na=False)] + if html_df.empty: + return result + + total = len(html_df) + + if "og_title" in html_df.columns: + has_og = (html_df["og_title"].fillna("").astype(str).str.strip() != "").sum() + result["og_coverage_pct"] = round(100 * int(has_og) / total, 1) + for _, row in html_df.iterrows(): + u = row.get("url") + if pd.isna(u): + continue + u = str(u).strip() + og = str(row.get("og_title") or "").strip() + if not og: + result["missing_og"].append(u) + + if "twitter_card" in html_df.columns: + has_tw = (html_df["twitter_card"].fillna("").astype(str).str.strip() != "").sum() + result["twitter_coverage_pct"] = round(100 * int(has_tw) / total, 1) + for _, row in html_df.iterrows(): + u = row.get("url") + if pd.isna(u): + continue + u = str(u).strip() + tw = str(row.get("twitter_card") or "").strip() + if not tw: + result["missing_twitter"].append(u) + + if "og_image" in html_df.columns: + has_og_img = (html_df["og_image"].fillna("").astype(str).str.strip() != "").sum() + result["og_image_coverage_pct"] = round(100 * int(has_og_img) / total, 1) + for _, row in html_df.iterrows(): + u = row.get("url") + if pd.isna(u): + continue + u = str(u).strip() + img = str(row.get("og_image") or "").strip() + if not img: + result["og_image_missing"].append(u) + + result["missing_og"] = result["missing_og"][:100] + result["missing_twitter"] = result["missing_twitter"][:100] + result["og_image_missing"] = result["og_image_missing"][:100] + return result + + +def _build_tech_stack_summary(df: pd.DataFrame) -> dict: + """Build tech stack summary: detected technologies with counts and sample URLs.""" + from collections import defaultdict + + result = {"technologies": [], "total_pages_analyzed": 0} + if "tech_stack" not in df.columns or df.empty: + return result + + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + html_df = success_df + if "content_type" in success_df.columns: + html_df = success_df[success_df["content_type"].fillna("").str.contains("text/html", case=False, na=False)] + if html_df.empty: + return result + + result["total_pages_analyzed"] = len(html_df) + tech_urls = defaultdict(list) + + for _, row in html_df.iterrows(): + u = str(row.get("url", "")).strip() + raw = row.get("tech_stack") or "[]" + try: + techs = json.loads(str(raw)) if isinstance(raw, str) else raw + if isinstance(techs, list): + for t in techs: + if isinstance(t, str) and t: + tech_urls[t].append(u) + except (json.JSONDecodeError, TypeError): + pass + + result["technologies"] = sorted( + [{"name": name, "count": len(urls), "sample_urls": urls[:3]} for name, urls in tech_urls.items()], + key=lambda x: x["count"], + reverse=True, + ) + return result + + +def _build_response_time_stats(df: pd.DataFrame) -> dict: + """Build response time statistics and distribution.""" + result = { + "p25": 0, "p50": 0, "p75": 0, "p95": 0, "p99": 0, + "slow_pages": [], + "distribution": {}, + } + if "response_time_ms" not in df.columns or df.empty: + return result + + rt = pd.to_numeric(df["response_time_ms"], errors="coerce").dropna() + if rt.empty: + return result + + result["p25"] = round(float(rt.quantile(0.25)), 0) + result["p50"] = round(float(rt.quantile(0.50)), 0) + result["p75"] = round(float(rt.quantile(0.75)), 0) + result["p95"] = round(float(rt.quantile(0.95)), 0) + result["p99"] = round(float(rt.quantile(0.99)), 0) + + rt_bins = [(0, 200), (200, 500), (500, 1000), (1000, 2000), (2000, 999999)] + rt_labels = ["<200ms", "200-500ms", "500ms-1s", "1-2s", ">2s"] + rt_full = pd.to_numeric(df["response_time_ms"], errors="coerce").fillna(0) + result["distribution"] = { + lbl: int(((rt_full >= lo) & (rt_full < hi)).sum()) for (lo, hi), lbl in zip(rt_bins, rt_labels) + } + + for _, row in df.iterrows(): + u = row.get("url") + ms = pd.to_numeric(row.get("response_time_ms"), errors="coerce") + if pd.isna(u) or pd.isna(ms) or ms <= 2000: + continue + result["slow_pages"].append({"url": str(u).strip(), "response_time_ms": int(ms)}) + result["slow_pages"] = sorted(result["slow_pages"], key=lambda x: x["response_time_ms"], reverse=True)[:50] + return result + + +def _build_depth_distribution(df: pd.DataFrame) -> dict: + """Build crawl depth distribution.""" + result = {"by_depth": {}, "max_depth": 0, "avg_depth": 0} + if "depth" not in df.columns or df.empty: + return result + + depths = pd.to_numeric(df["depth"], errors="coerce").dropna().astype(int) + if depths.empty: + return result + + result["max_depth"] = int(depths.max()) + result["avg_depth"] = round(float(depths.mean()), 1) + counts = depths.value_counts().sort_index() + result["by_depth"] = {str(int(k)): int(v) for k, v in counts.items()} + return result + + +def _build_keyword_opportunities(df: pd.DataFrame, config: dict[str, str] | None) -> dict[str, Any]: + if not get_bool(config or {}, "include_keyword_opportunities", True): + return {} + if "status" not in df.columns or df.empty: + return {"quick_wins": [], "high_value": [], "token_topic_clusters": []} + success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] + if success_df.empty: + return {"quick_wins": [], "high_value": [], "token_topic_clusters": []} + candidates = extract_candidates_from_df(success_df) + if not candidates: + return {"quick_wins": [], "high_value": [], "token_topic_clusters": []} + corpus_size = len(success_df) + scored = score_keywords(candidates, corpus_size=corpus_size) + clusters = cluster_keywords(scored) + quick_wins = [s for s in scored if s.get("difficulty", 100) < 60][:10] + high_value = [s for s in scored if (s.get("volume") or 0) >= 0.5][:10] + if not high_value: + high_value = scored[:10] + return { + "quick_wins": quick_wins[:10], + "high_value": high_value[:10], + "token_topic_clusters": filter_topic_clusters(clusters)[:50], + } + + +def _build_image_inventory( + links: list[dict[str, Any]], + config: Optional[dict[str, str]], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + from ..analysis.image_probe import collect_image_refs_from_links, probe_image_urls + + refs = collect_image_refs_from_links(links) + unoptimized_min_kb = get_int(config or {}, "image_unoptimized_min_kb", 200) or 200 + summary: dict[str, Any] = { + "probed": 0, + "failed": 0, + "total_bytes": 0, + "over_threshold_count": 0, + "unoptimized_min_kb": unoptimized_min_kb, + "inventory_available": False, + } + if not get_bool(config or {}, "probe_image_inventory", False): + return [], summary + + max_urls = get_int(config or {}, "max_image_probe_urls", 500) or 500 + concurrency = get_int(config or {}, "image_probe_concurrency", 6) or 6 + probe_timeout = get_int(config or {}, "image_probe_timeout", 8) or 8 + url_list = list(refs.keys())[:max_urls] + if not url_list: + return [], summary + + print(f" Probing up to {len(url_list)} image URL(s)...", flush=True) + probed = probe_image_urls( + url_list, + concurrency=concurrency, + timeout=probe_timeout, + ) + threshold_bytes = unoptimized_min_kb * 1024 + inventory: list[dict[str, Any]] = [] + for row in probed: + url = row.get("url") + meta = refs.get(str(url or ""), {"source_pages": set(), "kinds": set()}) + size = row.get("size_bytes") + entry = { + "url": url, + "status": row.get("status"), + "content_type": row.get("content_type"), + "size_bytes": size, + "error": row.get("error"), + "source_pages": sorted(meta.get("source_pages") or []), + "kinds": sorted(meta.get("kinds") or []), + } + inventory.append(entry) + summary["probed"] += 1 + if row.get("error") or row.get("status") is None: + summary["failed"] += 1 + if size is not None: + summary["total_bytes"] += int(size) + if int(size) >= threshold_bytes: + summary["over_threshold_count"] += 1 + summary["inventory_available"] = True + print(f" Image probe complete ({summary['probed']} URLs, {summary['failed']} failed).", flush=True) + return inventory, summary diff --git a/src/website_profiling/reporting/edges_report.py b/src/website_profiling/reporting/edges_report.py new file mode 100644 index 00000000..fff8ff59 --- /dev/null +++ b/src/website_profiling/reporting/edges_report.py @@ -0,0 +1,114 @@ +"""Build link edges from crawl data.""" +from __future__ import annotations + +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from urllib.parse import urlparse + +import pandas as pd +import requests +from bs4 import BeautifulSoup +from tqdm.auto import tqdm + +from ..common import LINK_COLUMN_NAMES, load_edges, normalize_link, parse_links_serialized + +def build_edges_from_df( + df: pd.DataFrame, + edges_csv: str, + same_domain_only: bool, + max_fetch_for_edges: int, + concurrency: int, + timeout: int, + polite_delay: float, + render_mode: str = "static", + js_timeout: int = 30, + js_concurrency: int = 3, + js_wait_until: str = "domcontentloaded", + js_extra_wait_ms: int = 1500, + js_block_resources: bool = True, +) -> list[tuple[str, str]]: + """Build or load edges; return list of (from, to) tuples.""" + edges = load_edges(edges_csv) if (edges_csv or "").strip() else [] + if edges: + return edges + + # Prefer columns that hold URL lists (e.g. outlink_targets); skip "outlinks" (numeric count) + candidate_cols = [ + c for c in df.columns + if c.lower() in LINK_COLUMN_NAMES and c.lower() != "outlinks" + ] + if candidate_cols: + for col in candidate_cols: + if df[col].notna().sum() == 0: + continue + for src, raw in zip(df["url"], df[col].fillna("")): + for t in parse_links_serialized(raw): + if not t: + continue + if same_domain_only and urlparse(src).netloc != urlparse(t).netloc: + continue + edges.append((src, t)) + if edges: + return edges + + session = requests.Session() + session.headers.update({"User-Agent": "WebsiteProfiling/1.0"}) + urls = df["url"].tolist()[:max_fetch_for_edges] + mode = (render_mode or "static").strip().lower() + use_js = mode in ("javascript", "auto") + fetcher = None + if use_js: + from ..crawl.fetchers import build_fetcher + + fetcher = build_fetcher( + render_mode="javascript" if mode == "javascript" else "auto", + timeout=timeout, + user_agent="WebsiteProfiling/1.0", + session=session, + js_timeout=js_timeout, + js_concurrency=js_concurrency, + js_wait_until=js_wait_until, + js_extra_wait_ms=js_extra_wait_ms, + js_block_resources=js_block_resources, + ) + + def fetch(src): + try: + if fetcher is not None: + r = fetcher.fetch(src) + if r.status != 200 or not r.text: + return [] + html = r.text + else: + resp = session.get(src, timeout=timeout, allow_redirects=True) + if resp.status_code != 200 or not resp.headers.get("Content-Type", "").lower().startswith("text/html"): + return [] + html = resp.text + soup = BeautifulSoup(html, "lxml") + out = set() + for a in soup.find_all("a", href=True): + ln = normalize_link(src, a["href"]) + if not ln or (same_domain_only and urlparse(src).netloc != urlparse(ln).netloc): + continue + out.add(ln) + if polite_delay: + time.sleep(polite_delay) + return list(out) + except Exception: + return [] + + try: + with ThreadPoolExecutor(max_workers=concurrency) as ex: + futures = {ex.submit(fetch, u): u for u in urls} + for f in tqdm(as_completed(futures), total=len(futures), desc="Extracting links"): + src = futures[f] + try: + outs = f.result() + except Exception: + outs = [] + for t in outs: + edges.append((src, t)) + finally: + if fetcher is not None: + fetcher.close() + return edges diff --git a/src/website_profiling/reporting/lighthouse_report.py b/src/website_profiling/reporting/lighthouse_report.py new file mode 100644 index 00000000..5ba6afbc --- /dev/null +++ b/src/website_profiling/reporting/lighthouse_report.py @@ -0,0 +1,195 @@ +"""Lighthouse report helpers and SSL certificate checks.""" +from __future__ import annotations + +import socket +import ssl +from datetime import datetime, timezone +from typing import Any, Optional +from urllib.parse import urlparse + + +def fetch_site_ssl_expires_iso(hostname: str, timeout: float = 5.0) -> Optional[str]: + """Return certificate notAfter as ISO 8601 UTC, or None on failure.""" + host = (hostname or "").strip().lower() + if not host: + return None + try: + ctx = ssl.create_default_context() + with socket.create_connection((host, 443), timeout=timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=host) as ssock: + cert = ssock.getpeercert() + if not cert: + return None + na = cert.get("notAfter") + if not na: + return None + ts = ssl.cert_time_to_seconds(na) + return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() + except Exception: + return None + + +def _strip_www(host: str) -> str: + h = (host or "").strip().lower() + return h[4:] if h.startswith("www.") else h + + +def _url_hostname(url: str) -> str: + if not url: + return "" + try: + return (urlparse(str(url).strip()).hostname or "").lower() + except Exception: + return "" + + +def _hosts_match(a: str, b: str) -> bool: + if not a or not b: + return False + a, b = a.lower(), b.lower() + return a == b or _strip_www(a) == _strip_www(b) + + +def filter_lighthouse_by_host(by_url: dict[str, Any], expected_host: str) -> dict[str, Any]: + """Keep only Lighthouse entries whose URL hostname matches expected_host (www.-tolerant).""" + if not by_url or not expected_host: + return by_url or {} + return {u: v for u, v in by_url.items() if _hosts_match(_url_hostname(u), expected_host)} + + +def _derive_expected_host(start_url: str, df) -> str: + host = _url_hostname(start_url) + if host: + return host + if df is not None and not df.empty and "url" in df.columns: + for u in df["url"]: + h = _url_hostname(str(u)) + if h: + return h + return "" + + +def _pick_lighthouse_summary( + lighthouse_by_url: dict[str, Any], + start_url: str, + global_summary: Optional[dict[str, Any]], + expected_host: str, +) -> Optional[dict[str, Any]]: + """Prefer per-URL summary for this crawl; only use global summary if hostname matches.""" + if lighthouse_by_url and start_url: + match = lighthouse_for_url(lighthouse_by_url, start_url) + if match: + return match + if lighthouse_by_url: + first_key = next(iter(lighthouse_by_url), None) + if first_key is not None: + return lighthouse_by_url[first_key] + if global_summary: + if not expected_host or _hosts_match( + _url_hostname(str(global_summary.get("url") or "")), expected_host + ): + return global_summary + return None + + +def build_lighthouse_by_url_for_report(conn: Any) -> dict[str, Any]: + """ + Merge per-URL Lighthouse page summaries with latest lighthouse_runs row: full audits/items + from normalized tables, uncapped top_failures and diagnostics from stored LHR JSON. + """ + from ..db import ( + read_lh_audits_with_items, + read_lh_runs_by_url, + read_lighthouse_page_summaries, + read_lighthouse_run_json, + ) + from ..lighthouse.runner import _evidence_from_audit, extract_from_lighthouse_json + from ..tools.warnings import parse_lighthouse_to_diagnostics, resolve_impact + + summaries = read_lighthouse_page_summaries(conn) + runs_map = read_lh_runs_by_url(conn) + + summaries_norm: dict[str, Any] = {} + for k, v in summaries.items(): + nk = str(k).strip().rstrip("/") + summaries_norm[nk] = v + + all_urls = set(summaries_norm.keys()) | set(runs_map.keys()) + out: dict[str, Any] = {} + + for u in sorted(all_urls): + base: dict[str, Any] = dict(summaries_norm[u]) if u in summaries_norm else {} + run_ids = runs_map.get(u, []) + run_id = run_ids[-1] if run_ids else None + + if run_id is not None: + raw = read_lighthouse_run_json(conn, run_id) + if not base and raw: + ex = extract_from_lighthouse_json(raw) + lr = raw.get("lighthouseResult") or raw + final_u = lr.get("finalUrl") or lr.get("requestedUrl") or u + base = { + "url": str(final_u).strip().rstrip("/"), + "median_metrics": { + "lcp_ms": ex.get("lcp_ms"), + "cls": ex.get("cls"), + "tbt_ms": ex.get("tbt_ms"), + "fcp_ms": ex.get("fcp_ms"), + "speed_index_ms": ex.get("speed_index_ms"), + "performance_score": ex.get("performance_score"), + "accessibility_score": ex.get("accessibility_score"), + "seo_score": ex.get("seo_score"), + "best_practices_score": ex.get("best_practices_score"), + "pwa_score": ex.get("pwa_score"), + }, + "category_scores": dict(ex.get("category_scores") or {}), + "strategy": "mobile", + "device": "mobile", + "mode": "navigation", + } + base["audits"] = read_lh_audits_with_items(conn, run_id) + if raw: + lr = raw.get("lighthouseResult") or raw + audits_map = lr.get("audits") or {} + failures: list[dict[str, Any]] = [] + for aid, a in audits_map.items(): + if not isinstance(a, dict): + continue + score = a.get("score") + if score is None or score >= 1: + continue + title = a.get("title") or aid + help_text = a.get("helpText") or "" + failures.append( + { + "id": aid, + "score": score, + "helpText": help_text, + "impact": resolve_impact(aid, title, help_text), + "evidence": _evidence_from_audit(a), + } + ) + failures.sort(key=lambda x: (x["score"] or 0)) + base["top_failures"] = failures + base["diagnostics"] = parse_lighthouse_to_diagnostics(raw, max_nodes_in_refs=None) + elif not base: + continue + + if not base.get("url"): + base["url"] = u + out[u] = base + + return out + + +def lighthouse_for_url(lighthouse_by_url: dict[str, Any], url: str) -> Optional[dict[str, Any]]: + """Resolve Lighthouse summary for a crawled URL (trailing-slash tolerant).""" + if not lighthouse_by_url or not url: + return None + u = str(url).strip().rstrip("/") + if u in lighthouse_by_url: + return lighthouse_by_url[u] + for k, v in lighthouse_by_url.items(): + if str(k).strip().rstrip("/") == u: + return v + return None diff --git a/src/website_profiling/reporting/report_metadata.py b/src/website_profiling/reporting/report_metadata.py new file mode 100644 index 00000000..e55ac80b --- /dev/null +++ b/src/website_profiling/reporting/report_metadata.py @@ -0,0 +1,239 @@ +"""Report metadata and URL-level aggregates.""" +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import Any, Optional +from urllib.parse import urlparse + +import pandas as pd + +from ..common import parse_links_serialized +from ..config import get_bool, get_int + +def _parse_page_analysis_cell(raw: object) -> dict[str, Any]: + if raw is None or (isinstance(raw, float) and pd.isna(raw)): + return {} + s = str(raw).strip() + if not s or s == "{}": + return {} + try: + o = json.loads(s) + return o if isinstance(o, dict) else {} + except json.JSONDecodeError: + return {} + + +def _build_outbound_link_domains( + df: pd.DataFrame, + start_url: str, + max_rows: int, +) -> list[dict[str, Any]]: + """Aggregate external hosts linked from crawled pages (outbound), not referring domains.""" + site_host = urlparse((start_url or "").strip()).netloc.lower() + host_pages: dict[str, set[str]] = {} + host_link_count: dict[str, int] = {} + for _, row in df.iterrows(): + st = str(row.get("status", "")).strip() + if st.startswith(("4", "5")): + continue + u = str(row.get("url") or "").strip().rstrip("/") + if not u: + continue + seen_on_page: set[str] = set() + pa = _parse_page_analysis_cell(row.get("page_analysis")) if "page_analysis" in df.columns else {} + for link in pa.get("external_links") or []: + if not isinstance(link, str): + continue + h = urlparse(link).netloc.lower() + if not h or h == site_host: + continue + host_pages.setdefault(h, set()).add(u) + host_link_count[h] = host_link_count.get(h, 0) + 1 + seen_on_page.add(link) + if "outlink_targets" in df.columns: + for link in parse_links_serialized(row.get("outlink_targets")): + h = urlparse(link).netloc.lower() + if not h or h == site_host: + continue + host_pages.setdefault(h, set()).add(u) + if link not in seen_on_page: + host_link_count[h] = host_link_count.get(h, 0) + 1 + seen_on_page.add(link) + rows: list[dict[str, Any]] = [] + for h in host_pages: + rows.append({ + "host": h, + "page_count": len(host_pages[h]), + "link_count": host_link_count.get(h, 0), + }) + rows.sort(key=lambda x: (-x["link_count"], -x["page_count"], x["host"])) + return rows[:max_rows] + + +def _build_url_fingerprints(df: pd.DataFrame) -> list[dict[str, Any]]: + """Stable fingerprints for comparing page content/structure between report runs (no raw HTML stored).""" + out: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + u = str(row.get("url") or "").strip().rstrip("/") + if not u: + continue + title = str(row.get("title") or "") + meta = str(row.get("meta_description") or "") + h1 = str(row.get("h1") or "") + headings = str(row.get("heading_sequence") or "") + wc = int(pd.to_numeric(row.get("word_count"), errors="coerce") or 0) + cl = int(pd.to_numeric(row.get("content_length"), errors="coerce") or 0) + h1c = int(pd.to_numeric(row.get("h1_count"), errors="coerce") or 0) + sc = int(pd.to_numeric(row.get("script_count"), errors="coerce") or 0) + lc = int(pd.to_numeric(row.get("link_stylesheet_count"), errors="coerce") or 0) + # heading_sequence is structural (h1,h2,...) — keep it in structure fingerprint only. + raw_c = "|".join([title, meta, h1, str(wc), str(cl)]).encode("utf-8") + content_fp = hashlib.sha256(raw_c).hexdigest() + raw_s = "|".join([str(cl), str(sc), str(lc), str(h1c), headings]).encode("utf-8") + structure_fp = hashlib.sha256(raw_s).hexdigest() + out.append({ + "url": u, + "content_fingerprint": content_fp, + "structure_fingerprint": structure_fp, + }) + return out + + +def _build_hreflang_summary(df: pd.DataFrame) -> dict[str, Any]: + total = 0 + missing_lang = 0 + with_hreflang = 0 + for _, row in df.iterrows(): + st = str(row.get("status", "")).strip() + if not st.startswith("2"): + continue + total += 1 + pa = _parse_page_analysis_cell(row.get("page_analysis")) if "page_analysis" in df.columns else {} + if not (pa.get("html_lang") or "").strip(): + missing_lang += 1 + if pa.get("hreflang_alternates"): + with_hreflang += 1 + return { + "pages_200": total, + "pages_missing_html_lang": missing_lang, + "pages_with_hreflang_links": with_hreflang, + } + + +def _validate_report_url_counts(report_data: dict[str, Any], df_row_count: int) -> None: + """Ensure crawled URL counts are consistent across report payload fields.""" + links = report_data.get("links") or [] + summary = report_data.get("summary") or {} + scope = (report_data.get("report_meta") or {}).get("crawl_scope") or {} + link_count = len(links) if isinstance(links, list) else 0 + total_urls = int(summary.get("total_urls") or 0) + pages_crawled = int(scope.get("pages_crawled") or 0) + counts = {link_count, total_urls, pages_crawled, df_row_count} + if len(counts) > 1: + msg = ( + f"report count mismatch: links={link_count}, " + f"summary.total_urls={total_urls}, " + f"pages_crawled={pages_crawled}, df_rows={df_row_count}" + ) + print(f" WARNING: {msg}", flush=True) + report_data.setdefault("ml_errors", []).append(msg) + + +def _build_report_metadata( + df: pd.DataFrame, + config: Optional[dict[str, str]], + lighthouse_summary: Optional[dict[str, Any]], + google_data: Optional[dict[str, Any]], + keywords_data: Optional[dict[str, Any]], + ml_bundle: dict[str, Any], + run_id: Optional[int], + crawl_run_created_at: Optional[str], + gsc_links_data: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + """Provenance and crawl scope for agency-facing audits.""" + sources: list[str] = ["crawl"] + if lighthouse_summary: + sources.append("lighthouse") + if google_data: + if google_data.get("gsc") or google_data.get("gsc_summary"): + sources.append("search_console") + if google_data.get("ga4") or google_data.get("ga4_summary"): + sources.append("analytics") + if gsc_links_data and "search_console" not in sources: + sources.append("search_console") + llm_meta = ml_bundle.get("llm_meta") + if isinstance(llm_meta, dict) and llm_meta.get("model"): + sources.append("ai") + kw_rows = (keywords_data or {}).get("rows") or [] + has_gsc_kw = any( + (r.get("gsc_impressions") or r.get("gsc_clicks")) and r.get("source") in ("gsc", "site+gsc", None) + for r in kw_rows[:500] + if isinstance(r, dict) + ) + if kw_rows and not has_gsc_kw and "estimated" not in sources: + sources.append("estimated") + + max_pages_cfg = get_int(config or {}, "max_pages", 0) or 0 + pages_crawled = len(df) + blocked = 0 + if not df.empty and "status" in df.columns: + blocked = int((df["status"].astype(str) == "blocked_by_robots").sum()) + + render_mode = (str((config or {}).get("crawl_render_mode") or "static")).strip().lower() + js_concurrency = get_int(config or {}, "crawl_js_concurrency", 3) or 3 + static_html_only = render_mode == "static" + + crawl_scope: dict[str, Any] = { + "pages_crawled": pages_crawled, + "max_pages_configured": max_pages_cfg or pages_crawled, + "robots_blocked_count": blocked, + "static_html_only": static_html_only, + "render_mode": render_mode, + "js_concurrency": js_concurrency if not static_html_only else None, + "crawl_limited": bool(max_pages_cfg and pages_crawled >= max_pages_cfg), + } + if not df.empty and "fetch_method" in df.columns: + fm = df["fetch_method"].astype(str).str.strip().str.lower() + pages_static = int((fm == "static").sum()) + pages_rendered = int((fm == "rendered").sum()) + if render_mode == "auto" or pages_rendered > 0: + crawl_scope["pages_static"] = pages_static + crawl_scope["pages_rendered"] = pages_rendered + + from ..crawl.fetchers.browser_diagnostics import aggregate_browser_diagnostics_df + + browser_agg = aggregate_browser_diagnostics_df(df) + if browser_agg and (render_mode != "static" or browser_agg.get("total_console_errors", 0) > 0): + crawl_scope["browser_diagnostics"] = browser_agg + + meta: dict[str, Any] = { + "data_sources": sources, + "generated_at": datetime.now(timezone.utc).isoformat(), + "crawl_scope": crawl_scope, + } + if run_id is not None: + meta["crawl_run_id"] = run_id + if crawl_run_created_at: + meta["crawl_run_created_at"] = crawl_run_created_at + if google_data: + meta["google_fetched_at"] = google_data.get("fetched_at") + meta["google_date_range_days"] = google_data.get("date_range_days") + gsc = google_data.get("gsc") or {} + if isinstance(gsc, dict) and gsc.get("row_count") is not None: + meta["gsc_row_count"] = gsc.get("row_count") + if keywords_data: + meta["keywords_enriched_at"] = keywords_data.get("enriched_at") or keywords_data.get("fetched_at") + if gsc_links_data: + meta["gsc_links_imported_at"] = gsc_links_data.get("imported_at") + meta["gsc_links_referring_domains"] = len(gsc_links_data.get("top_linking_sites") or []) + sample_n = len(gsc_links_data.get("sample_links") or []) + latest_n = len(gsc_links_data.get("latest_links") or []) + meta["gsc_links_sample_count"] = sample_n + latest_n + if isinstance(llm_meta, dict): + meta["llm"] = llm_meta + logo_url = (str((config or {}).get("export_logo_url") or "")).strip() + if logo_url: + meta["export_logo_url"] = logo_url + return meta diff --git a/src/website_profiling/reporting/seo_summary.py b/src/website_profiling/reporting/seo_summary.py new file mode 100644 index 00000000..bd49de98 --- /dev/null +++ b/src/website_profiling/reporting/seo_summary.py @@ -0,0 +1,166 @@ +"""SEO summary and issue computation for reports.""" +from __future__ import annotations + +import pandas as pd + +# SEO thresholds for recommendations +TITLE_LEN_MIN = 30 +TITLE_LEN_MAX = 60 +META_DESC_LEN_MIN = 70 +META_DESC_LEN_MAX = 160 +THIN_CONTENT_CHARS = 300 + +def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: + """Compute crawl summary, SEO health metrics, issues list, and recommendations from crawl DataFrame.""" + total = len(df) + status_str = df["status"].astype(str) if "status" in df.columns else pd.Series(["unknown"] * len(df)) + count_2xx = int((status_str.str.match(r"2\d{2}").fillna(False)).sum()) + count_3xx = int((status_str.str.match(r"3\d{2}").fillna(False)).sum()) + count_4xx = int((status_str.str.match(r"4\d{2}").fillna(False)).sum()) + count_5xx = int((status_str.str.match(r"5\d{2}").fillna(False)).sum()) + count_error = int((status_str.isin(["error", "blocked_by_robots"])).sum()) + success_rate = round(100 * count_2xx / total, 1) if total else 0 + + outlinks = ( + pd.to_numeric(df["outlinks"], errors="coerce").fillna(0).astype(int) + if "outlinks" in df.columns + else pd.Series([0] * len(df)) + ) + title_len = ( + df["title"].fillna("").astype(str).apply(len) + if "title" in df.columns + else pd.Series([0] * len(df)) + ) + crawl_time_s = float(df["crawl_time_s"].iloc[0]) if "crawl_time_s" in df.columns and len(df) else None + + summary = { + "total_urls": total, + "count_2xx": count_2xx, + "count_3xx": count_3xx, + "count_4xx": count_4xx, + "count_5xx": count_5xx, + "count_error": count_error, + "success_rate": success_rate, + "avg_outlinks": round(float(outlinks.mean()), 1) if total else 0, + "avg_title_len": round(float(title_len.mean()), 1) if total else 0, + "crawl_time_s": round(crawl_time_s, 1) if crawl_time_s is not None else None, + } + + # SEO health (when columns exist) + seo_health = {} + if "title" in df.columns: + titles = df["title"].fillna("").astype(str) + seo_health["missing_title"] = int((titles.str.len() == 0).sum()) + seo_health["title_short"] = int(((title_len > 0) & (title_len < TITLE_LEN_MIN)).sum()) + seo_health["title_long"] = int((title_len > TITLE_LEN_MAX).sum()) + seo_health["title_ok"] = int(((title_len >= TITLE_LEN_MIN) & (title_len <= TITLE_LEN_MAX)).sum()) + if "meta_description_len" in df.columns: + md_len = pd.to_numeric(df["meta_description_len"], errors="coerce").fillna(0).astype(int) + seo_health["missing_meta_desc"] = int((md_len == 0).sum()) + seo_health["meta_desc_short"] = int(((md_len > 0) & (md_len < META_DESC_LEN_MIN)).sum()) + seo_health["meta_desc_long"] = int((md_len > META_DESC_LEN_MAX).sum()) + seo_health["meta_desc_ok"] = int(((md_len >= META_DESC_LEN_MIN) & (md_len <= META_DESC_LEN_MAX)).sum()) + if "h1_count" in df.columns: + h1c = pd.to_numeric(df["h1_count"], errors="coerce").fillna(-1).astype(int) + seo_health["h1_zero"] = int((h1c == 0).sum()) + seo_health["h1_one"] = int((h1c == 1).sum()) + seo_health["h1_multi"] = int((h1c > 1).sum()) + if "content_length" in df.columns: + cl = pd.to_numeric(df["content_length"], errors="coerce").fillna(0).astype(int) + seo_health["thin_content"] = int(((cl > 0) & (cl < THIN_CONTENT_CHARS)).sum()) + + # Issues: broken, redirects, SEO + issues = {"broken": [], "redirects": [], "seo": []} + for _, row in df.iterrows(): + u = row.get("url") + if pd.isna(u) or not u: + continue + u = str(u).strip() + st = str(row.get("status", "")).strip() + if st.startswith("4") or st.startswith("5") or st in ("error", "blocked_by_robots"): + issues["broken"].append({"url": u, "status": st}) + elif st.startswith("3"): + final = row.get("final_url") or "" + issues["redirects"].append({"url": u, "status": st, "final_url": str(final) if pd.notna(final) else ""}) + + if "title" in df.columns: + for _, row in df.iterrows(): + u = row.get("url") + if pd.isna(u): + continue + u = str(u).strip() + t = row.get("title") or "" + tl = len(str(t).strip()) + if tl == 0: + issues["seo"].append({"type": "missing_title", "url": u, "message": "Missing title"}) + elif tl < TITLE_LEN_MIN: + issues["seo"].append({"type": "title_short", "url": u, "message": f"Title too short ({tl} chars)"}) + elif tl > TITLE_LEN_MAX: + issues["seo"].append({"type": "title_long", "url": u, "message": f"Title too long ({tl} chars)"}) + if "meta_description_len" in df.columns: + for _, row in df.iterrows(): + md_len = pd.to_numeric(row.get("meta_description_len"), errors="coerce") + if pd.isna(md_len) or md_len == 0: + continue + u = row.get("url") + if pd.isna(u): + continue + u = str(u).strip() + ml = int(md_len) + if ml < META_DESC_LEN_MIN: + issues["seo"].append({"type": "meta_desc_short", "url": u, "message": f"Meta description too short ({ml} chars)"}) + elif ml > META_DESC_LEN_MAX: + issues["seo"].append({"type": "meta_desc_long", "url": u, "message": f"Meta description too long ({ml} chars)"}) + if "h1_count" in df.columns: + for _, row in df.iterrows(): + h1c = pd.to_numeric(row.get("h1_count"), errors="coerce") + if pd.isna(h1c) or h1c == 1: + continue + u = row.get("url") + if pd.isna(u): + continue + u = str(u).strip() + if int(h1c) == 0: + issues["seo"].append({"type": "h1_missing", "url": u, "message": "Missing H1"}) + else: + issues["seo"].append({"type": "h1_multi", "url": u, "message": f"Multiple H1s ({int(h1c)})"}) + if "content_length" in df.columns: + for _, row in df.iterrows(): + cl = pd.to_numeric(row.get("content_length"), errors="coerce") + cl = 0 if pd.isna(cl) else int(cl) + if cl >= THIN_CONTENT_CHARS or cl == 0: + continue + u = row.get("url") + if pd.isna(u): + continue + issues["seo"].append({"type": "thin_content", "url": str(u).strip(), "message": f"Thin content ({int(cl)} chars)"}) + + # Recommendations (actionable bullets) + recommendations = [] + if issues["broken"]: + recommendations.append(f"Fix {len(issues['broken'])} broken or error URL(s).") + if issues["redirects"]: + recommendations.append(f"Review {len(issues['redirects'])} redirect(s); consolidate if possible.") + if seo_health.get("missing_title", 0) > 0: + recommendations.append(f"Add titles to {seo_health['missing_title']} page(s).") + if seo_health.get("title_short", 0) + seo_health.get("title_long", 0) > 0: + n = seo_health.get("title_short", 0) + seo_health.get("title_long", 0) + recommendations.append(f"Optimize title length on {n} page(s) (aim 30–60 chars).") + if seo_health.get("missing_meta_desc", 0) > 0: + recommendations.append(f"Add meta descriptions to {seo_health['missing_meta_desc']} page(s).") + if seo_health.get("meta_desc_short", 0) + seo_health.get("meta_desc_long", 0) > 0: + n = seo_health.get("meta_desc_short", 0) + seo_health.get("meta_desc_long", 0) + recommendations.append(f"Optimize meta description length on {n} page(s) (aim 70–160 chars).") + if seo_health.get("h1_zero", 0) > 0: + recommendations.append(f"Add one H1 per page on {seo_health['h1_zero']} page(s).") + if seo_health.get("h1_multi", 0) > 0: + recommendations.append(f"Use a single H1 per page on {seo_health['h1_multi']} page(s).") + if seo_health.get("thin_content", 0) > 0: + recommendations.append(f"Expand thin content on {seo_health['thin_content']} page(s) (under {THIN_CONTENT_CHARS} chars).") + + return { + "summary": summary, + "seo_health": seo_health, + "issues": issues, + "recommendations": recommendations, + } diff --git a/src/website_profiling/reporting/site_level.py b/src/website_profiling/reporting/site_level.py new file mode 100644 index 00000000..0c398334 --- /dev/null +++ b/src/website_profiling/reporting/site_level.py @@ -0,0 +1,49 @@ +"""Site-level file checks (robots, sitemap, ads.txt).""" +from __future__ import annotations + +from typing import Any +from urllib.parse import urlparse + +import requests + +def _fetch_site_level(start_url: str, timeout: int = 8) -> dict: + """Fetch robots.txt, sitemap.xml, ads.txt, and security.txt from start_url origin.""" + from .site_files import fetch_ads_txt, fetch_security_txt, merge_site_file_fields + + parsed = urlparse(start_url) + if not parsed.scheme or not parsed.netloc: + return { + "robots_present": False, + "sitemap_present": False, + "sitemap_valid": False, + "ads_txt_present": False, + "security_txt_present": False, + } + base = f"{parsed.scheme}://{parsed.netloc}" + session = requests.Session() + session.headers.update({"User-Agent": "WebsiteProfiling/1.0"}) + out: dict[str, Any] = { + "robots_present": False, + "sitemap_present": False, + "sitemap_valid": False, + } + try: + r = session.get(f"{base}/robots.txt", timeout=timeout) + if r.status_code == 200 and r.text: + out["robots_present"] = True + for line in r.text.splitlines(): + line = line.strip() + if line.lower().startswith("sitemap:"): + break + except Exception: + pass + try: + r = session.get(f"{base}/sitemap.xml", timeout=timeout) + if r.status_code == 200 and r.text: + out["sitemap_present"] = True + out["sitemap_valid"] = "<" in r.text and ">" in r.text and ("urlset" in r.text or "sitemapindex" in r.text) + except Exception: + pass + merge_site_file_fields(out, fetch_ads_txt(session, base, timeout=timeout)) + merge_site_file_fields(out, fetch_security_txt(session, base, timeout=timeout)) + return out diff --git a/src/website_profiling/tools/export_audit.py b/src/website_profiling/tools/export_audit.py index 599a4c78..03cfe76b 100644 --- a/src/website_profiling/tools/export_audit.py +++ b/src/website_profiling/tools/export_audit.py @@ -6,26 +6,36 @@ import io import json from datetime import datetime, timezone -from typing import Any, Optional +from typing import Optional from ..db import db_session, read_report_payload from ..reporting.terminology import category_display_name - -_GLOSSARY_ROWS: list[tuple[str, str]] = [ - ("Crawl", "URLs fetched by the site spider (status codes, titles, inlinks)."), - ("Lighthouse", "Lab Core Web Vitals audit (LCP, CLS, TBT, and category scores)."), - ("Google Search Console", "Queries, pages, clicks, impressions, and average position from GSC."), - ("Google Analytics 4", "Sessions, users, and engagement from GA4."), - ("Estimated", "Derived from crawl text only — not Google search volume or rankings."), - ("AI insights", "Optional LLM summaries — verify before client delivery."), -] - -_ISSUE_LIMIT_HTML = 200 -_ISSUE_LIMIT_PDF = 80 -_LINK_LIMIT = 50 - - -def _load_payload(report_id: Optional[int] = None) -> dict[str, Any]: +from .export_audit_data import ( + _GLOSSARY_ROWS, + _ISSUE_LIMIT_HTML, + _ISSUE_LIMIT_PDF, + _LINK_LIMIT, + _executive_export_data, + _executive_source_label, + _format_report_date, + _issue_priority_counts, + _issue_recommendation, + _issues_rows, + _overall_score, + _priority_sort_key, + _score_band, + _summary_lines, +) +from .export_audit_html import ( + _category_cards_html, + _executive_summary_html, + _priority_stats_html, + _report_html_styles, +) + + +def _load_payload(report_id: Optional[int] = None) -> dict: + """Load report payload from DB (uses module-level db_session for test patches).""" with db_session() as conn: payload = read_report_payload(conn, report_id) if not payload: @@ -33,522 +43,6 @@ def _load_payload(report_id: Optional[int] = None) -> dict[str, Any]: return payload -def _issue_recommendation(issue: dict[str, Any]) -> tuple[str, str]: - """Return (display recommendation, llm_recommendation if distinct).""" - rule = str(issue.get("recommendation") or "").strip() - llm = str(issue.get("llm_recommendation") or "").strip() - if llm and llm != rule: - display = llm if llm else rule - return display, llm - return llm or rule, llm - - -def _issues_rows(payload: dict[str, Any]) -> list[dict[str, str]]: - rows: list[dict[str, str]] = [] - for cat in payload.get("categories") or []: - if not isinstance(cat, dict): - continue - cat_name = str(cat.get("name") or "") - ui_name = category_display_name(cat_name) - for issue in cat.get("issues") or []: - if not isinstance(issue, dict): - continue - rec, llm_rec = _issue_recommendation(issue) - rows.append({ - "category": ui_name, - "priority": str(issue.get("priority") or ""), - "message": str(issue.get("message") or ""), - "url": str(issue.get("url") or ""), - "recommendation": rec, - "llm_recommendation": llm_rec, - }) - return rows - - -def _executive_export_data(payload: dict[str, Any]) -> dict[str, Any]: - """Normalize executive_summary and legacy recommendations for export.""" - exec_sum = payload.get("executive_summary") - summary = "" - priorities: list[str] = [] - top_issues: list[dict[str, Any]] = [] - source = "" - if isinstance(exec_sum, dict): - summary = str(exec_sum.get("summary") or "").strip() - source = str(exec_sum.get("source") or "").strip() - raw_pri = exec_sum.get("priorities") or [] - if isinstance(raw_pri, list): - priorities = [str(p).strip() for p in raw_pri if str(p).strip()] - raw_top = exec_sum.get("top_issues") or [] - if isinstance(raw_top, list): - top_issues = [i for i in raw_top if isinstance(i, dict)][:8] - - legacy_recs = payload.get("recommendations") or [] - legacy_list: list[str] = [] - if isinstance(legacy_recs, list): - legacy_list = [str(r).strip() for r in legacy_recs if str(r).strip()] - - if not summary and legacy_list: - summary = "\n".join(f"• {r}" for r in legacy_list[:12]) - - return { - "summary": summary, - "priorities": priorities, - "top_issues": top_issues, - "source": source, - "legacy_recommendations": legacy_list, - } - - -def _executive_source_label(source: str) -> str: - if source == "ai_insights": - return "AI insights" - if source == "deterministic": - return "Measured + Search Console" - return source or "Audit data" - - -def _executive_summary_html(payload: dict[str, Any]) -> str: - data = _executive_export_data(payload) - if not data["summary"] and not data["priorities"] and not data["top_issues"]: - return "" - - parts: list[str] = ['

Executive summary

'] - if data["source"]: - parts.append( - f'

Source: {html.escape(_executive_source_label(data["source"]))}

' - ) - if data["summary"]: - summary_html = html.escape(data["summary"]).replace("\n", "
") - parts.append(f'

{summary_html}

') - - if data["priorities"]: - pri_items = "".join(f"
  • {html.escape(p)}
    • " for p in data["priorities"][:8]) - parts.append(f"

    Priorities

      {pri_items}
    ") - - if data["top_issues"]: - rows = "" - for iss in data["top_issues"]: - pri = str(iss.get("priority") or "").lower() - badge_cls = f"badge-{pri}" if pri in {"critical", "high", "medium", "low"} else "badge-low" - clicks = iss.get("gsc_clicks") - clicks_txt = "" - if clicks is not None: - try: - if float(clicks) > 0: - clicks_txt = f' · {int(float(clicks))} GSC clicks' - except (TypeError, ValueError): - pass - rows += ( - "" - f"{html.escape(str(iss.get('priority') or ''))}" - f"{html.escape(str(iss.get('message') or ''))}" - f"{html.escape(str(iss.get('url') or ''))}" - f"{html.escape(clicks_txt.lstrip(' · ') if clicks_txt else '—')}" - "" - ) - parts.append( - "

    Top traffic-impacting issues

    " - '' - "" - f"{rows}
    PriorityIssueURLGSC clicks
    " - ) - - parts.append("
    ") - return "".join(parts) - - -def _priority_sort_key(row: dict[str, str]) -> int: - order = {"critical": 0, "high": 1, "medium": 2, "low": 3} - return order.get(row["priority"].lower(), 9) - - -def _summary_lines(payload: dict[str, Any]) -> list[tuple[str, str]]: - lines: list[tuple[str, str]] = [] - site = str(payload.get("site_name") or "Site") - lines.append(("Property", site)) - if payload.get("report_generated_at"): - lines.append(("Report generated", str(payload["report_generated_at"]))) - meta = payload.get("report_meta") or {} - if isinstance(meta, dict): - sources = meta.get("data_sources") or [] - if sources: - lines.append(("Data sources", ", ".join(str(s) for s in sources))) - scope = meta.get("crawl_scope") or {} - if isinstance(scope, dict) and scope.get("pages_crawled") is not None: - pages = scope.get("pages_crawled") - max_p = scope.get("max_pages_configured") - scope_txt = f"{pages} pages crawled" - if max_p: - scope_txt += f" (limit {max_p})" - if scope.get("crawl_limited"): - scope_txt += " — crawl limit reached" - render_mode = scope.get("render_mode") - if render_mode == "javascript": - js_c = scope.get("js_concurrency") - scope_txt += " — JavaScript rendering" - if js_c: - scope_txt += f" ({js_c} parallel pages)" - elif render_mode == "auto": - scope_txt += " — auto rendering (static + JS fallback)" - ps = scope.get("pages_static") - pr = scope.get("pages_rendered") - if ps is not None and pr is not None: - scope_txt += f" ({ps} static, {pr} JavaScript-rendered)" - elif scope.get("static_html_only"): - scope_txt += " — static HTML only" - lines.append(("Crawl scope", scope_txt)) - browser_diag = scope.get("browser_diagnostics") - if isinstance(browser_diag, dict): - pce = browser_diag.get("pages_with_console_errors") - tce = browser_diag.get("total_console_errors") - ppe = browser_diag.get("pages_with_page_errors") - if pce or ppe: - parts = [] - if pce: - parts.append(f"{pce} page(s) with console errors ({tce or 0} total)") - if ppe: - parts.append(f"{ppe} page(s) with uncaught JS errors") - lines.append(("Browser diagnostics", "; ".join(parts))) - if meta.get("google_fetched_at"): - lines.append(("Google data fetched", str(meta["google_fetched_at"]))) - summary = payload.get("summary") or {} - if isinstance(summary, dict): - for key, label in ( - ("total_urls", "URLs in crawl"), - ("indexable", "Indexable URLs"), - ("issues_count", "Total issues"), - ("critical_issues", "Critical issues"), - ): - if summary.get(key) is not None: - lines.append((label, str(summary[key]))) - status = payload.get("status_counts") or {} - if isinstance(status, dict) and status: - parts = [f"{k}: {v}" for k, v in sorted(status.items(), key=lambda x: -int(x[1] or 0))[:8]] - lines.append(("HTTP status mix", ", ".join(parts))) - return lines - - -def _format_report_date(value: str) -> str: - if not value: - return "—" - try: - dt = datetime.fromisoformat(value.replace("Z", "+00:00")) - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc).strftime("%d %B %Y, %H:%M UTC") - except ValueError: - return value - - -def _overall_score(payload: dict[str, Any]) -> Optional[int]: - scores: list[float] = [] - for cat in payload.get("categories") or []: - if not isinstance(cat, dict): - continue - raw = cat.get("score") - if raw is None: - continue - try: - scores.append(float(raw)) - except (TypeError, ValueError): - continue - if not scores: - return None - return int(round(sum(scores) / len(scores))) - - -def _score_band(score: Optional[float]) -> tuple[str, str]: - if score is None: - return "—", "score-na" - rounded = int(round(score)) - if rounded >= 80: - return str(rounded), "score-good" - if rounded >= 60: - return str(rounded), "score-fair" - return str(rounded), "score-poor" - - -def _issue_priority_counts(rows: list[dict[str, str]]) -> dict[str, int]: - counts = {"critical": 0, "high": 0, "medium": 0, "low": 0} - for row in rows: - key = row["priority"].lower() - if key in counts: - counts[key] += 1 - return counts - - -def _category_cards_html(categories: Any) -> str: - cards: list[str] = [] - for cat in categories or []: - if not isinstance(cat, dict): - continue - name = html.escape(category_display_name(str(cat.get("name") or "Category"))) - score_val: float | None = None - if cat.get("score") is not None: - try: - score_val = float(cat["score"]) - except (TypeError, ValueError): - score_val = None - score_txt, score_cls = _score_band(score_val) - issue_n = len(cat.get("issues") or []) - cards.append( - f'
    ' - f'
    {score_txt}
    ' - f'
    {name}
    ' - f'
    {issue_n} issue{"s" if issue_n != 1 else ""}
    ' - f"
    " - ) - return "".join(cards) or '

    No category scores available.

    ' - - -def _priority_stats_html(counts: dict[str, int]) -> str: - labels = ( - ("critical", "Critical"), - ("high", "High"), - ("medium", "Medium"), - ("low", "Low"), - ) - parts: list[str] = [] - for key, label in labels: - n = counts.get(key, 0) - parts.append( - f'
    ' - f'{n}' - f'{label}' - f"
    " - ) - return "".join(parts) - - -def _report_html_styles() -> str: - return """ - :root { - --ink: #0f172a; - --muted: #64748b; - --line: #e2e8f0; - --surface: #ffffff; - --surface-muted: #f8fafc; - --brand: #0b0f19; - --brand-accent: #2563eb; - --good: #059669; - --good-bg: #ecfdf5; - --fair: #d97706; - --fair-bg: #fffbeb; - --poor: #dc2626; - --poor-bg: #fef2f2; - --critical: #991b1b; - --high: #c2410c; - --medium: #a16207; - --low: #475569; - } - * { box-sizing: border-box; } - body { - margin: 0; - background: #eef2f7; - color: var(--ink); - font: 400 15px/1.55 "Segoe UI", system-ui, -apple-system, sans-serif; - } - .report { max-width: 920px; margin: 0 auto; background: var(--surface); } - .cover { - background: linear-gradient(135deg, #0b0f19 0%, #111827 55%, #1e3a5f 100%); - color: #f8fafc; - padding: 2.5rem 2.75rem 2rem; - } - .cover-brand { - font-size: 0.72rem; - letter-spacing: 0.14em; - text-transform: uppercase; - color: #93c5fd; - font-weight: 700; - margin-bottom: 1rem; - } - .cover h1 { - margin: 0; - font-size: clamp(1.6rem, 4vw, 2.1rem); - font-weight: 700; - line-height: 1.15; - } - .cover-subtitle { - margin: 0.5rem 0 0; - color: #cbd5e1; - font-size: 1rem; - } - .cover-meta { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); - gap: 0.75rem 1.5rem; - margin-top: 1.75rem; - padding-top: 1.25rem; - border-top: 1px solid rgba(255,255,255,0.12); - font-size: 0.82rem; - } - .cover-meta dt { color: #94a3b8; margin: 0 0 0.15rem; font-weight: 500; } - .cover-meta dd { margin: 0; color: #f1f5f9; font-weight: 600; } - .content { padding: 2rem 2.75rem 2.5rem; } - .hero-score { - display: flex; - flex-wrap: wrap; - align-items: center; - gap: 1.25rem 2rem; - padding: 1.25rem 1.5rem; - border: 1px solid var(--line); - border-radius: 12px; - background: var(--surface-muted); - margin-bottom: 1.75rem; - } - .hero-score-ring { - width: 88px; - height: 88px; - border-radius: 50%; - display: grid; - place-items: center; - font-size: 1.65rem; - font-weight: 800; - border: 4px solid currentColor; - flex-shrink: 0; - } - .hero-score-ring.score-good { color: var(--good); background: var(--good-bg); } - .hero-score-ring.score-fair { color: var(--fair); background: var(--fair-bg); } - .hero-score-ring.score-poor { color: var(--poor); background: var(--poor-bg); } - .hero-score-ring.score-na { color: var(--muted); background: #f1f5f9; border-color: #cbd5e1; } - .hero-score-copy h2 { margin: 0 0 0.35rem; font-size: 1.05rem; } - .hero-score-copy p { margin: 0; color: var(--muted); font-size: 0.92rem; } - .stats-row { - display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); - gap: 0.75rem; - margin-bottom: 1.75rem; - } - .stat { - border: 1px solid var(--line); - border-radius: 10px; - padding: 0.85rem 0.75rem; - text-align: center; - background: var(--surface); - } - .stat-value { display: block; font-size: 1.35rem; font-weight: 800; line-height: 1.1; } - .stat-label { - display: block; - margin-top: 0.25rem; - font-size: 0.72rem; - text-transform: uppercase; - letter-spacing: 0.06em; - color: var(--muted); - font-weight: 600; - } - .stat-critical .stat-value { color: var(--critical); } - .stat-high .stat-value { color: var(--high); } - .stat-medium .stat-value { color: var(--medium); } - .stat-low .stat-value { color: var(--low); } - .score-grid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); - gap: 0.75rem; - margin-bottom: 1.75rem; - } - .score-card { - border: 1px solid var(--line); - border-radius: 10px; - padding: 0.9rem 0.75rem; - background: var(--surface); - } - .score-card .score-value { font-size: 1.5rem; font-weight: 800; line-height: 1; } - .score-card .score-name { margin-top: 0.45rem; font-size: 0.78rem; font-weight: 600; line-height: 1.25; } - .score-card .score-meta { margin-top: 0.25rem; font-size: 0.72rem; color: var(--muted); } - .score-card.score-good .score-value { color: var(--good); } - .score-card.score-fair .score-value { color: var(--fair); } - .score-card.score-poor .score-value { color: var(--poor); } - .score-card.score-na .score-value { color: var(--muted); } - section { margin-bottom: 2rem; page-break-inside: avoid; } - section h2 { - margin: 0 0 0.85rem; - font-size: 1rem; - font-weight: 700; - letter-spacing: 0.02em; - text-transform: uppercase; - color: var(--ink); - padding-bottom: 0.45rem; - border-bottom: 2px solid var(--brand); - } - .callout { - border-left: 4px solid var(--brand-accent); - background: #eff6ff; - padding: 1rem 1.15rem; - border-radius: 0 10px 10px 0; - margin-bottom: 0.5rem; - } - .callout ul { margin: 0; padding-left: 1.15rem; } - .callout li { margin: 0.35rem 0; } - table.data { - width: 100%; - border-collapse: collapse; - font-size: 0.84rem; - border: 1px solid var(--line); - border-radius: 10px; - overflow: hidden; - } - table.data th, - table.data td { - padding: 0.55rem 0.65rem; - text-align: left; - vertical-align: top; - border-bottom: 1px solid var(--line); - } - table.data th { - background: var(--surface-muted); - font-size: 0.72rem; - text-transform: uppercase; - letter-spacing: 0.05em; - color: var(--muted); - font-weight: 700; - } - table.data tbody tr:last-child td { border-bottom: none; } - table.data tbody tr:nth-child(even) td { background: #fcfdff; } - table.kv th { - width: 34%; - font-weight: 600; - color: var(--ink); - background: var(--surface-muted); - } - .url { word-break: break-all; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.78rem; } - .badge { - display: inline-block; - padding: 0.15rem 0.5rem; - border-radius: 999px; - font-size: 0.68rem; - font-weight: 700; - text-transform: uppercase; - letter-spacing: 0.04em; - } - .badge-critical { background: #fee2e2; color: var(--critical); } - .badge-high { background: #ffedd5; color: var(--high); } - .badge-medium { background: #fef3c7; color: var(--medium); } - .badge-low { background: #f1f5f9; color: var(--low); } - .muted { color: var(--muted); font-size: 0.86rem; margin: 0.35rem 0 0.75rem; } - .report-footer { - border-top: 1px solid var(--line); - padding: 1.25rem 2.75rem 2rem; - color: var(--muted); - font-size: 0.78rem; - line-height: 1.5; - } - @media print { - body { background: #fff; } - .report { max-width: none; } - .cover { -webkit-print-color-adjust: exact; print-color-adjust: exact; } - .content { padding: 1.2cm 1.4cm; } - section { page-break-inside: auto; } - table.data { page-break-inside: auto; } - table.data tr { page-break-inside: avoid; } - .report-footer { padding-left: 1.4cm; padding-right: 1.4cm; } - } - @media (max-width: 640px) { - .cover, .content, .report-footer { padding-left: 1.25rem; padding-right: 1.25rem; } - .stats-row { grid-template-columns: repeat(2, minmax(0, 1fr)); } - } -""" - - def export_audit_csv(report_id: Optional[int] = None) -> str: payload = _load_payload(report_id) buf = io.StringIO() diff --git a/src/website_profiling/tools/export_audit_data.py b/src/website_profiling/tools/export_audit_data.py new file mode 100644 index 00000000..56974a47 --- /dev/null +++ b/src/website_profiling/tools/export_audit_data.py @@ -0,0 +1,216 @@ +"""Audit export data helpers.""" +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any, Optional + +from ..reporting.terminology import category_display_name + +_GLOSSARY_ROWS: list[tuple[str, str]] = [ + ("Crawl", "URLs fetched by the site spider (status codes, titles, inlinks)."), + ("Lighthouse", "Lab Core Web Vitals audit (LCP, CLS, TBT, and category scores)."), + ("Google Search Console", "Queries, pages, clicks, impressions, and average position from GSC."), + ("Google Analytics 4", "Sessions, users, and engagement from GA4."), + ("Estimated", "Derived from crawl text only — not Google search volume or rankings."), + ("AI insights", "Optional LLM summaries — verify before client delivery."), +] + +_ISSUE_LIMIT_HTML = 200 +_ISSUE_LIMIT_PDF = 80 +_LINK_LIMIT = 50 + + +def _issue_recommendation(issue: dict[str, Any]) -> tuple[str, str]: + """Return (display recommendation, llm_recommendation if distinct).""" + rule = str(issue.get("recommendation") or "").strip() + llm = str(issue.get("llm_recommendation") or "").strip() + if llm and llm != rule: + display = llm if llm else rule + return display, llm + return llm or rule, llm + + +def _issues_rows(payload: dict[str, Any]) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + cat_name = str(cat.get("name") or "") + ui_name = category_display_name(cat_name) + for issue in cat.get("issues") or []: + if not isinstance(issue, dict): + continue + rec, llm_rec = _issue_recommendation(issue) + rows.append({ + "category": ui_name, + "priority": str(issue.get("priority") or ""), + "message": str(issue.get("message") or ""), + "url": str(issue.get("url") or ""), + "recommendation": rec, + "llm_recommendation": llm_rec, + }) + return rows + + +def _executive_export_data(payload: dict[str, Any]) -> dict[str, Any]: + """Normalize executive_summary and legacy recommendations for export.""" + exec_sum = payload.get("executive_summary") + summary = "" + priorities: list[str] = [] + top_issues: list[dict[str, Any]] = [] + source = "" + if isinstance(exec_sum, dict): + summary = str(exec_sum.get("summary") or "").strip() + source = str(exec_sum.get("source") or "").strip() + raw_pri = exec_sum.get("priorities") or [] + if isinstance(raw_pri, list): + priorities = [str(p).strip() for p in raw_pri if str(p).strip()] + raw_top = exec_sum.get("top_issues") or [] + if isinstance(raw_top, list): + top_issues = [i for i in raw_top if isinstance(i, dict)][:8] + + legacy_recs = payload.get("recommendations") or [] + legacy_list: list[str] = [] + if isinstance(legacy_recs, list): + legacy_list = [str(r).strip() for r in legacy_recs if str(r).strip()] + + if not summary and legacy_list: + summary = "\n".join(f"• {r}" for r in legacy_list[:12]) + + return { + "summary": summary, + "priorities": priorities, + "top_issues": top_issues, + "source": source, + "legacy_recommendations": legacy_list, + } + + +def _executive_source_label(source: str) -> str: + if source == "ai_insights": + return "AI insights" + if source == "deterministic": + return "Measured + Search Console" + return source or "Audit data" + +def _priority_sort_key(row: dict[str, str]) -> int: + order = {"critical": 0, "high": 1, "medium": 2, "low": 3} + return order.get(row["priority"].lower(), 9) + + +def _summary_lines(payload: dict[str, Any]) -> list[tuple[str, str]]: + lines: list[tuple[str, str]] = [] + site = str(payload.get("site_name") or "Site") + lines.append(("Property", site)) + if payload.get("report_generated_at"): + lines.append(("Report generated", str(payload["report_generated_at"]))) + meta = payload.get("report_meta") or {} + if isinstance(meta, dict): + sources = meta.get("data_sources") or [] + if sources: + lines.append(("Data sources", ", ".join(str(s) for s in sources))) + scope = meta.get("crawl_scope") or {} + if isinstance(scope, dict) and scope.get("pages_crawled") is not None: + pages = scope.get("pages_crawled") + max_p = scope.get("max_pages_configured") + scope_txt = f"{pages} pages crawled" + if max_p: + scope_txt += f" (limit {max_p})" + if scope.get("crawl_limited"): + scope_txt += " — crawl limit reached" + render_mode = scope.get("render_mode") + if render_mode == "javascript": + js_c = scope.get("js_concurrency") + scope_txt += " — JavaScript rendering" + if js_c: + scope_txt += f" ({js_c} parallel pages)" + elif render_mode == "auto": + scope_txt += " — auto rendering (static + JS fallback)" + ps = scope.get("pages_static") + pr = scope.get("pages_rendered") + if ps is not None and pr is not None: + scope_txt += f" ({ps} static, {pr} JavaScript-rendered)" + elif scope.get("static_html_only"): + scope_txt += " — static HTML only" + lines.append(("Crawl scope", scope_txt)) + browser_diag = scope.get("browser_diagnostics") + if isinstance(browser_diag, dict): + pce = browser_diag.get("pages_with_console_errors") + tce = browser_diag.get("total_console_errors") + ppe = browser_diag.get("pages_with_page_errors") + if pce or ppe: + parts = [] + if pce: + parts.append(f"{pce} page(s) with console errors ({tce or 0} total)") + if ppe: + parts.append(f"{ppe} page(s) with uncaught JS errors") + lines.append(("Browser diagnostics", "; ".join(parts))) + if meta.get("google_fetched_at"): + lines.append(("Google data fetched", str(meta["google_fetched_at"]))) + summary = payload.get("summary") or {} + if isinstance(summary, dict): + for key, label in ( + ("total_urls", "URLs in crawl"), + ("indexable", "Indexable URLs"), + ("issues_count", "Total issues"), + ("critical_issues", "Critical issues"), + ): + if summary.get(key) is not None: + lines.append((label, str(summary[key]))) + status = payload.get("status_counts") or {} + if isinstance(status, dict) and status: + parts = [f"{k}: {v}" for k, v in sorted(status.items(), key=lambda x: -int(x[1] or 0))[:8]] + lines.append(("HTTP status mix", ", ".join(parts))) + return lines + + +def _format_report_date(value: str) -> str: + if not value: + return "—" + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).strftime("%d %B %Y, %H:%M UTC") + except ValueError: + return value + + +def _overall_score(payload: dict[str, Any]) -> Optional[int]: + scores: list[float] = [] + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + raw = cat.get("score") + if raw is None: + continue + try: + scores.append(float(raw)) + except (TypeError, ValueError): + continue + if not scores: + return None + return int(round(sum(scores) / len(scores))) + + +def _score_band(score: Optional[float]) -> tuple[str, str]: + if score is None: + return "—", "score-na" + rounded = int(round(score)) + if rounded >= 80: + return str(rounded), "score-good" + if rounded >= 60: + return str(rounded), "score-fair" + return str(rounded), "score-poor" + + +def _issue_priority_counts(rows: list[dict[str, str]]) -> dict[str, int]: + counts = {"critical": 0, "high": 0, "medium": 0, "low": 0} + for row in rows: + key = row["priority"].lower() + if key in counts: + counts[key] += 1 + return counts + + diff --git a/src/website_profiling/tools/export_audit_html.py b/src/website_profiling/tools/export_audit_html.py new file mode 100644 index 00000000..2232e6b2 --- /dev/null +++ b/src/website_profiling/tools/export_audit_html.py @@ -0,0 +1,343 @@ +"""Audit export HTML generation.""" +from __future__ import annotations + +import html +from typing import Any, Optional + +from ..reporting.terminology import category_display_name +from .export_audit_data import ( + _GLOSSARY_ROWS, + _ISSUE_LIMIT_HTML, + _ISSUE_LIMIT_PDF, + _LINK_LIMIT, + _executive_export_data, + _executive_source_label, + _format_report_date, + _issue_priority_counts, + _issues_rows, + _overall_score, + _priority_sort_key, + _score_band, + _summary_lines, +) + +def _executive_summary_html(payload: dict[str, Any]) -> str: + data = _executive_export_data(payload) + if not data["summary"] and not data["priorities"] and not data["top_issues"]: + return "" + + parts: list[str] = ['

    Executive summary

    '] + if data["source"]: + parts.append( + f'

    Source: {html.escape(_executive_source_label(data["source"]))}

    ' + ) + if data["summary"]: + summary_html = html.escape(data["summary"]).replace("\n", "
    ") + parts.append(f'

    {summary_html}

    ') + + if data["priorities"]: + pri_items = "".join(f"
  • {html.escape(p)}
    • " for p in data["priorities"][:8]) + parts.append(f"

    Priorities

      {pri_items}
    ") + + if data["top_issues"]: + rows = "" + for iss in data["top_issues"]: + pri = str(iss.get("priority") or "").lower() + badge_cls = f"badge-{pri}" if pri in {"critical", "high", "medium", "low"} else "badge-low" + clicks = iss.get("gsc_clicks") + clicks_txt = "" + if clicks is not None: + try: + if float(clicks) > 0: + clicks_txt = f' · {int(float(clicks))} GSC clicks' + except (TypeError, ValueError): + pass + rows += ( + "" + f"{html.escape(str(iss.get('priority') or ''))}" + f"{html.escape(str(iss.get('message') or ''))}" + f"{html.escape(str(iss.get('url') or ''))}" + f"{html.escape(clicks_txt.lstrip(' · ') if clicks_txt else '—')}" + "" + ) + parts.append( + "

    Top traffic-impacting issues

    " + '' + "" + f"{rows}
    PriorityIssueURLGSC clicks
    " + ) + + parts.append("
    ") + return "".join(parts) + + +def _category_cards_html(categories: Any) -> str: + cards: list[str] = [] + for cat in categories or []: + if not isinstance(cat, dict): + continue + name = html.escape(category_display_name(str(cat.get("name") or "Category"))) + score_val: float | None = None + if cat.get("score") is not None: + try: + score_val = float(cat["score"]) + except (TypeError, ValueError): + score_val = None + score_txt, score_cls = _score_band(score_val) + issue_n = len(cat.get("issues") or []) + cards.append( + f'
    ' + f'
    {score_txt}
    ' + f'
    {name}
    ' + f'
    {issue_n} issue{"s" if issue_n != 1 else ""}
    ' + f"
    " + ) + return "".join(cards) or '

    No category scores available.

    ' + + +def _priority_stats_html(counts: dict[str, int]) -> str: + labels = ( + ("critical", "Critical"), + ("high", "High"), + ("medium", "Medium"), + ("low", "Low"), + ) + parts: list[str] = [] + for key, label in labels: + n = counts.get(key, 0) + parts.append( + f'
    ' + f'{n}' + f'{label}' + f"
    " + ) + return "".join(parts) + + +def _report_html_styles() -> str: + return """ + :root { + --ink: #0f172a; + --muted: #64748b; + --line: #e2e8f0; + --surface: #ffffff; + --surface-muted: #f8fafc; + --brand: #0b0f19; + --brand-accent: #2563eb; + --good: #059669; + --good-bg: #ecfdf5; + --fair: #d97706; + --fair-bg: #fffbeb; + --poor: #dc2626; + --poor-bg: #fef2f2; + --critical: #991b1b; + --high: #c2410c; + --medium: #a16207; + --low: #475569; + } + * { box-sizing: border-box; } + body { + margin: 0; + background: #eef2f7; + color: var(--ink); + font: 400 15px/1.55 "Segoe UI", system-ui, -apple-system, sans-serif; + } + .report { max-width: 920px; margin: 0 auto; background: var(--surface); } + .cover { + background: linear-gradient(135deg, #0b0f19 0%, #111827 55%, #1e3a5f 100%); + color: #f8fafc; + padding: 2.5rem 2.75rem 2rem; + } + .cover-brand { + font-size: 0.72rem; + letter-spacing: 0.14em; + text-transform: uppercase; + color: #93c5fd; + font-weight: 700; + margin-bottom: 1rem; + } + .cover h1 { + margin: 0; + font-size: clamp(1.6rem, 4vw, 2.1rem); + font-weight: 700; + line-height: 1.15; + } + .cover-subtitle { + margin: 0.5rem 0 0; + color: #cbd5e1; + font-size: 1rem; + } + .cover-meta { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 0.75rem 1.5rem; + margin-top: 1.75rem; + padding-top: 1.25rem; + border-top: 1px solid rgba(255,255,255,0.12); + font-size: 0.82rem; + } + .cover-meta dt { color: #94a3b8; margin: 0 0 0.15rem; font-weight: 500; } + .cover-meta dd { margin: 0; color: #f1f5f9; font-weight: 600; } + .content { padding: 2rem 2.75rem 2.5rem; } + .hero-score { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 1.25rem 2rem; + padding: 1.25rem 1.5rem; + border: 1px solid var(--line); + border-radius: 12px; + background: var(--surface-muted); + margin-bottom: 1.75rem; + } + .hero-score-ring { + width: 88px; + height: 88px; + border-radius: 50%; + display: grid; + place-items: center; + font-size: 1.65rem; + font-weight: 800; + border: 4px solid currentColor; + flex-shrink: 0; + } + .hero-score-ring.score-good { color: var(--good); background: var(--good-bg); } + .hero-score-ring.score-fair { color: var(--fair); background: var(--fair-bg); } + .hero-score-ring.score-poor { color: var(--poor); background: var(--poor-bg); } + .hero-score-ring.score-na { color: var(--muted); background: #f1f5f9; border-color: #cbd5e1; } + .hero-score-copy h2 { margin: 0 0 0.35rem; font-size: 1.05rem; } + .hero-score-copy p { margin: 0; color: var(--muted); font-size: 0.92rem; } + .stats-row { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 0.75rem; + margin-bottom: 1.75rem; + } + .stat { + border: 1px solid var(--line); + border-radius: 10px; + padding: 0.85rem 0.75rem; + text-align: center; + background: var(--surface); + } + .stat-value { display: block; font-size: 1.35rem; font-weight: 800; line-height: 1.1; } + .stat-label { + display: block; + margin-top: 0.25rem; + font-size: 0.72rem; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--muted); + font-weight: 600; + } + .stat-critical .stat-value { color: var(--critical); } + .stat-high .stat-value { color: var(--high); } + .stat-medium .stat-value { color: var(--medium); } + .stat-low .stat-value { color: var(--low); } + .score-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); + gap: 0.75rem; + margin-bottom: 1.75rem; + } + .score-card { + border: 1px solid var(--line); + border-radius: 10px; + padding: 0.9rem 0.75rem; + background: var(--surface); + } + .score-card .score-value { font-size: 1.5rem; font-weight: 800; line-height: 1; } + .score-card .score-name { margin-top: 0.45rem; font-size: 0.78rem; font-weight: 600; line-height: 1.25; } + .score-card .score-meta { margin-top: 0.25rem; font-size: 0.72rem; color: var(--muted); } + .score-card.score-good .score-value { color: var(--good); } + .score-card.score-fair .score-value { color: var(--fair); } + .score-card.score-poor .score-value { color: var(--poor); } + .score-card.score-na .score-value { color: var(--muted); } + section { margin-bottom: 2rem; page-break-inside: avoid; } + section h2 { + margin: 0 0 0.85rem; + font-size: 1rem; + font-weight: 700; + letter-spacing: 0.02em; + text-transform: uppercase; + color: var(--ink); + padding-bottom: 0.45rem; + border-bottom: 2px solid var(--brand); + } + .callout { + border-left: 4px solid var(--brand-accent); + background: #eff6ff; + padding: 1rem 1.15rem; + border-radius: 0 10px 10px 0; + margin-bottom: 0.5rem; + } + .callout ul { margin: 0; padding-left: 1.15rem; } + .callout li { margin: 0.35rem 0; } + table.data { + width: 100%; + border-collapse: collapse; + font-size: 0.84rem; + border: 1px solid var(--line); + border-radius: 10px; + overflow: hidden; + } + table.data th, + table.data td { + padding: 0.55rem 0.65rem; + text-align: left; + vertical-align: top; + border-bottom: 1px solid var(--line); + } + table.data th { + background: var(--surface-muted); + font-size: 0.72rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--muted); + font-weight: 700; + } + table.data tbody tr:last-child td { border-bottom: none; } + table.data tbody tr:nth-child(even) td { background: #fcfdff; } + table.kv th { + width: 34%; + font-weight: 600; + color: var(--ink); + background: var(--surface-muted); + } + .url { word-break: break-all; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.78rem; } + .badge { + display: inline-block; + padding: 0.15rem 0.5rem; + border-radius: 999px; + font-size: 0.68rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; + } + .badge-critical { background: #fee2e2; color: var(--critical); } + .badge-high { background: #ffedd5; color: var(--high); } + .badge-medium { background: #fef3c7; color: var(--medium); } + .badge-low { background: #f1f5f9; color: var(--low); } + .muted { color: var(--muted); font-size: 0.86rem; margin: 0.35rem 0 0.75rem; } + .report-footer { + border-top: 1px solid var(--line); + padding: 1.25rem 2.75rem 2rem; + color: var(--muted); + font-size: 0.78rem; + line-height: 1.5; + } + @media print { + body { background: #fff; } + .report { max-width: none; } + .cover { -webkit-print-color-adjust: exact; print-color-adjust: exact; } + .content { padding: 1.2cm 1.4cm; } + section { page-break-inside: auto; } + table.data { page-break-inside: auto; } + table.data tr { page-break-inside: avoid; } + .report-footer { padding-left: 1.4cm; padding-right: 1.4cm; } + } + @media (max-width: 640px) { + .cover, .content, .report-footer { padding-left: 1.25rem; padding-right: 1.25rem; } + .stats-row { grid-template-columns: repeat(2, minmax(0, 1fr)); } + } +""" diff --git a/tests/test_categories_coverage.py b/tests/test_categories_coverage.py index f857995c..0017d638 100644 --- a/tests/test_categories_coverage.py +++ b/tests/test_categories_coverage.py @@ -467,7 +467,7 @@ def test_category_html_accessibility_alt_thin_reading_level() -> None: def test_category_html_accessibility_score_zero_floor() -> None: df = pd.DataFrame([{"url": "https://example.com/", "status": "200", "h1_count": 1}]) with patch( - "website_profiling.reporting.categories._score_deductions", + "website_profiling.reporting.categories.accessibility._score_deductions", return_value=0, ): cat = category_html_accessibility(df) diff --git a/tests/test_crawl_frontier.py b/tests/test_crawl_frontier.py new file mode 100644 index 00000000..4e8b40e9 --- /dev/null +++ b/tests/test_crawl_frontier.py @@ -0,0 +1,70 @@ +"""Unit tests for crawl frontier (no network).""" + +from website_profiling.crawl.config import CrawlConfig +from website_profiling.crawl.frontier import CrawlFrontier, url_matches_exclude + + +def test_url_matches_exclude_prefix() -> None: + assert url_matches_exclude("https://a.com/blog/post", ["https://a.com/blog"]) + assert not url_matches_exclude("https://a.com/about", ["https://a.com/blog"]) + assert url_matches_exclude("https://a.com/blog", ["https://a.com/blog"]) + + +def test_enqueue_seed_respects_exclude_and_domain() -> None: + frontier = CrawlFrontier( + "https://example.com", + exclude_urls=["https://example.com/private"], + allow_external=False, + ) + frontier.enqueue_seed("https://example.com/page", 0) + frontier.enqueue_seed("https://other.com/page", 0) + frontier.enqueue_seed("https://example.com/private", 0) + assert frontier.queue.qsize() == 1 + assert frontier.depths["https://example.com/page"] == 0 + + +def test_try_enqueue_link_depth_limit() -> None: + frontier = CrawlFrontier("https://example.com", max_depth=0, follow_links=True) + frontier.depths["https://example.com"] = 0 + assert frontier.try_enqueue_link("https://example.com/child", "https://example.com") is False + + frontier.max_depth = 1 + assert frontier.try_enqueue_link("https://example.com/child", "https://example.com") is True + assert frontier.depths["https://example.com/child"] == 1 + + +def test_mark_visited_dedupes() -> None: + frontier = CrawlFrontier("https://example.com") + assert frontier.mark_visited("https://example.com/a") is True + assert frontier.mark_visited("https://example.com/a") is False + + +def test_try_enqueue_link_skips_already_visited() -> None: + frontier = CrawlFrontier("https://example.com", follow_links=True) + frontier.depths["https://example.com"] = 0 + frontier.visited.add("https://example.com/child") + assert frontier.try_enqueue_link("https://example.com/child", "https://example.com") is False + + +def test_queue_contains_returns_false_on_queue_error() -> None: + class BadDeque: + def __iter__(self): + raise RuntimeError("broken") + + class BadQueueWrapper: + queue = BadDeque() + + frontier = CrawlFrontier("https://example.com") + frontier.queue = BadQueueWrapper() + assert frontier.queue_contains("https://example.com/x") is False + + +def test_crawl_config_javascript_render_properties() -> None: + cfg = CrawlConfig( + start_url="https://example.com", + render_mode="javascript", + js_concurrency=5, + concurrency=10, + ) + assert cfg.effective_concurrency == 5 + assert cfg.fetcher_render_mode == "javascript" diff --git a/tests/test_crawl_gap_coverage.py b/tests/test_crawl_gap_coverage.py index aa14403a..2468ae01 100644 --- a/tests/test_crawl_gap_coverage.py +++ b/tests/test_crawl_gap_coverage.py @@ -251,7 +251,7 @@ def test_crawler_robots_txt_override_parses(monkeypatch) -> None: lambda *_a, **_k: [], ) monkeypatch.setattr( - "website_profiling.crawl.crawler.load_robots", + "website_profiling.crawl.frontier.load_robots", lambda _u: (_ for _ in ()).throw(AssertionError("load_robots should not run")), ) c = Crawler( @@ -269,7 +269,7 @@ def test_crawler_loads_robots_when_no_override(monkeypatch) -> None: "website_profiling.crawl.sitemap.discover_sitemap_urls", lambda *_a, **_k: [], ) - monkeypatch.setattr("website_profiling.crawl.crawler.load_robots", lambda _u: object()) + monkeypatch.setattr("website_profiling.crawl.frontier.load_robots", lambda _u: object()) c = Crawler(start_url="https://site.com", ignore_robots=False) assert c.rp is not None diff --git a/tests/test_crawler_deep.py b/tests/test_crawler_deep.py index fe3e2d99..75c560fa 100644 --- a/tests/test_crawler_deep.py +++ b/tests/test_crawler_deep.py @@ -8,6 +8,7 @@ def test_worker_success_path_populates_many_fields(monkeypatch): import website_profiling.crawl.crawler as mod + import website_profiling.crawl.page_record as pr_mod from website_profiling.crawl.fetchers.base import FetchResult monkeypatch.setattr( @@ -34,7 +35,7 @@ def test_worker_success_path_populates_many_fields(monkeypatch): fetch_method="static", ) monkeypatch.setattr( - mod, + pr_mod, "parse_link_edges", lambda _u, _t: ( "T", @@ -44,9 +45,9 @@ def test_worker_success_path_populates_many_fields(monkeypatch): ], ), ) - monkeypatch.setattr(mod, "parse_seo", lambda *_a, **_k: ("desc", 4, "h1", 1, "https://site.com/canon")) + monkeypatch.setattr(pr_mod, "parse_seo", lambda *_a, **_k: ("desc", 4, "h1", 1, "https://site.com/canon")) monkeypatch.setattr( - mod, + pr_mod, "parse_seo_extended", lambda *_a, **_k: { "viewport_present": True, @@ -62,9 +63,9 @@ def test_worker_success_path_populates_many_fields(monkeypatch): "mixed_content_count": 0, }, ) - monkeypatch.setattr(mod, "parse_resources", lambda *_a, **_k: {"script_count": 1, "link_stylesheet_count": 1}) + monkeypatch.setattr(pr_mod, "parse_resources", lambda *_a, **_k: {"script_count": 1, "link_stylesheet_count": 1}) monkeypatch.setattr( - mod, + pr_mod, "parse_content_text", lambda *_a, **_k: { "word_count": 10, @@ -75,7 +76,7 @@ def test_worker_success_path_populates_many_fields(monkeypatch): }, ) monkeypatch.setattr( - mod, + pr_mod, "parse_social_meta", lambda *_a, **_k: { "og_title": "og", @@ -87,8 +88,8 @@ def test_worker_success_path_populates_many_fields(monkeypatch): "twitter_image": "", }, ) - monkeypatch.setattr(mod, "parse_tech_stack", lambda *_a, **_k: "[]") - monkeypatch.setattr(mod, "analyze_html", lambda *_a, **_k: {"ok": True}) + monkeypatch.setattr(pr_mod, "parse_tech_stack", lambda *_a, **_k: "[]") + monkeypatch.setattr(pr_mod, "analyze_html", lambda *_a, **_k: {"ok": True}) out = c.worker("https://site.com") assert out["status"] == 200 diff --git a/tests/test_crawler_unit.py b/tests/test_crawler_unit.py index 339e1350..06717b3e 100644 --- a/tests/test_crawler_unit.py +++ b/tests/test_crawler_unit.py @@ -8,7 +8,7 @@ def _mock_sitemap_unless_seeding_test(monkeypatch, request): if request.node.name == "test_crawler_seeds_sitemap_urls": return monkeypatch.setattr( - "website_profiling.crawl.crawler.discover_sitemap_urls", + "website_profiling.crawl.sitemap.discover_sitemap_urls", lambda *_a, **_k: [], ) @@ -45,7 +45,7 @@ def test_crawler_sitemap_seed_exception_is_ignored(monkeypatch) -> None: def _boom(*_a, **_k): raise RuntimeError("sitemap unavailable") - monkeypatch.setattr("website_profiling.crawl.crawler.discover_sitemap_urls", _boom) + monkeypatch.setattr("website_profiling.crawl.sitemap.discover_sitemap_urls", _boom) c = Crawler(start_url="https://site.com", ignore_robots=True, use_wappalyzer=False) assert c.queue.qsize() == 1 @@ -54,7 +54,7 @@ def test_crawler_sitemap_seed_filters_exclude_external_and_duplicates(monkeypatc from website_profiling.crawl.crawler import Crawler monkeypatch.setattr( - "website_profiling.crawl.crawler.discover_sitemap_urls", + "website_profiling.crawl.sitemap.discover_sitemap_urls", lambda *_a, **_k: [ "https://site.com/skip-me", "https://external.com/page", @@ -81,7 +81,7 @@ def test_crawler_seeds_sitemap_urls(monkeypatch) -> None: from website_profiling.crawl.crawler import Crawler monkeypatch.setattr( - "website_profiling.crawl.crawler.discover_sitemap_urls", + "website_profiling.crawl.sitemap.discover_sitemap_urls", lambda *_a, **_k: ["https://site.com/from-sitemap"], ) c = Crawler( @@ -300,7 +300,7 @@ def test_worker_uses_wappalyzer_when_enabled(monkeypatch) -> None: html = "Tok" monkeypatch.setattr( - "website_profiling.crawl.crawler.detect_tech_wappalyzer", + "website_profiling.crawl.page_record.detect_tech_wappalyzer", lambda *_a, **_k: '["React"]', ) c = Crawler( diff --git a/tests/test_page_record.py b/tests/test_page_record.py new file mode 100644 index 00000000..cbd8dbab --- /dev/null +++ b/tests/test_page_record.py @@ -0,0 +1,50 @@ +"""Unit tests for crawl page record builder (no network).""" + +from website_profiling.crawl.page_record import PageRecordBuilder +from website_profiling.crawl.schema import CRAWL_ROW_COLUMNS, empty_crawl_row, empty_crawl_row_ext + + +def test_empty_crawl_row_has_core_fields() -> None: + row = empty_crawl_row(url="https://a.com", status=200) + assert row["url"] == "https://a.com" + assert row["status"] == 200 + assert row["fetch_method"] == "static" + assert row["page_analysis"] == "{}" + + +def test_empty_crawl_row_ext_headers() -> None: + ext = empty_crawl_row_ext( + "https://a.com", + headers_dict={"Cache-Control": "no-cache", "X-Robots-Tag": "noindex"}, + ) + assert ext["cache_control"] == "no-cache" + assert ext["x_robots_tag"] == "noindex" + + +def test_crawl_row_columns_match_dataframe_schema() -> None: + assert "url" in CRAWL_ROW_COLUMNS + assert "fetch_method" in CRAWL_ROW_COLUMNS + assert "page_analysis" in CRAWL_ROW_COLUMNS + + +def test_build_robots_blocked_row() -> None: + row = PageRecordBuilder.build_robots_blocked_row( + "https://a.com", store_outlinks=True + ) + assert row["status"] == "blocked_by_robots" + assert row["outlink_targets"] == "[]" + + +def test_parse_page_content_minimal_html() -> None: + builder = PageRecordBuilder(use_wappalyzer=False) + html = "Hi

    Hello

    " + parsed = builder.parse_page_content( + "https://a.com", + html, + "https://a.com", + {}, + 0, + ) + assert parsed["title"] == "Hi" + assert parsed["h1_text"] == "Hello" + assert parsed["h1_count"] == 1 diff --git a/tests/test_property_profile.py b/tests/test_property_profile.py index 684d9e41..b7929765 100644 --- a/tests/test_property_profile.py +++ b/tests/test_property_profile.py @@ -553,7 +553,7 @@ def fake_get(url, timeout=8): session = MagicMock() session.get.side_effect = fake_get - monkeypatch.setattr("website_profiling.reporting.builder.requests.Session", lambda: session) + monkeypatch.setattr("website_profiling.reporting.site_level.requests.Session", lambda: session) out = _fetch_site_level("https://example.com/", timeout=1) assert out["ads_txt_present"] is True assert out["security_txt_present"] is True diff --git a/tests/test_reporting_builder_modules.py b/tests/test_reporting_builder_modules.py new file mode 100644 index 00000000..209bbd52 --- /dev/null +++ b/tests/test_reporting_builder_modules.py @@ -0,0 +1,839 @@ +"""Coverage for reporting modules split from builder.py (reporting gate).""" +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from website_profiling.reporting import content_analytics, edges_report, lighthouse_report, report_metadata, seo_summary, site_level + + +def _crawl_df() -> pd.DataFrame: + return pd.DataFrame( + [ + { + "url": "https://example.com/", + "status": "200", + "title": "A" * 45, + "meta_description_len": 100, + "meta_description": "desc", + "h1_count": 1, + "h1": "Heading", + "content_length": 500, + "word_count": 250, + "reading_level": 10, + "content_html_ratio": 25, + "outlinks": 3, + "depth": 0, + "response_time_ms": 2500, + "content_type": "text/html", + "og_title": "OG", + "twitter_card": "summary", + "og_image": "https://cdn.example.com/img.png", + "tech_stack": json.dumps(["WordPress", "nginx"]), + "top_keywords": json.dumps([{"word": "seo audit", "count": 4}]), + "heading_sequence": "h1,h2", + "script_count": 2, + "link_stylesheet_count": 1, + "page_analysis": json.dumps( + { + "html_lang": "en", + "hreflang_alternates": [{"hreflang": "en", "href": "https://example.com/"}], + "external_links": ["https://partner.com/page"], + } + ), + "outlink_targets": "https://example.com/about,https://partner.com/x", + "fetch_method": "static", + }, + { + "url": "https://example.com/about", + "status": "200", + "title": "Short", + "meta_description_len": 50, + "h1_count": 0, + "content_length": 150, + "word_count": 80, + "depth": 1, + "response_time_ms": 150, + "content_type": "text/html", + "og_title": "", + "twitter_card": "", + "og_image": "", + "tech_stack": "[]", + "top_keywords": "not-json", + "script_count": 0, + "link_stylesheet_count": 0, + "fetch_method": "rendered", + }, + { + "url": "https://example.com/missing", + "status": "404", + "title": "", + "meta_description": "", + "h1": "", + "heading_sequence": "", + "word_count": 0, + "content_length": 0, + "h1_count": 0, + "script_count": 0, + "link_stylesheet_count": 0, + }, + { + "url": "https://example.com/redirect", + "status": "301", + "final_url": "https://example.com/dest", + "title": "T" * 70, + "meta_description": "", + "h1": "", + "heading_sequence": "", + "word_count": 0, + "meta_description_len": 200, + "h1_count": 3, + "content_length": 50, + "script_count": 0, + "link_stylesheet_count": 0, + }, + { + "url": "https://example.com/error", + "status": "error", + "title": "", + "meta_description": "", + "h1": "", + "heading_sequence": "", + "word_count": 0, + "content_length": 0, + "h1_count": 0, + "script_count": 0, + "link_stylesheet_count": 0, + }, + ] + ) + + +def test_compute_summary_seo_issues() -> None: + out = seo_summary._compute_summary_seo_issues(_crawl_df()) + assert out["summary"]["total_urls"] == 5 + assert out["issues"]["broken"] + assert out["issues"]["redirects"] + assert out["issues"]["seo"] + assert out["recommendations"] + + +def test_content_analytics_helpers() -> None: + df = _crawl_df() + content = content_analytics._build_content_analytics(df) + assert content["word_count_stats"]["mean"] > 0 + assert content["thin_pages"] + assert content_analytics._parse_top_keywords_items(None) == [] + assert content_analytics._parse_top_keywords_items('["list"]') == [] + assert content_analytics._parse_top_keywords_items(json.dumps([{"word": "x", "count": 2}]))[0]["word"] == "x" + + social = content_analytics._build_social_coverage(df) + assert social["og_coverage_pct"] > 0 + assert social["missing_og"] + + tech = content_analytics._build_tech_stack_summary(df) + assert tech["technologies"] + + rt = content_analytics._build_response_time_stats(df) + assert rt["slow_pages"] + assert rt["p50"] > 0 + + depth = content_analytics._build_depth_distribution(df) + assert depth["max_depth"] == 1 + + kw = content_analytics._build_keyword_opportunities(df, {"include_keyword_opportunities": "true"}) + assert "quick_wins" in kw + assert content_analytics._build_keyword_opportunities(df, {"include_keyword_opportunities": "false"}) == {} + + +def test_build_image_inventory_empty_urls(capsys) -> None: + with patch("website_profiling.analysis.image_probe.probe_image_urls") as probe: + inventory, summary = content_analytics._build_image_inventory( + [{"url": "https://ex.com/", "page_analysis": {}}], + {"probe_image_inventory": "true"}, + ) + probe.assert_not_called() + assert inventory == [] + assert summary["inventory_available"] is False + + +def test_build_image_inventory_counts_failures(capsys) -> None: + links = [{"url": "https://ex.com/p", "page_analysis": {"image_urls": ["https://cdn.ex.com/a.png"]}}] + probed = [{"url": "https://cdn.ex.com/a.png", "status": None, "content_type": None, "size_bytes": None, "error": "timeout"}] + with patch("website_profiling.analysis.image_probe.probe_image_urls", return_value=probed): + inventory, summary = content_analytics._build_image_inventory( + links, + {"probe_image_inventory": "true", "max_image_probe_urls": "10"}, + ) + assert summary["failed"] == 1 + assert summary["inventory_available"] is True + assert inventory[0]["error"] == "timeout" + + +def test_report_metadata_helpers(capsys) -> None: + df = _crawl_df() + assert report_metadata._parse_page_analysis_cell(float("nan")) == {} + assert report_metadata._parse_page_analysis_cell("{bad") == {} + + outbound = report_metadata._build_outbound_link_domains(df, "https://example.com/", 10) + assert outbound[0]["host"] == "partner.com" + + fps = report_metadata._build_url_fingerprints(df) + assert fps[0]["content_fingerprint"] + + hreflang = report_metadata._build_hreflang_summary(df) + assert hreflang["pages_with_hreflang_links"] == 1 + + payload = {"links": [{}], "summary": {"total_urls": 2}, "report_meta": {"crawl_scope": {"pages_crawled": 3}}} + report_metadata._validate_report_url_counts(payload, 1) + assert payload["ml_errors"] + + meta = report_metadata._build_report_metadata( + df, + {"max_pages": "2", "export_logo_url": "https://logo.example/logo.png", "crawl_render_mode": "static"}, + {"url": "https://example.com/"}, + { + "fetched_at": "2026-01-01", + "date_range_days": 28, + "gsc": {"row_count": 10}, + "ga4": {"sessions": 1}, + }, + {"rows": [{"word": "test", "source": "crawl"}], "enriched_at": "2026-01-02"}, + {"llm_meta": {"model": "gpt-test"}}, + 42, + "2026-01-01T00:00:00Z", + { + "imported_at": "2026-01-03", + "top_linking_sites": ["a.com"], + "sample_links": [{"url": "x"}], + "latest_links": [], + }, + ) + assert "lighthouse" in meta["data_sources"] + assert "search_console" in meta["data_sources"] + assert "analytics" in meta["data_sources"] + assert "ai" in meta["data_sources"] + assert "estimated" in meta["data_sources"] + assert meta["export_logo_url"] == "https://logo.example/logo.png" + assert meta["crawl_run_id"] == 42 + + +def test_site_level_branches(monkeypatch) -> None: + assert site_level._fetch_site_level("not-a-url")["robots_present"] is False + + session = MagicMock() + session.get.side_effect = RuntimeError("network") + monkeypatch.setattr("website_profiling.reporting.site_level.requests.Session", lambda: session) + out = site_level._fetch_site_level("https://example.com/", timeout=1) + assert out["robots_present"] is False + assert out["sitemap_present"] is False + + +def test_lighthouse_report_helpers() -> None: + assert lighthouse_report._strip_www("WWW.Example.COM") == "example.com" + assert lighthouse_report._url_hostname("") == "" + assert lighthouse_report._url_hostname("not a url") == "" + assert lighthouse_report._hosts_match("www.example.com", "example.com") + assert not lighthouse_report._hosts_match("", "example.com") + + by_url = {"https://example.com/": {"score": 90}, "https://other.com/": {"score": 50}} + filtered = lighthouse_report.filter_lighthouse_by_host(by_url, "example.com") + assert len(filtered) == 1 + + assert lighthouse_report.lighthouse_for_url(by_url, "https://example.com")["score"] == 90 + assert lighthouse_report.lighthouse_for_url(by_url, "https://missing.com") is None + + host = lighthouse_report._derive_expected_host("", pd.DataFrame({"url": ["https://derived.com/x"]})) + assert host == "derived.com" + + picked = lighthouse_report._pick_lighthouse_summary( + by_url, + "https://example.com/", + {"url": "https://example.com/", "performance": 88}, + "example.com", + ) + assert picked is not None + + with patch("website_profiling.reporting.lighthouse_report.ssl.create_default_context") as ctx_mock: + cert_sock = MagicMock() + cert_sock.getpeercert.return_value = {"notAfter": "Jan 1 00:00:00 2030 GMT"} + ctx_mock.return_value.wrap_socket.return_value.__enter__.return_value = cert_sock + with patch("website_profiling.reporting.lighthouse_report.socket.create_connection"): + iso = lighthouse_report.fetch_site_ssl_expires_iso("example.com") + assert iso is not None + assert lighthouse_report.fetch_site_ssl_expires_iso("") is None + with patch("website_profiling.reporting.lighthouse_report.socket.create_connection", side_effect=OSError("fail")): + assert lighthouse_report.fetch_site_ssl_expires_iso("example.com") is None + + +def test_build_lighthouse_by_url_for_report(monkeypatch) -> None: + conn = MagicMock() + raw = { + "lighthouseResult": { + "finalUrl": "https://example.com/", + "audits": { + "first-contentful-paint": { + "score": 0.5, + "title": "FCP", + "helpText": "Improve FCP", + } + }, + } + } + + monkeypatch.setattr( + "website_profiling.db.read_lighthouse_page_summaries", + lambda _c: {"https://example.com": {"url": "https://example.com/", "median_metrics": {}}}, + ) + monkeypatch.setattr( + "website_profiling.db.read_lh_runs_by_url", + lambda _c: {"https://example.com": [99]}, + ) + monkeypatch.setattr("website_profiling.db.read_lighthouse_run_json", lambda _c, _id: raw) + monkeypatch.setattr( + "website_profiling.db.read_lh_audits_with_items", + lambda _c, _id: [{"id": "fcp"}], + ) + monkeypatch.setattr( + "website_profiling.lighthouse.runner.extract_from_lighthouse_json", + lambda _raw: {"performance_score": 50, "category_scores": {"performance": 50}}, + ) + monkeypatch.setattr( + "website_profiling.tools.warnings.parse_lighthouse_to_diagnostics", + lambda _raw, max_nodes_in_refs=None: [{"id": "diag"}], + ) + monkeypatch.setattr( + "website_profiling.tools.warnings.resolve_impact", + lambda *_a, **_k: "high", + ) + monkeypatch.setattr( + "website_profiling.lighthouse.runner._evidence_from_audit", + lambda _a: "evidence", + ) + + out = lighthouse_report.build_lighthouse_by_url_for_report(conn) + assert "https://example.com" in out + assert out["https://example.com"]["top_failures"] + + +def test_build_edges_from_df_paths(monkeypatch, tmp_path) -> None: + edges_csv = str(tmp_path / "edges.csv") + with patch("website_profiling.reporting.edges_report.load_edges", return_value=[("https://a.com", "https://a.com/b")]): + loaded = edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://a.com"]}), + edges_csv, + True, + 10, + 2, + 5, + 0.0, + ) + assert loaded == [("https://a.com", "https://a.com/b")] + + df = pd.DataFrame( + { + "url": ["https://example.com/", "https://other.com/"], + "outlink_targets": ["https://example.com/about", "https://other.com/x"], + } + ) + from_column = edges_report.build_edges_from_df(df, "", False, 10, 2, 5, 0.0) + assert ("https://example.com/", "https://example.com/about") in from_column + + html = 'AboutExt' + + class FakeResp: + status_code = 200 + text = html + + @property + def headers(self): + return {"Content-Type": "text/html"} + + session = MagicMock() + session.get.return_value = FakeResp() + monkeypatch.setattr("website_profiling.reporting.edges_report.requests.Session", lambda: session) + + fetched = edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://example.com/"]}), + "", + True, + 10, + 1, + 5, + 0.0, + ) + assert any(t == "https://example.com/about" for _s, t in fetched) + + fetcher = MagicMock() + fetcher.fetch.return_value = SimpleNamespace(status=200, text=html) + fetcher.close = MagicMock() + monkeypatch.setattr("website_profiling.crawl.fetchers.build_fetcher", lambda **_k: fetcher) + js_edges = edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://example.com/"]}), + "", + True, + 10, + 1, + 5, + 0.0, + render_mode="javascript", + ) + fetcher.close.assert_called_once() + assert js_edges + + fetcher.fetch.return_value = SimpleNamespace(status=404, text="") + assert edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://example.com/"]}), + "", + True, + 10, + 1, + 5, + 0.0, + render_mode="auto", + ) == [] + + session.get.side_effect = RuntimeError("boom") + assert edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://example.com/"]}), + "", + True, + 10, + 1, + 5, + 0.0, + ) == [] + + +def test_content_analytics_edge_branches() -> None: + assert content_analytics._build_content_analytics(pd.DataFrame())["word_count_stats"]["mean"] == 0 + assert content_analytics._build_content_analytics(pd.DataFrame({"url": ["x"], "status": ["404"]}))["word_count_stats"]["mean"] == 0 + + df = pd.DataFrame( + [ + {"url": float("nan"), "status": "200", "word_count": 100}, + {"url": "https://example.com/thin", "status": "200", "word_count": 50}, + ] + ) + assert content_analytics._build_content_analytics(df)["thin_pages"] == [{"url": "https://example.com/thin", "word_count": 50}] + + assert content_analytics._parse_top_keywords_items(json.dumps(["bad"])) == [] + assert content_analytics._parse_top_keywords_items(json.dumps([{"word": "", "count": 1}])) == [] + + hist_df = pd.DataFrame( + [ + { + "url": f"https://example.com/{i}", + "status": "200", + "top_keywords": json.dumps([{"word": "shared-term", "count": 1}]), + } + for i in range(25) + ] + ) + hist = content_analytics._build_text_content_analysis(hist_df)["keyword_frequency_histogram"] + assert hist["21+"] == 1 + + assert content_analytics._build_content_analytics(pd.DataFrame({"url": ["x"], "status": ["404"], "word_count": [0]}))["word_count_stats"]["mean"] == 0 + + assert content_analytics._build_social_coverage(pd.DataFrame())["og_coverage_pct"] == 0 + non_html = pd.DataFrame([{"url": "https://example.com/x", "status": "200", "content_type": "application/json", "og_title": "x"}]) + assert content_analytics._build_social_coverage(non_html)["og_coverage_pct"] == 0 + + social_df = pd.DataFrame( + [ + {"url": float("nan"), "status": "200", "content_type": "text/html", "og_title": "x", "twitter_card": "x", "og_image": "x"}, + {"url": "https://example.com/a", "status": "200", "content_type": "text/html", "og_title": "", "twitter_card": "", "og_image": ""}, + ] + ) + social = content_analytics._build_social_coverage(social_df) + assert social["missing_og"] == ["https://example.com/a"] + + assert content_analytics._build_tech_stack_summary(pd.DataFrame({"url": ["x"], "status": ["200"]}))["technologies"] == [] + tech_df = pd.DataFrame( + [ + {"url": "https://example.com/a", "status": "200", "content_type": "application/json", "tech_stack": '["React"]'}, + {"url": "https://example.com/b", "status": "200", "content_type": "text/html", "tech_stack": "bad-json"}, + ] + ) + assert content_analytics._build_tech_stack_summary(tech_df)["technologies"] == [] + + assert content_analytics._build_response_time_stats(pd.DataFrame({"url": ["x"]}))["p50"] == 0 + assert content_analytics._build_response_time_stats(pd.DataFrame({"url": ["x"], "response_time_ms": [None]}))["p50"] == 0 + assert content_analytics._build_depth_distribution(pd.DataFrame({"url": ["x"]}))["max_depth"] == 0 + assert content_analytics._build_depth_distribution(pd.DataFrame({"url": ["x"], "depth": [None]}))["max_depth"] == 0 + + empty_kw = content_analytics._build_keyword_opportunities( + pd.DataFrame({"url": ["https://example.com"], "status": ["404"]}), + {"include_keyword_opportunities": "true"}, + ) + assert empty_kw["quick_wins"] == [] + + +def test_report_metadata_extra_branches() -> None: + assert report_metadata._parse_page_analysis_cell("{}") == {} + df = pd.DataFrame( + [ + { + "url": "", + "status": "200", + "page_analysis": json.dumps({"external_links": ["https://partner.com/x", 123]}), + "outlink_targets": "https://example.com/,https://partner.com/y", + }, + { + "url": "https://example.com/ok", + "status": "200", + "page_analysis": json.dumps({"external_links": ["https://partner.com/z"]}), + }, + ] + ) + hosts = report_metadata._build_outbound_link_domains(df, "https://example.com/", 10) + assert any(h["host"] == "partner.com" for h in hosts) + + hreflang = report_metadata._build_hreflang_summary( + pd.DataFrame( + [ + {"url": "https://example.com/a", "status": "200", "page_analysis": json.dumps({"html_lang": "", "hreflang_alternates": []})}, + {"url": "https://example.com/b", "status": "404", "page_analysis": "{}"}, + ] + ) + ) + assert hreflang["pages_missing_html_lang"] == 1 + + meta = report_metadata._build_report_metadata( + pd.DataFrame([{"url": "https://example.com", "status": "200"}]), + {}, + None, + {"gsc": {"row_count": 1}}, + None, + {}, + None, + None, + {"imported_at": "2026-01-01", "top_linking_sites": [], "sample_links": [], "latest_links": []}, + ) + assert meta["gsc_links_imported_at"] == "2026-01-01" + + +def test_seo_summary_skips_blank_urls() -> None: + df = pd.DataFrame( + [ + {"url": float("nan"), "status": "200", "title": "Missing URL row"}, + {"url": "https://example.com/a", "status": "200", "title": "", "meta_description_len": 10, "h1_count": 0, "content_length": 100}, + ] + ) + out = seo_summary._compute_summary_seo_issues(df) + assert out["issues"]["seo"] + + +def test_site_level_reads_robots_sitemap_hint(monkeypatch) -> None: + class FakeResp: + def __init__(self, code, text): + self.status_code = code + self.text = text + + session = MagicMock() + session.get.side_effect = lambda url, timeout=8: { + "https://example.com/robots.txt": FakeResp(200, "User-agent: *\nSitemap: https://example.com/sitemap.xml"), + "https://example.com/sitemap.xml": FakeResp(200, ""), + "https://example.com/ads.txt": FakeResp(404, ""), + "https://example.com/.well-known/security.txt": FakeResp(404, ""), + }[url] + monkeypatch.setattr("website_profiling.reporting.site_level.requests.Session", lambda: session) + out = site_level._fetch_site_level("https://example.com/", timeout=1) + assert out["robots_present"] is True + + +def test_lighthouse_report_extra_branches(monkeypatch) -> None: + assert lighthouse_report.filter_lighthouse_by_host({}, "example.com") == {} + assert lighthouse_report._derive_expected_host("", pd.DataFrame()) == "" + + cert_sock = MagicMock() + cert_sock.getpeercert.return_value = {} + ctx = MagicMock() + ctx.wrap_socket.return_value.__enter__.return_value = cert_sock + with patch("website_profiling.reporting.lighthouse_report.ssl.create_default_context", return_value=ctx), patch( + "website_profiling.reporting.lighthouse_report.socket.create_connection" + ): + assert lighthouse_report.fetch_site_ssl_expires_iso("example.com") is None + + with patch("website_profiling.reporting.lighthouse_report.urlparse", side_effect=ValueError("bad")): + assert lighthouse_report._url_hostname("bad://") == "" + + picked = lighthouse_report._pick_lighthouse_summary( + {}, + "", + {"url": "https://other.com/", "performance": 1}, + "example.com", + ) + assert picked is None + + assert lighthouse_report.lighthouse_for_url({}, "https://example.com") is None + + conn = MagicMock() + monkeypatch.setattr("website_profiling.db.read_lighthouse_page_summaries", lambda _c: {}) + monkeypatch.setattr("website_profiling.db.read_lh_runs_by_url", lambda _c: {}) + assert lighthouse_report.build_lighthouse_by_url_for_report(conn) == {} + + raw_only = { + "lighthouseResult": { + "finalUrl": "https://only-raw.com/", + "audits": {"ok-audit": {"score": 1}, "bad": "text", "fail-audit": {"score": 0.2, "title": "Fail", "helpText": ""}}, + } + } + monkeypatch.setattr("website_profiling.db.read_lighthouse_page_summaries", lambda _c: {}) + monkeypatch.setattr("website_profiling.db.read_lh_runs_by_url", lambda _c: {"https://only-raw.com": [2]}) + monkeypatch.setattr("website_profiling.db.read_lighthouse_run_json", lambda _c, _id: raw_only) + monkeypatch.setattr("website_profiling.db.read_lh_audits_with_items", lambda _c, _id: []) + monkeypatch.setattr( + "website_profiling.lighthouse.runner.extract_from_lighthouse_json", + lambda _raw: {"performance_score": 40, "category_scores": {}, "lcp_ms": 1}, + ) + monkeypatch.setattr( + "website_profiling.tools.warnings.parse_lighthouse_to_diagnostics", + lambda _raw, max_nodes_in_refs=None: [], + ) + monkeypatch.setattr("website_profiling.tools.warnings.resolve_impact", lambda *_a, **_k: "medium") + monkeypatch.setattr("website_profiling.lighthouse.runner._evidence_from_audit", lambda _a: "") + out = lighthouse_report.build_lighthouse_by_url_for_report(conn) + assert out["https://only-raw.com"]["top_failures"] + + +def test_build_edges_from_df_more_branches(monkeypatch) -> None: + df = pd.DataFrame( + { + "url": ["https://example.com/"], + "links": [""], + "outlink_targets": ["https://other.com/page"], + } + ) + edges = edges_report.build_edges_from_df(df, "", True, 10, 1, 5, 0.0) + assert edges == [] + + class FakeResp: + status_code = 200 + text = 'Link' + + @property + def headers(self): + return {"Content-Type": "text/html"} + + session = MagicMock() + session.get.return_value = FakeResp() + monkeypatch.setattr("website_profiling.reporting.edges_report.requests.Session", lambda: session) + with patch("website_profiling.reporting.edges_report.time.sleep") as sleep: + edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://example.com/"]}), + "", + True, + 10, + 1, + 5, + 0.5, + ) + sleep.assert_called() + + def raise_result(_self): + raise RuntimeError("worker failed") + + with patch("website_profiling.reporting.edges_report.as_completed") as completed: + future = MagicMock() + future.result = raise_result + completed.return_value = [future] + monkeypatch.setattr( + "website_profiling.reporting.edges_report.ThreadPoolExecutor", + lambda *a, **k: MagicMock(__enter__=lambda s: s, __exit__=lambda *a: None, submit=lambda fn, u: future), + ) + session.get.return_value = FakeResp() + result = edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://example.com/"]}), + "", + True, + 10, + 1, + 5, + 0.0, + ) + assert result == [] + + +def test_content_analytics_remaining_lines() -> None: + assert content_analytics._parse_top_keywords_items(object()) == [] # type: ignore[arg-type] + assert content_analytics._parse_top_keywords_items("{bad-json") == [] + + na_kw = pd.DataFrame([{"url": float("nan"), "status": "200", "top_keywords": json.dumps([{"word": "x", "count": 1}])}]) + assert content_analytics._build_text_content_analysis(na_kw)["vocabulary_stats"]["unique_terms"] == 0 + + assert content_analytics._build_keyword_opportunities(pd.DataFrame({"url": ["x"]}), {"include_keyword_opportunities": "true"})["quick_wins"] == [] + + only_404 = pd.DataFrame([{"url": "https://example.com/x", "status": "404", "top_keywords": "[]"}]) + assert content_analytics._build_text_content_analysis(only_404)["keyword_index"] == [] + + spread_df = pd.DataFrame( + [ + { + "url": f"https://example.com/{i}", + "status": "200", + "top_keywords": json.dumps([{"word": "shared-six", "count": 1}]), + } + for i in range(6) + ] + ) + assert content_analytics._build_text_content_analysis(spread_df)["keyword_frequency_histogram"]["6-20"] == 1 + + assert content_analytics._build_tech_stack_summary( + pd.DataFrame([{"url": "https://example.com", "status": "200", "content_type": "application/pdf", "tech_stack": '["X"]'}]) + )["total_pages_analyzed"] == 0 + + kw_df = pd.DataFrame([{"url": "https://example.com", "status": "200", "content_text": "uniquewords " * 20}]) + with patch("website_profiling.reporting.content_analytics.extract_candidates_from_df", return_value=[]): + assert content_analytics._build_keyword_opportunities(kw_df, {"include_keyword_opportunities": "true"})["quick_wins"] == [] + with patch("website_profiling.reporting.content_analytics.extract_candidates_from_df", return_value=[{"word": "a", "volume": 0.1}]), patch( + "website_profiling.reporting.content_analytics.score_keywords", + return_value=[{"word": "a", "volume": 0.1, "difficulty": 80}], + ), patch("website_profiling.reporting.content_analytics.cluster_keywords", return_value=[]): + out = content_analytics._build_keyword_opportunities(kw_df, {"include_keyword_opportunities": "true"}) + assert out["high_value"] + + +def test_report_metadata_remaining_lines() -> None: + df = pd.DataFrame( + [ + { + "url": "https://example.com/", + "status": "200", + "page_analysis": json.dumps({"external_links": ["https://example.com/internal", 99]}), + "word_count": 0, + "content_length": 0, + "h1_count": 0, + "script_count": 0, + "link_stylesheet_count": 0, + }, + ] + ) + assert report_metadata._build_outbound_link_domains(df, "https://example.com/", 5) == [] + assert report_metadata._build_url_fingerprints(pd.DataFrame([{"url": " ", "status": "200"}])) == [] + + meta = report_metadata._build_report_metadata( + pd.DataFrame([{"url": "https://example.com", "status": "200"}]), + {}, + None, + None, + None, + {}, + None, + None, + {"imported_at": "2026-01-01", "top_linking_sites": [], "sample_links": [], "latest_links": []}, + ) + assert "search_console" in meta["data_sources"] + + +def test_seo_summary_remaining_lines() -> None: + df = pd.DataFrame( + [ + { + "url": "https://example.com/a", + "status": "200", + "title": "ok", + "meta_description_len": 0, + "h1_count": 2, + "content_length": 100, + }, + { + "url": float("nan"), + "status": "200", + "title": "x", + "meta_description_len": 80, + "h1_count": 0, + "content_length": 100, + }, + ] + ) + out = seo_summary._compute_summary_seo_issues(df) + assert any(i["type"] == "h1_multi" for i in out["issues"]["seo"]) + + +def test_lighthouse_report_remaining_lines(monkeypatch) -> None: + cert_sock = MagicMock() + cert_sock.getpeercert.return_value = {"notAfter": None} + ctx = MagicMock() + ctx.wrap_socket.return_value.__enter__.return_value = cert_sock + with patch("website_profiling.reporting.lighthouse_report.ssl.create_default_context", return_value=ctx), patch( + "website_profiling.reporting.lighthouse_report.socket.create_connection" + ): + assert lighthouse_report.fetch_site_ssl_expires_iso("example.com") is None + + by_url = {"https://www.example.com/": {"score": 1}} + assert lighthouse_report._pick_lighthouse_summary(by_url, "https://www.example.com/", None, "example.com") is not None + assert lighthouse_report._pick_lighthouse_summary(by_url, "", None, "example.com")["score"] == 1 + assert lighthouse_report._derive_expected_host("", pd.DataFrame({"url": [float("nan")]})) == "" + + conn = MagicMock() + monkeypatch.setattr("website_profiling.db.read_lighthouse_page_summaries", lambda _c: {"https://empty.com": {}}) + monkeypatch.setattr("website_profiling.db.read_lh_runs_by_url", lambda _c: {"https://empty.com": []}) + assert lighthouse_report.build_lighthouse_by_url_for_report(conn) == {} + + assert lighthouse_report._derive_expected_host("https://host-only.com/", pd.DataFrame()) == "host-only.com" + + mismatch = lighthouse_report._pick_lighthouse_summary( + {}, + "https://example.com/", + {"url": "https://other-host.com/", "performance": 1}, + "example.com", + ) + assert mismatch is None + + matched = lighthouse_report._pick_lighthouse_summary( + {}, + "", + {"url": "https://www.example.com/", "performance": 99}, + "example.com", + ) + assert matched["performance"] == 99 + + conn2 = MagicMock() + monkeypatch.setattr( + "website_profiling.db.read_lighthouse_page_summaries", + lambda _c: {"https://needs-url.com": {"median_metrics": {"performance_score": 1}}}, + ) + monkeypatch.setattr("website_profiling.db.read_lh_runs_by_url", lambda _c: {}) + out2 = lighthouse_report.build_lighthouse_by_url_for_report(conn2) + assert out2["https://needs-url.com"]["url"] == "https://needs-url.com" + + assert lighthouse_report.lighthouse_for_url({"https://example.com/": {"score": 9}}, "https://example.com")["score"] == 9 + assert lighthouse_report.lighthouse_for_url({"https://example.com": {"score": 7}}, "https://example.com/")["score"] == 7 + + +def test_build_edges_remaining_lines(monkeypatch) -> None: + df = pd.DataFrame({"url": ["https://example.com/"], "links": [None]}) + assert edges_report.build_edges_from_df(df, "", True, 10, 1, 5, 0.0) == [] + + df_blank = pd.DataFrame({"url": ["https://example.com/"], "outlink_targets": [[" ", "https://example.com/about"]]}) + edges_blank = edges_report.build_edges_from_df(df_blank, "", False, 10, 1, 5, 0.0) + assert ("https://example.com/", "https://example.com/about") in edges_blank + + df_external = pd.DataFrame({"url": ["https://example.com/"], "outlink_targets": ["https://external.com/page"]}) + assert edges_report.build_edges_from_df(df_external, "", True, 10, 1, 5, 0.0) == [] + + class FakeResp: + status_code = 404 + text = "" + + @property + def headers(self): + return {"Content-Type": "text/html"} + + session = MagicMock() + session.get.return_value = FakeResp() + monkeypatch.setattr("website_profiling.reporting.edges_report.requests.Session", lambda: session) + assert edges_report.build_edges_from_df( + pd.DataFrame({"url": ["https://example.com/"]}), + "", + True, + 10, + 1, + 5, + 0.0, + ) == [] From 84eee191856b8c1ce5dedfa44491dd57301f5316 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 12:25:02 +0530 Subject: [PATCH 3/5] maintainable i guess --- alembic/versions/015_crawl_page_html.py | 38 ++++ input.txt.example | 5 + pipeline-config.example.txt | 5 + .../commands/config_resolve.py | 1 + .../commands/pipeline_cmd.py | 50 ++++- .../content_analysis/__init__.py | 7 + .../content_analysis/batch.py | 78 +++++++ .../content_analysis/constants.py | 17 ++ .../content_analysis/dom_cleanup.py | 16 ++ .../content_analysis/excerpt.py | 14 ++ .../content_analysis/html_loader.py | 8 + .../content_analysis/html_ratio.py | 7 + .../content_analysis/keywords.py | 18 ++ .../content_analysis/main_content.py | 25 +++ .../content_analysis/page.py | 45 ++++ .../content_analysis/pipeline.py | 50 +++++ .../content_analysis/reading_level.py | 39 ++++ .../content_analysis/text_extract.py | 8 + .../content_analysis/tokenize.py | 12 + src/website_profiling/crawl/config.py | 15 ++ src/website_profiling/crawl/crawler.py | 69 +++++- src/website_profiling/crawl/db_writer.py | 61 ++++-- src/website_profiling/crawl/html_capture.py | 60 +++++ src/website_profiling/crawl/page_record.py | 15 +- src/website_profiling/db/crawl_store.py | 37 ++++ src/website_profiling/db/historical.py | 2 +- src/website_profiling/db/html_store.py | 112 ++++++++++ src/website_profiling/db/storage.py | 12 + src/website_profiling/parsing/content.py | 83 +------ src/website_profiling/reporting/builder.py | 11 +- tests/test_cli_dispatch.py | 2 +- tests/test_config_schema_keys.py | 5 + tests/test_content_analysis.py | 59 +++++ tests/test_content_analysis_coverage.py | 163 ++++++++++++++ tests/test_content_analysis_pipeline.py | 104 +++++++++ tests/test_crawl_db_writer_imports.py | 46 +++- tests/test_crawl_html_storage.py | 126 +++++++++++ tests/test_crawler_deep.py | 82 ++++++- ...st_historical_keywords_crawl_store_unit.py | 2 +- tests/test_html_store.py | 151 +++++++++++++ tests/test_pipeline_cmd_run_unit.py | 46 ++++ tests/test_reporting_builder_modules.py | 10 + web/app/api/crawl/page-html/route.ts | 75 +++++++ .../pipeline/CrawlPageHtmlManager.tsx | 207 ++++++++++++++++++ .../pipeline/PipelineSettingsPanel.tsx | 16 +- web/src/lib/formatPipelineLog.ts | 7 +- web/src/lib/loadReportDb.ts | 64 ++++++ web/src/lib/pipelineConfigSchema.ts | 39 ++++ web/src/lib/pipelineLiveEstimate.ts | 1 + web/src/server/crawlPageHtmlRoute.test.ts | 80 +++++++ web/src/strings.json | 21 ++ web/src/types/report.ts | 10 + 52 files changed, 2119 insertions(+), 117 deletions(-) create mode 100644 alembic/versions/015_crawl_page_html.py create mode 100644 src/website_profiling/content_analysis/__init__.py create mode 100644 src/website_profiling/content_analysis/batch.py create mode 100644 src/website_profiling/content_analysis/constants.py create mode 100644 src/website_profiling/content_analysis/dom_cleanup.py create mode 100644 src/website_profiling/content_analysis/excerpt.py create mode 100644 src/website_profiling/content_analysis/html_loader.py create mode 100644 src/website_profiling/content_analysis/html_ratio.py create mode 100644 src/website_profiling/content_analysis/keywords.py create mode 100644 src/website_profiling/content_analysis/main_content.py create mode 100644 src/website_profiling/content_analysis/page.py create mode 100644 src/website_profiling/content_analysis/pipeline.py create mode 100644 src/website_profiling/content_analysis/reading_level.py create mode 100644 src/website_profiling/content_analysis/text_extract.py create mode 100644 src/website_profiling/content_analysis/tokenize.py create mode 100644 src/website_profiling/crawl/html_capture.py create mode 100644 src/website_profiling/db/html_store.py create mode 100644 tests/test_content_analysis.py create mode 100644 tests/test_content_analysis_coverage.py create mode 100644 tests/test_content_analysis_pipeline.py create mode 100644 tests/test_crawl_html_storage.py create mode 100644 tests/test_html_store.py create mode 100644 web/app/api/crawl/page-html/route.ts create mode 100644 web/src/components/pipeline/CrawlPageHtmlManager.tsx create mode 100644 web/src/server/crawlPageHtmlRoute.test.ts diff --git a/alembic/versions/015_crawl_page_html.py b/alembic/versions/015_crawl_page_html.py new file mode 100644 index 00000000..8c894ad8 --- /dev/null +++ b/alembic/versions/015_crawl_page_html.py @@ -0,0 +1,38 @@ +"""Add crawl_page_html table for per-URL raw HTML storage. + +Revision ID: 015_crawl_page_html +Revises: 014_pipeline_log_truncated +""" +from __future__ import annotations + +from alembic import op + +revision = "015_crawl_page_html" +down_revision = "014_pipeline_log_truncated" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE crawl_page_html ( + crawl_run_id BIGINT NOT NULL REFERENCES crawl_runs(id) ON DELETE CASCADE, + url TEXT NOT NULL, + html TEXT NOT NULL, + status TEXT, + content_type TEXT, + fetch_method TEXT NOT NULL DEFAULT 'static', + byte_length INTEGER NOT NULL DEFAULT 0, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (crawl_run_id, url) + ); + CREATE INDEX IF NOT EXISTS idx_crawl_page_html_run + ON crawl_page_html (crawl_run_id); + """) + + +def downgrade() -> None: + op.execute(""" + DROP INDEX IF EXISTS idx_crawl_page_html_run; + DROP TABLE IF EXISTS crawl_page_html; + """) diff --git a/input.txt.example b/input.txt.example index a6a9c56e..7494b553 100644 --- a/input.txt.example +++ b/input.txt.example @@ -15,6 +15,11 @@ allow_external = false store_outlinks = true store_content_excerpt = true content_excerpt_max_chars = 4096 +store_page_html = false +max_stored_html_bytes = 2097152 +run_content_analysis = false +content_analysis_strategy = main_only +content_analysis_workers = 4 preserve_crawl_history = true crawl_stream_to_db = false crawl_exclude_urls = diff --git a/pipeline-config.example.txt b/pipeline-config.example.txt index a43055d3..d1957b9e 100644 --- a/pipeline-config.example.txt +++ b/pipeline-config.example.txt @@ -16,6 +16,11 @@ allow_external = false store_outlinks = true store_content_excerpt = true content_excerpt_max_chars = 4096 +store_page_html = false +max_stored_html_bytes = 2097152 +run_content_analysis = false +content_analysis_strategy = main_only +content_analysis_workers = 4 preserve_crawl_history = true crawl_stream_to_db = false crawl_exclude_urls = diff --git a/src/website_profiling/commands/config_resolve.py b/src/website_profiling/commands/config_resolve.py index 690a8a20..5468b51d 100644 --- a/src/website_profiling/commands/config_resolve.py +++ b/src/website_profiling/commands/config_resolve.py @@ -268,6 +268,7 @@ def build_parser() -> argparse.ArgumentParser: nargs="?", choices=[ "crawl", + "content_analysis", "report", "plot", "lighthouse", diff --git a/src/website_profiling/commands/pipeline_cmd.py b/src/website_profiling/commands/pipeline_cmd.py index 24c00679..d50078fa 100644 --- a/src/website_profiling/commands/pipeline_cmd.py +++ b/src/website_profiling/commands/pipeline_cmd.py @@ -125,6 +125,10 @@ def run(cfg: dict, args: argparse.Namespace) -> None: use_database = True run_crawl = args.command == "crawl" or (args.command is None and get_bool(cfg, "run_crawl", True)) + run_content_analysis = ( + args.command == "content_analysis" + or (args.command is None and get_bool(cfg, "run_content_analysis", False)) + ) run_report = args.command == "report" or (args.command is None and get_bool(cfg, "run_report", True)) run_plot = args.command == "plot" or (args.command is None and get_bool(cfg, "run_plot", False)) run_lighthouse = args.command is None and get_bool(cfg, "run_lighthouse", False) @@ -132,12 +136,14 @@ def run(cfg: dict, args: argparse.Namespace) -> None: lighthouse_max_pages = _cfg_int(cfg, "lighthouse_max_pages", 20) if args.command is None and ( - run_crawl or run_lighthouse or run_lighthouse_on_pages or run_report or run_plot + run_crawl or run_content_analysis or run_lighthouse or run_lighthouse_on_pages or run_report or run_plot ): emit_phase_start("config", message="Resolving pipeline configuration") steps = [] if run_crawl: steps.append("crawl") + if run_content_analysis: + steps.append("content-analysis") if run_lighthouse_on_pages: steps.append("lighthouse-on-pages") elif run_lighthouse: @@ -154,6 +160,11 @@ def run(cfg: dict, args: argparse.Namespace) -> None: if run_crawl: phase_results.append(run_pipeline_phase("crawl", lambda: _run_crawl(cfg, use_database))) + if run_content_analysis and use_database: + phase_results.append( + run_pipeline_phase("content_analysis", lambda: _run_content_analysis(cfg, use_database)) + ) + if run_lighthouse_on_pages and use_database: phase_results.append( run_pipeline_phase( @@ -215,6 +226,11 @@ def _run_crawl(cfg: dict, use_database: bool) -> None: preserve_crawl_history = get_bool(cfg, "preserve_crawl_history", True) store_content_excerpt = get_bool(cfg, "store_content_excerpt", False) content_excerpt_max_chars = _cfg_int(cfg, "content_excerpt_max_chars", 4096) + store_page_html = get_bool(cfg, "store_page_html", False) + max_stored_html_bytes = _cfg_int(cfg, "max_stored_html_bytes", 2_097_152) + run_content_analysis = get_bool(cfg, "run_content_analysis", False) + content_analysis_strategy = (cfg.get("content_analysis_strategy") or "main_only").strip() + content_analysis_workers = _cfg_int(cfg, "content_analysis_workers", 4) crawl_stream_to_db = get_bool(cfg, "crawl_stream_to_db", False) property_id = active_property_id_from_cfg(cfg) render_mode = _normalize_render_mode(cfg) @@ -258,6 +274,11 @@ def _run_crawl(cfg: dict, use_database: bool) -> None: preserve_crawl_history=preserve_crawl_history, store_content_excerpt=store_content_excerpt, content_excerpt_max_chars=content_excerpt_max_chars, + store_page_html=store_page_html, + max_stored_html_bytes=max_stored_html_bytes, + run_content_analysis=run_content_analysis, + content_analysis_strategy=content_analysis_strategy, + content_analysis_workers=content_analysis_workers, crawl_stream_to_db=crawl_stream_to_db, property_id=property_id, render_mode=render_mode, @@ -289,6 +310,33 @@ def _run_crawl(cfg: dict, use_database: bool) -> None: console_print("Crawl results: PostgreSQL") +def _run_content_analysis(cfg: dict, use_database: bool) -> None: + if not use_database: + console_print("[Content analysis] Skipped (database required).", flush=True) + return + if not get_bool(cfg, "store_page_html", False): + console_print( + "[Content analysis] Skipped: enable store_page_html to persist HTML for analysis.", + flush=True, + ) + return + + from ..content_analysis import run_content_analysis + + store_content_excerpt = get_bool(cfg, "store_content_excerpt", False) + excerpt_max = _cfg_int(cfg, "content_excerpt_max_chars", 4096) + strategy = (cfg.get("content_analysis_strategy") or "main_only").strip().lower() + workers = _cfg_int(cfg, "content_analysis_workers", 4) + + console_print("[Content analysis] Starting...", flush=True) + run_content_analysis( + excerpt_max_chars=excerpt_max if store_content_excerpt else 0, + strategy=strategy, + workers=workers, + ) + console_print("[Content analysis] Done.", flush=True) + + def _run_lighthouse_on_pages(cfg: dict, lighthouse_max_pages: int) -> None: from ..db import db_session, get_latest_crawl_run_id, read_crawl from ..lighthouse.runner import run_lighthouse_on_pages as do_lighthouse_on_pages diff --git a/src/website_profiling/content_analysis/__init__.py b/src/website_profiling/content_analysis/__init__.py new file mode 100644 index 00000000..71e1aded --- /dev/null +++ b/src/website_profiling/content_analysis/__init__.py @@ -0,0 +1,7 @@ +"""Post-crawl content analysis from stored HTML.""" +from __future__ import annotations + +from .page import analyze_page_html +from .pipeline import run_content_analysis + +__all__ = ["analyze_page_html", "run_content_analysis"] diff --git a/src/website_profiling/content_analysis/batch.py b/src/website_profiling/content_analysis/batch.py new file mode 100644 index 00000000..7e77a686 --- /dev/null +++ b/src/website_profiling/content_analysis/batch.py @@ -0,0 +1,78 @@ +"""Batch content analysis for a crawl run.""" +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Iterator + +from psycopg import Connection + +from ..db.html_store import read_page_html_for_run +from .page import ContentStrategy, analyze_page_html + +_PAGE_BATCH = 500 + + +def iter_html_pages(conn: Connection, crawl_run_id: int) -> Iterator[dict[str, Any]]: + offset = 0 + while True: + chunk = list(read_page_html_for_run(conn, crawl_run_id, limit=_PAGE_BATCH, offset=offset)) + if not chunk: + break + for row in chunk: + yield row + if len(chunk) < _PAGE_BATCH: + break + offset += _PAGE_BATCH + + +def _analyze_row( + row: dict[str, Any], + *, + excerpt_max_chars: int, + strategy: ContentStrategy, +) -> dict[str, Any] | None: + html = row.get("html") + url = row.get("url") + if not url or not html: + return None + fields = analyze_page_html( + str(html), + excerpt_max_chars=excerpt_max_chars, + strategy=strategy, + ) + return {"url": str(url).rstrip("/"), **fields} + + +def analyze_run_html( + conn: Connection, + crawl_run_id: int, + *, + excerpt_max_chars: int = 0, + strategy: ContentStrategy = "main_only", + workers: int = 4, +) -> list[dict[str, Any]]: + """Analyze all stored HTML for a crawl run; returns merge payloads keyed by url.""" + rows = list(iter_html_pages(conn, crawl_run_id)) + if not rows: + return [] + + worker_count = max(1, int(workers)) + if worker_count == 1 or len(rows) == 1: + out: list[dict[str, Any]] = [] + for row in rows: + merged = _analyze_row(row, excerpt_max_chars=excerpt_max_chars, strategy=strategy) + if merged: + out.append(merged) + return out + + results: list[dict[str, Any]] = [] + with ThreadPoolExecutor(max_workers=worker_count) as pool: + futures = [ + pool.submit(_analyze_row, row, excerpt_max_chars=excerpt_max_chars, strategy=strategy) + for row in rows + ] + for fut in as_completed(futures): + merged = fut.result() + if merged: + results.append(merged) + return results diff --git a/src/website_profiling/content_analysis/constants.py b/src/website_profiling/content_analysis/constants.py new file mode 100644 index 00000000..d1d7b7c8 --- /dev/null +++ b/src/website_profiling/content_analysis/constants.py @@ -0,0 +1,17 @@ +"""Shared constants for content analysis.""" +from __future__ import annotations + +STOP_WORDS = frozenset({ + "the", "and", "for", "that", "this", "with", "from", "your", "have", "are", + "was", "were", "been", "will", "would", "could", "should", "about", "which", + "their", "there", "what", "when", "where", "more", "some", "than", "them", + "other", "into", "over", "also", "just", "after", "before", "only", "then", + "very", "most", "each", "such", "like", "does", "here", "because", "being", + "well", "while", "these", "those", "both", "many", "much", "even", "back", + "through", "still", "between", "every", "under", "last", "long", "great", + "make", "same", "come", "take", "know", "they", "page", "site", "home", + "click", "read", "view", "next", "menu", "main", "skip", "content", "link", + "http", "https", "www", "html", "class", "none", "true", "false", "null", +}) + +CHROME_TAGS = ("nav", "footer", "header", "aside") diff --git a/src/website_profiling/content_analysis/dom_cleanup.py b/src/website_profiling/content_analysis/dom_cleanup.py new file mode 100644 index 00000000..b35f554b --- /dev/null +++ b/src/website_profiling/content_analysis/dom_cleanup.py @@ -0,0 +1,16 @@ +"""Remove non-content DOM nodes before text extraction.""" +from __future__ import annotations + +from bs4 import BeautifulSoup + +from .constants import CHROME_TAGS + + +def cleanup_dom(soup: BeautifulSoup) -> BeautifulSoup: + for tag in soup.find_all(["script", "style", "noscript"]): + tag.decompose() + for tag in soup.find_all(CHROME_TAGS): + tag.decompose() + for tag in soup.find_all(attrs={"aria-hidden": "true"}): + tag.decompose() + return soup diff --git a/src/website_profiling/content_analysis/excerpt.py b/src/website_profiling/content_analysis/excerpt.py new file mode 100644 index 00000000..ae33a75e --- /dev/null +++ b/src/website_profiling/content_analysis/excerpt.py @@ -0,0 +1,14 @@ +"""Plain-text excerpt generation.""" +from __future__ import annotations + +import re + + +def build_excerpt(body_text: str, max_chars: int) -> str: + if not max_chars or max_chars <= 0 or not body_text: + return "" + excerpt = re.sub(r"\s+", " ", body_text.strip()) + if len(excerpt) <= max_chars: + return excerpt + truncated = excerpt[:max_chars].rsplit(" ", 1)[0].strip() + return truncated or excerpt[:max_chars] diff --git a/src/website_profiling/content_analysis/html_loader.py b/src/website_profiling/content_analysis/html_loader.py new file mode 100644 index 00000000..9e34a2f5 --- /dev/null +++ b/src/website_profiling/content_analysis/html_loader.py @@ -0,0 +1,8 @@ +"""Load BeautifulSoup from raw HTML.""" +from __future__ import annotations + +from bs4 import BeautifulSoup + + +def load_soup(raw_html: str) -> BeautifulSoup: + return BeautifulSoup(raw_html or "", "lxml") diff --git a/src/website_profiling/content_analysis/html_ratio.py b/src/website_profiling/content_analysis/html_ratio.py new file mode 100644 index 00000000..f14f399b --- /dev/null +++ b/src/website_profiling/content_analysis/html_ratio.py @@ -0,0 +1,7 @@ +"""Content-to-HTML size ratio.""" +from __future__ import annotations + + +def content_html_ratio(body_text: str, raw_html: str) -> float: + html_len = max(1, len(raw_html or "")) + return round(len(body_text or "") / html_len * 100, 1) diff --git a/src/website_profiling/content_analysis/keywords.py b/src/website_profiling/content_analysis/keywords.py new file mode 100644 index 00000000..b6368c3c --- /dev/null +++ b/src/website_profiling/content_analysis/keywords.py @@ -0,0 +1,18 @@ +"""Per-page keyword extraction.""" +from __future__ import annotations + +import json +from collections import Counter + +from .constants import STOP_WORDS + + +def top_keywords_json(words: list[str], *, limit: int = 10) -> str: + keyword_words = [w.lower() for w in words if len(w) >= 4 and w.lower() not in STOP_WORDS] + top_keywords = Counter(keyword_words).most_common(limit) + max_kw = top_keywords[0][1] if top_keywords else 0 + kw_rows = [] + for w, c in top_keywords: + score = round(100 * c / max_kw) if max_kw else 0 + kw_rows.append({"word": w, "count": c, "score": int(score)}) + return json.dumps(kw_rows) diff --git a/src/website_profiling/content_analysis/main_content.py b/src/website_profiling/content_analysis/main_content.py new file mode 100644 index 00000000..0a00e956 --- /dev/null +++ b/src/website_profiling/content_analysis/main_content.py @@ -0,0 +1,25 @@ +"""Select the primary content root element from a page.""" +from __future__ import annotations + +from typing import Literal + +from bs4 import BeautifulSoup, Tag + +ContentStrategy = Literal["main_only", "full_body"] + + +def find_main_content(soup: BeautifulSoup, strategy: ContentStrategy = "main_only") -> Tag | BeautifulSoup: + if strategy == "full_body": + return soup.find("body") or soup + + candidates: list[Tag | None] = [ + soup.find("main"), + soup.find("article"), + soup.find(attrs={"role": "main"}), + soup.find(id="content"), + soup.find(class_="content"), + ] + for el in candidates: + if el is not None and (el.get_text(separator=" ", strip=True) or "").strip(): + return el + return soup.find("body") or soup diff --git a/src/website_profiling/content_analysis/page.py b/src/website_profiling/content_analysis/page.py new file mode 100644 index 00000000..6384dc4a --- /dev/null +++ b/src/website_profiling/content_analysis/page.py @@ -0,0 +1,45 @@ +"""Per-page content analysis orchestration.""" +from __future__ import annotations + +from typing import Literal + +from .dom_cleanup import cleanup_dom +from .excerpt import build_excerpt +from .html_loader import load_soup +from .html_ratio import content_html_ratio +from .keywords import top_keywords_json +from .main_content import find_main_content +from .reading_level import flesch_kincaid_grade +from .text_extract import extract_text +from .tokenize import count_words, tokenize_words + +ContentStrategy = Literal["main_only", "full_body"] + +CONTENT_FIELDS = ( + "word_count", + "reading_level", + "content_html_ratio", + "top_keywords", + "content_excerpt", +) + + +def analyze_page_html( + raw_html: str, + *, + excerpt_max_chars: int = 0, + strategy: ContentStrategy = "main_only", +) -> dict: + """Analyze stored HTML and return crawl row content fields.""" + soup = load_soup(raw_html) + cleaned = cleanup_dom(soup) + root = find_main_content(cleaned, strategy=strategy) + body_text = extract_text(root) + words = tokenize_words(body_text) + return { + "word_count": count_words(words), + "reading_level": flesch_kincaid_grade(words, body_text), + "content_html_ratio": content_html_ratio(body_text, raw_html), + "top_keywords": top_keywords_json(words), + "content_excerpt": build_excerpt(body_text, excerpt_max_chars), + } diff --git a/src/website_profiling/content_analysis/pipeline.py b/src/website_profiling/content_analysis/pipeline.py new file mode 100644 index 00000000..97f7a3d6 --- /dev/null +++ b/src/website_profiling/content_analysis/pipeline.py @@ -0,0 +1,50 @@ +"""Pipeline entrypoint for post-crawl content analysis.""" +from __future__ import annotations + +from typing import Any, Optional + +from ..console_io import console_print +from ..db import db_session, get_latest_crawl_run_id +from ..db.crawl_store import merge_crawl_result_fields_batch +from ..progress import emit_phase_done, emit_phase_start, emit_progress +from .batch import analyze_run_html +from .page import ContentStrategy + + +def run_content_analysis( + crawl_run_id: Optional[int] = None, + *, + excerpt_max_chars: int = 0, + strategy: str = "main_only", + workers: int = 4, +) -> dict[str, Any]: + """Analyze stored HTML for a crawl run and merge metrics into crawl_results.""" + emit_phase_start("content_analysis", message="Analyzing stored page HTML") + console_print(" Content analysis: reading stored HTML...", flush=True) + + strat: ContentStrategy = "full_body" if strategy == "full_body" else "main_only" + summary: dict[str, Any] = {"crawl_run_id": None, "pages_analyzed": 0, "strategy": strat} + + with db_session() as conn: + run_id = crawl_run_id if crawl_run_id is not None else get_latest_crawl_run_id(conn) + if run_id is None: + console_print(" Content analysis skipped: no crawl run in database.", flush=True) + emit_phase_done("content_analysis") + return summary + + summary["crawl_run_id"] = int(run_id) + emit_progress("content_analysis", "analyze", message="Analyzing page HTML") + updates = analyze_run_html( + conn, + int(run_id), + excerpt_max_chars=excerpt_max_chars, + strategy=strat, + workers=workers, + ) + if updates: + merge_crawl_result_fields_batch(conn, int(run_id), updates) + summary["pages_analyzed"] = len(updates) + + console_print(f" Content analysis complete ({summary['pages_analyzed']} pages).", flush=True) + emit_phase_done("content_analysis") + return summary diff --git a/src/website_profiling/content_analysis/reading_level.py b/src/website_profiling/content_analysis/reading_level.py new file mode 100644 index 00000000..92e6876a --- /dev/null +++ b/src/website_profiling/content_analysis/reading_level.py @@ -0,0 +1,39 @@ +"""Reading level (Flesch-Kincaid grade) helpers.""" +from __future__ import annotations + +import re + + +def count_syllables(word: str) -> int: + word = word.lower().strip() + if len(word) <= 3: + return 1 + vowels = "aeiouy" + count = 0 + prev_vowel = False + for ch in word: + is_vowel = ch in vowels + if is_vowel and not prev_vowel: + count += 1 + prev_vowel = is_vowel + if word.endswith("e") and count > 1: + count -= 1 + return max(1, count) + + +def split_sentences(body_text: str) -> list[str]: + return [s.strip() for s in re.split(r"[.!?]+", body_text or "") if len(s.strip()) > 5] + + +def flesch_kincaid_grade(words: list[str], body_text: str) -> float: + word_count = len(words) + if word_count <= 30: + return 0.0 + sentence_count = max(1, len(split_sentences(body_text))) + total_syllables = sum(count_syllables(w) for w in words) + reading_level = ( + 0.39 * (word_count / sentence_count) + + 11.8 * (total_syllables / max(1, word_count)) + - 15.59 + ) + return max(0.0, min(18.0, round(reading_level, 1))) diff --git a/src/website_profiling/content_analysis/text_extract.py b/src/website_profiling/content_analysis/text_extract.py new file mode 100644 index 00000000..e65cccdf --- /dev/null +++ b/src/website_profiling/content_analysis/text_extract.py @@ -0,0 +1,8 @@ +"""Plain-text extraction from a DOM subtree.""" +from __future__ import annotations + +from bs4 import BeautifulSoup, Tag + + +def extract_text(root: Tag | BeautifulSoup) -> str: + return root.get_text(separator=" ", strip=True) if root is not None else "" diff --git a/src/website_profiling/content_analysis/tokenize.py b/src/website_profiling/content_analysis/tokenize.py new file mode 100644 index 00000000..b5bc86db --- /dev/null +++ b/src/website_profiling/content_analysis/tokenize.py @@ -0,0 +1,12 @@ +"""Word tokenization for content metrics.""" +from __future__ import annotations + +import re + + +def tokenize_words(body_text: str) -> list[str]: + return [w for w in re.findall(r"[a-zA-Z]+", body_text or "") if len(w) >= 2] + + +def count_words(tokens: list[str]) -> int: + return len(tokens) diff --git a/src/website_profiling/crawl/config.py b/src/website_profiling/crawl/config.py index 22f09898..2acbffe2 100644 --- a/src/website_profiling/crawl/config.py +++ b/src/website_profiling/crawl/config.py @@ -41,6 +41,11 @@ class CrawlConfig: use_wappalyzer: bool = True store_content_excerpt: bool = False content_excerpt_max_chars: int = 4096 + store_page_html: bool = False + max_stored_html_bytes: int = 2_097_152 + run_content_analysis: bool = False + content_analysis_strategy: str = "main_only" + content_analysis_workers: int = 4 render_mode: str = "static" js_concurrency: int = 3 js_timeout: int = 30 @@ -85,6 +90,12 @@ def normalized(self) -> CrawlConfig: self.exclude_urls = list(self.exclude_urls) if self.exclude_urls else [] self.store_content_excerpt = bool(self.store_content_excerpt) self.content_excerpt_max_chars = max(0, int(self.content_excerpt_max_chars or 0)) + self.store_page_html = bool(self.store_page_html) + self.max_stored_html_bytes = max(1, int(self.max_stored_html_bytes or 2_097_152)) + self.run_content_analysis = bool(self.run_content_analysis) + strat = (self.content_analysis_strategy or "main_only").strip().lower() + self.content_analysis_strategy = strat if strat in ("main_only", "full_body") else "main_only" + self.content_analysis_workers = max(1, int(self.content_analysis_workers or 4)) self.custom_extraction_regex = (self.custom_extraction_regex or "").strip() self.custom_extractors = list(self.custom_extractors or []) self.crawl_ignore_params = list(self.crawl_ignore_params or []) @@ -98,6 +109,10 @@ def normalized(self) -> CrawlConfig: ) return self + @property + def defer_content_analysis(self) -> bool: + return self.store_page_html and self.run_content_analysis + @property def effective_concurrency(self) -> int: if self.render_mode == "javascript": diff --git a/src/website_profiling/crawl/crawler.py b/src/website_profiling/crawl/crawler.py index faa62b54..a00b4c9d 100644 --- a/src/website_profiling/crawl/crawler.py +++ b/src/website_profiling/crawl/crawler.py @@ -61,6 +61,11 @@ def __init__( use_wappalyzer: bool = True, store_content_excerpt: bool = False, content_excerpt_max_chars: int = 4096, + store_page_html: bool = False, + max_stored_html_bytes: int = 2_097_152, + run_content_analysis: bool = False, + content_analysis_strategy: str = "main_only", + content_analysis_workers: int = 4, render_mode: str = "static", js_concurrency: int = 3, js_timeout: int = 30, @@ -103,6 +108,11 @@ def __init__( use_wappalyzer=use_wappalyzer, store_content_excerpt=store_content_excerpt, content_excerpt_max_chars=content_excerpt_max_chars, + store_page_html=store_page_html, + max_stored_html_bytes=max_stored_html_bytes, + run_content_analysis=run_content_analysis, + content_analysis_strategy=content_analysis_strategy, + content_analysis_workers=content_analysis_workers, render_mode=render_mode, js_concurrency=js_concurrency, js_timeout=js_timeout, @@ -145,11 +155,16 @@ def __init__( self.crawl_ignore_params = config.crawl_ignore_params self.custom_extraction_regex = config.custom_extraction_regex self.custom_extractors = config.custom_extractors + self.store_page_html = config.store_page_html + self.max_stored_html_bytes = config.max_stored_html_bytes + self._html_buffer: list[dict] = [] + self._db_writer: Optional[CrawlDbWriter] = None self.page_builder = PageRecordBuilder( use_wappalyzer=config.use_wappalyzer, store_content_excerpt=config.store_content_excerpt, content_excerpt_max_chars=config.content_excerpt_max_chars, + defer_content_analysis=config.defer_content_analysis, custom_extraction_regex=config.custom_extraction_regex, custom_extractors=config.custom_extractors, ) @@ -229,6 +244,32 @@ def _queue_contains(self, item: str) -> bool: def fetch(self, url: str) -> FetchResult: return self.fetcher.fetch(url) + def _capture_page_html( + self, + url: str, + text: str | None, + status: object, + content_type: str | None, + fetch_method: str, + ) -> None: + from .html_capture import build_page_html_record + + record = build_page_html_record( + url=url, + html=text or "", + status=status, + content_type=content_type, + fetch_method=fetch_method, + max_bytes=self.max_stored_html_bytes, + enabled=self.store_page_html, + ) + if record is None: + return + if self._db_writer is not None: + self._db_writer.enqueue_html(record) + else: + self._html_buffer.append(record) + def worker(self, url: str) -> dict: if not self.allowed_by_robots(url): return PageRecordBuilder.build_robots_blocked_row( @@ -340,6 +381,9 @@ def worker(self, url: str) -> dict: PageRecordBuilder.merge_browser_diagnostics(ext, result) + if text: + self._capture_page_html(url, text, status, ct, fetch_method) + res = { "url": url, "status": status, @@ -372,8 +416,15 @@ def crawl( futures = [] db_writer: Optional[CrawlDbWriter] = None pages_crawled = 0 + self._db_writer = None + self._html_buffer = [] if stream_crawl_run_id is not None: - db_writer = _CrawlDbWriter(stream_crawl_run_id, stream_batch_size) + db_writer = _CrawlDbWriter( + stream_crawl_run_id, + stream_batch_size, + store_page_html=self.store_page_html, + ) + self._db_writer = db_writer db_writer.start() use_tqdm = show_progress and stream_crawl_run_id is None pbar = tqdm( @@ -439,6 +490,7 @@ def crawl( db_writer.finish() db_writer.join() db_writer.raise_if_failed() + self._db_writer = None elapsed = time.time() - start_time df = pd.DataFrame(self.results) if df.empty: @@ -464,6 +516,11 @@ def run_crawler( preserve_crawl_history: bool = True, store_content_excerpt: bool = False, content_excerpt_max_chars: int = 4096, + store_page_html: bool = False, + max_stored_html_bytes: int = 2_097_152, + run_content_analysis: bool = False, + content_analysis_strategy: str = "main_only", + content_analysis_workers: int = 4, crawl_stream_to_db: bool = False, property_id: Optional[int] = None, render_mode: str = "static", @@ -513,6 +570,11 @@ def run_crawler( exclude_urls=exclude_urls, store_content_excerpt=store_content_excerpt, content_excerpt_max_chars=content_excerpt_max_chars, + store_page_html=store_page_html, + max_stored_html_bytes=max_stored_html_bytes, + run_content_analysis=run_content_analysis, + content_analysis_strategy=content_analysis_strategy, + content_analysis_workers=content_analysis_workers, render_mode=render_mode, js_concurrency=js_concurrency, js_timeout=js_timeout, @@ -602,6 +664,11 @@ def run_crawler( discovery_mode=disc_label, ) write_crawl(conn, df, crawl_run_id=run_id) + html_buffer = getattr(crawler, "_html_buffer", None) or [] + if getattr(crawler, "store_page_html", False) and html_buffer: + from ..db.html_store import write_page_html_batch + + write_page_html_batch(conn, html_buffer, run_id, commit=True) if crawler.link_edges_accum: from ..db.crawl_store import write_link_edges diff --git a/src/website_profiling/crawl/db_writer.py b/src/website_profiling/crawl/db_writer.py index ac78af99..302ba7fc 100644 --- a/src/website_profiling/crawl/db_writer.py +++ b/src/website_profiling/crawl/db_writer.py @@ -1,4 +1,4 @@ -"""Background thread: batch-insert crawl rows via PostgreSQL connection pool.""" +"""Background thread: batch-insert crawl rows and optional HTML via PostgreSQL connection pool.""" from __future__ import annotations @@ -10,43 +10,68 @@ class CrawlDbWriter(threading.Thread): - """Background thread: batch-insert crawl rows via PostgreSQL connection pool.""" + """Background thread: batch-insert crawl rows and optional page HTML.""" - def __init__(self, crawl_run_id: int, batch_size: int = 500) -> None: + def __init__(self, crawl_run_id: int, batch_size: int = 500, *, store_page_html: bool = False) -> None: super().__init__(daemon=True) self.crawl_run_id = crawl_run_id self.batch_size = max(50, batch_size) + self.store_page_html = bool(store_page_html) self._queue: Queue = Queue() self._error: Optional[BaseException] = None def enqueue(self, record: dict) -> None: - self._queue.put(record) + self._queue.put(("crawl", record)) + + def enqueue_html(self, record: dict) -> None: + if not self.store_page_html: + return + self._queue.put(("html", record)) def finish(self) -> None: self._queue.put(None) - def run(self) -> None: + def _flush_crawl(self, buffer: list[dict]) -> None: + if not buffer: + return from ..db import db_session from ..db.crawl_store import _crawl_rows_from_df, write_crawl_batch - buffer: list[dict] = [] + chunk = pd.DataFrame(buffer) + with db_session() as conn: + rows = _crawl_rows_from_df(chunk, self.crawl_run_id) + write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) + + def _flush_html(self, buffer: list[dict]) -> None: + if not buffer: + return + from ..db import db_session + from ..db.html_store import write_page_html_batch + + with db_session() as conn: + write_page_html_batch(conn, buffer, self.crawl_run_id, commit=True) + + def run(self) -> None: + crawl_buffer: list[dict] = [] + html_buffer: list[dict] = [] try: while True: item = self._queue.get() if item is None: - if buffer: - chunk = pd.DataFrame(buffer) - with db_session() as conn: - rows = _crawl_rows_from_df(chunk, self.crawl_run_id) - write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) + self._flush_crawl(crawl_buffer) + self._flush_html(html_buffer) break - buffer.append(item) - if len(buffer) >= self.batch_size: - chunk = pd.DataFrame(buffer) - buffer = [] - with db_session() as conn: - rows = _crawl_rows_from_df(chunk, self.crawl_run_id) - write_crawl_batch(conn, rows, self.crawl_run_id, commit=True) + kind, payload = item + if kind == "html": + html_buffer.append(payload) + if len(html_buffer) >= self.batch_size: + self._flush_html(html_buffer) + html_buffer = [] + else: + crawl_buffer.append(payload) + if len(crawl_buffer) >= self.batch_size: + self._flush_crawl(crawl_buffer) + crawl_buffer = [] except BaseException as e: self._error = e diff --git a/src/website_profiling/crawl/html_capture.py b/src/website_profiling/crawl/html_capture.py new file mode 100644 index 00000000..021947ee --- /dev/null +++ b/src/website_profiling/crawl/html_capture.py @@ -0,0 +1,60 @@ +"""Helpers for deciding whether and how to persist fetched HTML.""" +from __future__ import annotations + +from typing import Any, Optional + + +def _is_html_content_type(content_type: str | None) -> bool: + ct = (content_type or "").lower() + return "text/html" in ct or "application/xhtml+xml" in ct + + +def should_store_page_html( + *, + enabled: bool, + status: Any, + content_type: str | None, + html: str | None, + max_bytes: int, +) -> bool: + """True when HTML should be persisted for a crawl URL.""" + if not enabled or not html or not str(html).strip(): + return False + try: + if int(status) != 200: + return False + except (TypeError, ValueError): + return False + if not _is_html_content_type(content_type): + return False + return len(str(html).encode("utf-8")) <= max(1, int(max_bytes)) + + +def build_page_html_record( + *, + url: str, + html: str, + status: Any, + content_type: str | None, + fetch_method: str, + max_bytes: int, + enabled: bool = True, +) -> Optional[dict[str, Any]]: + """Build a side-channel HTML record, or None if storage should be skipped.""" + if not should_store_page_html( + enabled=enabled, + status=status, + content_type=content_type, + html=html, + max_bytes=max_bytes, + ): + return None + text = str(html) + return { + "url": str(url or "").rstrip("/"), + "html": text, + "status": str(status), + "content_type": str(content_type or ""), + "fetch_method": str(fetch_method or "static").strip() or "static", + "byte_length": len(text.encode("utf-8")), + } diff --git a/src/website_profiling/crawl/page_record.py b/src/website_profiling/crawl/page_record.py index b4c89791..c76322a1 100644 --- a/src/website_profiling/crawl/page_record.py +++ b/src/website_profiling/crawl/page_record.py @@ -34,12 +34,14 @@ def __init__( use_wappalyzer: bool = True, store_content_excerpt: bool = False, content_excerpt_max_chars: int = 4096, + defer_content_analysis: bool = False, custom_extraction_regex: str = "", custom_extractors: Optional[list[dict]] = None, ) -> None: self.use_wappalyzer = use_wappalyzer self.store_content_excerpt = store_content_excerpt self.content_excerpt_max_chars = content_excerpt_max_chars + self.defer_content_analysis = defer_content_analysis self.custom_extraction_regex = custom_extraction_regex self.custom_extractors = list(custom_extractors or []) self._wappalyzer_instance = None @@ -89,12 +91,13 @@ def parse_page_content( _soup = _BS(text, "lxml") excerpt_max = self.content_excerpt_max_chars if self.store_content_excerpt else 0 - ct_data = parse_content_text(_soup, text, excerpt_max_chars=excerpt_max) - ext["word_count"] = ct_data.get("word_count", 0) - ext["reading_level"] = ct_data.get("reading_level", 0.0) - ext["content_html_ratio"] = ct_data.get("content_html_ratio", 0.0) - ext["top_keywords"] = ct_data.get("top_keywords", "[]") - ext["content_excerpt"] = ct_data.get("content_excerpt") or "" + if not self.defer_content_analysis: + ct_data = parse_content_text(_soup, text, excerpt_max_chars=excerpt_max) + ext["word_count"] = ct_data.get("word_count", 0) + ext["reading_level"] = ct_data.get("reading_level", 0.0) + ext["content_html_ratio"] = ct_data.get("content_html_ratio", 0.0) + ext["top_keywords"] = ct_data.get("top_keywords", "[]") + ext["content_excerpt"] = ct_data.get("content_excerpt") or "" social = parse_social_meta(_soup) ext["og_title"] = social.get("og_title", "") ext["og_description"] = social.get("og_description", "") diff --git a/src/website_profiling/db/crawl_store.py b/src/website_profiling/db/crawl_store.py index a4122d17..fc50d327 100644 --- a/src/website_profiling/db/crawl_store.py +++ b/src/website_profiling/db/crawl_store.py @@ -234,6 +234,10 @@ def write_crawl(conn: Connection, df: pd.DataFrame, crawl_run_id: Optional[int] with conn.transaction(): if crawl_run_id is not None: conn.execute("DELETE FROM crawl_results WHERE crawl_run_id = %s", (crawl_run_id,)) + try: + conn.execute("DELETE FROM crawl_page_html WHERE crawl_run_id = %s", (crawl_run_id,)) + except Exception: + pass target_run_id = crawl_run_id else: conn.execute("DELETE FROM crawl_results") @@ -251,6 +255,39 @@ def write_crawl(conn: Connection, df: pd.DataFrame, crawl_run_id: Optional[int] _write_crawl_rows(conn, rows) +_MERGE_FIELDS_BATCH_SIZE = 200 +_MERGE_FIELDS_SQL = """UPDATE crawl_results +SET data = COALESCE(data, '{}'::jsonb) || %s::jsonb +WHERE crawl_run_id = %s AND url = %s""" + + +def merge_crawl_result_fields_batch( + conn: Connection, + crawl_run_id: int, + updates: list[dict[str, Any]], + *, + commit: bool = True, +) -> int: + """Merge per-URL content fields into crawl_results.data JSONB. Returns rows updated.""" + if not updates: + return 0 + params: list[tuple] = [] + for item in updates: + url = str(item.get("url") or "").rstrip("/") + if not url: + continue + fields = {k: _sanitize_for_json(v) for k, v in item.items() if k != "url"} + if not fields: + continue + params.append((_json_val(fields), crawl_run_id, url)) + if not params: + return 0 + _executemany(conn, _MERGE_FIELDS_SQL, params, page_size=_MERGE_FIELDS_BATCH_SIZE) + if commit: + conn.commit() + return len(params) + + def read_crawl(conn: Connection, run_id: Optional[int] = None) -> pd.DataFrame: try: return _read_crawl_rows(conn, run_id, include_fetch_method=True) diff --git a/src/website_profiling/db/historical.py b/src/website_profiling/db/historical.py index 0879c28e..b1e62798 100644 --- a/src/website_profiling/db/historical.py +++ b/src/website_profiling/db/historical.py @@ -207,7 +207,7 @@ def _bulk( def ensure_crawl_tables_cleared(conn: Connection) -> None: """Clear crawl-scoped tables before a non-append crawl (preserves reports, Google, etc.).""" - conn.execute("TRUNCATE crawl_results, edges, nodes RESTART IDENTITY CASCADE") + conn.execute("TRUNCATE crawl_results, crawl_page_html, edges, nodes RESTART IDENTITY CASCADE") conn.commit() diff --git a/src/website_profiling/db/html_store.py b/src/website_profiling/db/html_store.py new file mode 100644 index 00000000..5b5a96d5 --- /dev/null +++ b/src/website_profiling/db/html_store.py @@ -0,0 +1,112 @@ +"""Per-URL raw HTML storage for crawl runs.""" +from __future__ import annotations + +from typing import Any, Iterator, Optional + +from psycopg import Connection + +from ._common import _executemany, _now_iso + +_HTML_BATCH_SIZE = 200 + +_HTML_UPSERT_SQL = """INSERT INTO crawl_page_html ( + crawl_run_id, url, html, status, content_type, fetch_method, byte_length, captured_at +) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (crawl_run_id, url) DO UPDATE SET + html = EXCLUDED.html, + status = EXCLUDED.status, + content_type = EXCLUDED.content_type, + fetch_method = EXCLUDED.fetch_method, + byte_length = EXCLUDED.byte_length, + captured_at = EXCLUDED.captured_at""" + + +def _normalize_url(url: str) -> str: + return str(url or "").rstrip("/") + + +def _rows_from_records(records: list[dict[str, Any]], crawl_run_id: int) -> list[tuple]: + rows: list[tuple] = [] + captured_at = _now_iso() + for rec in records: + url = _normalize_url(str(rec.get("url") or "")) + html = rec.get("html") + if not url or not html: + continue + status = str(rec.get("status") or "") if rec.get("status") is not None else None + content_type = str(rec.get("content_type") or "") if rec.get("content_type") is not None else None + fetch_method = str(rec.get("fetch_method") or "static").strip() or "static" + byte_length = int(rec.get("byte_length") or len(str(html).encode("utf-8"))) + rows.append( + (crawl_run_id, url, str(html), status, content_type, fetch_method, byte_length, captured_at) + ) + return rows + + +def write_page_html_batch( + conn: Connection, + records: list[dict[str, Any]], + crawl_run_id: int, + *, + commit: bool = True, +) -> None: + """Upsert HTML rows for a crawl run (each record: url, html, status, content_type, fetch_method, byte_length).""" + rows = _rows_from_records(records, crawl_run_id) + if not rows: + return + _executemany(conn, _HTML_UPSERT_SQL, rows, page_size=_HTML_BATCH_SIZE) + if commit: + conn.commit() + + +def read_page_html(conn: Connection, crawl_run_id: int, url: str) -> Optional[dict[str, Any]]: + """Return stored HTML and metadata for one URL, or None.""" + norm = _normalize_url(url) + if not norm: + return None + try: + cur = conn.execute( + """SELECT url, html, status, content_type, fetch_method, byte_length, captured_at + FROM crawl_page_html + WHERE crawl_run_id = %s AND url = %s""", + (crawl_run_id, norm), + ) + row = cur.fetchone() + if row is None: + return None + return dict(row) + except Exception: + return None + + +def read_page_html_for_run( + conn: Connection, + crawl_run_id: int, + *, + limit: int = 5000, + offset: int = 0, +) -> Iterator[dict[str, Any]]: + """Yield stored HTML rows for a crawl run (paginated).""" + try: + cur = conn.execute( + """SELECT url, html, status, content_type, fetch_method, byte_length, captured_at + FROM crawl_page_html + WHERE crawl_run_id = %s + ORDER BY url + LIMIT %s OFFSET %s""", + (crawl_run_id, max(1, int(limit)), max(0, int(offset))), + ) + for row in cur.fetchall(): + yield dict(row) + except Exception: + return + + +def delete_page_html_for_run(conn: Connection, crawl_run_id: int, *, commit: bool = True) -> None: + """Delete all stored HTML for a crawl run.""" + try: + conn.execute("DELETE FROM crawl_page_html WHERE crawl_run_id = %s", (crawl_run_id,)) + if commit: + conn.commit() + except Exception: + pass diff --git a/src/website_profiling/db/storage.py b/src/website_profiling/db/storage.py index 443b7bf2..0999017a 100644 --- a/src/website_profiling/db/storage.py +++ b/src/website_profiling/db/storage.py @@ -29,9 +29,16 @@ read_nodes, write_crawl, write_crawl_batch, + merge_crawl_result_fields_batch, write_edges, write_nodes, ) +from .html_store import ( + delete_page_html_for_run, + read_page_html, + read_page_html_for_run, + write_page_html_batch, +) from .historical import backup_db_if_exists, ensure_crawl_tables_cleared, read_historical_data, restore_historical_data from .lighthouse_store import ( read_latest_lighthouse_run_json, @@ -59,6 +66,7 @@ "close_db_pool", "create_crawl_run", "create_session", + "delete_page_html_for_run", "delete_session", "db_session", "ensure_crawl_tables_cleared", @@ -70,7 +78,10 @@ "get_latest_crawl_run_id", "init_schema", "list_sessions", + "merge_crawl_result_fields_batch", "read_crawl", + "read_page_html", + "read_page_html_for_run", "read_edges", "read_historical_data", "read_latest_lighthouse_run_json", @@ -90,6 +101,7 @@ "update_session_title", "write_crawl", "write_crawl_batch", + "write_page_html_batch", "write_edges", "write_lh_audits_from_run", "write_lighthouse_page_summary", diff --git a/src/website_profiling/parsing/content.py b/src/website_profiling/parsing/content.py index b61de2f3..553f9f80 100644 --- a/src/website_profiling/parsing/content.py +++ b/src/website_profiling/parsing/content.py @@ -1,37 +1,13 @@ """Content text and social meta parsing.""" from __future__ import annotations -import json - -_STOP_WORDS = frozenset({ - "the", "and", "for", "that", "this", "with", "from", "your", "have", "are", - "was", "were", "been", "will", "would", "could", "should", "about", "which", - "their", "there", "what", "when", "where", "more", "some", "than", "them", - "other", "into", "over", "also", "just", "after", "before", "only", "then", - "very", "most", "each", "such", "like", "does", "here", "because", "being", - "well", "while", "these", "those", "both", "many", "much", "even", "back", - "through", "still", "between", "every", "under", "last", "long", "great", - "make", "same", "come", "take", "know", "they", "page", "site", "home", - "click", "read", "view", "next", "menu", "main", "skip", "content", "link", - "http", "https", "www", "html", "class", "none", "true", "false", "null", -}) +from ..content_analysis.page import analyze_page_html def _count_syllables(word: str) -> int: - word = word.lower().strip() - if len(word) <= 3: - return 1 - vowels = "aeiouy" - count = 0 - prev_vowel = False - for ch in word: - is_vowel = ch in vowels - if is_vowel and not prev_vowel: - count += 1 - prev_vowel = is_vowel - if word.endswith("e") and count > 1: - count -= 1 - return max(1, count) + from ..content_analysis.reading_level import count_syllables + + return count_syllables(word) def parse_content_text(soup, raw_html: str, excerpt_max_chars: int = 0) -> dict: @@ -40,55 +16,8 @@ def parse_content_text(soup, raw_html: str, excerpt_max_chars: int = 0) -> dict: excerpt_max_chars: when > 0, strip script/style from body and store a whitespace-normalized plain-text excerpt (truncated) in ``content_excerpt`` for analysis / AI / UI. """ - import re - from collections import Counter - - body = soup.find("body") - if body: - for tag in body.find_all(["script", "style", "noscript"]): - tag.decompose() - body_text = body.get_text(separator=" ", strip=True) if body else "" - words = [w for w in re.findall(r"[a-zA-Z]+", body_text) if len(w) >= 2] - word_count = len(words) - - sentences = [s.strip() for s in re.split(r"[.!?]+", body_text) if len(s.strip()) > 5] - sentence_count = max(1, len(sentences)) - - total_syllables = sum(_count_syllables(w) for w in words) if words else 0 - - reading_level = 0.0 - if word_count > 30: - reading_level = ( - 0.39 * (word_count / sentence_count) - + 11.8 * (total_syllables / max(1, word_count)) - - 15.59 - ) - reading_level = max(0.0, min(18.0, round(reading_level, 1))) - - html_len = max(1, len(raw_html)) - content_html_ratio = round(len(body_text) / html_len * 100, 1) - - keyword_words = [w.lower() for w in words if len(w) >= 4 and w.lower() not in _STOP_WORDS] - top_keywords = Counter(keyword_words).most_common(10) - max_kw = top_keywords[0][1] if top_keywords else 0 - kw_rows = [] - for w, c in top_keywords: - score = round(100 * c / max_kw) if max_kw else 0 - kw_rows.append({"word": w, "count": c, "score": int(score)}) - - excerpt = "" - if excerpt_max_chars and excerpt_max_chars > 0 and body_text: - excerpt = re.sub(r"\s+", " ", body_text.strip()) - if len(excerpt) > excerpt_max_chars: - excerpt = excerpt[: excerpt_max_chars].rsplit(" ", 1)[0].strip() or excerpt[:excerpt_max_chars] - - return { - "word_count": word_count, - "reading_level": reading_level, - "content_html_ratio": content_html_ratio, - "top_keywords": json.dumps(kw_rows), - "content_excerpt": excerpt, - } + del soup # analyze_page_html loads from raw_html for a single code path + return analyze_page_html(raw_html, excerpt_max_chars=excerpt_max_chars, strategy="full_body") def parse_social_meta(soup) -> dict: diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py index baf28c2c..b5e0fc75 100644 --- a/src/website_profiling/reporting/builder.py +++ b/src/website_profiling/reporting/builder.py @@ -5,6 +5,7 @@ import json import os +from datetime import datetime, timezone from typing import Any, Optional from urllib.parse import urlparse @@ -12,8 +13,9 @@ import requests from ..analysis import merge_bundles, run_local_enrichment +from ..analysis.text_hygiene import is_junk_semantic_term from ..config import get_bool, get_int -from ..llm.enrich import run_llm_enrichment +from ..llm.enrich import cluster_keywords_llm, run_llm_enrichment from ..llm_config import load_llm_config_from_db, llm_is_enabled from ..security_scanner import run_security_scan from .categories import build_categories @@ -45,7 +47,12 @@ _parse_page_analysis_cell, _validate_report_url_counts, ) -from .seo_summary import _compute_summary_seo_issues +from .seo_summary import ( + META_DESC_LEN_MAX, + META_DESC_LEN_MIN, + THIN_CONTENT_CHARS, + _compute_summary_seo_issues, +) from .site_level import _fetch_site_level # Backward-compatible re-exports for tests and external imports. diff --git a/tests/test_cli_dispatch.py b/tests/test_cli_dispatch.py index ebec6d01..bba1d27e 100644 --- a/tests/test_cli_dispatch.py +++ b/tests/test_cli_dispatch.py @@ -17,5 +17,5 @@ def test_cli_help_lists_commands(): timeout=15, ) assert proc.returncode == 0 - for cmd in ("crawl", "report", "plot", "lighthouse", "keywords", "warnings", "enrich", "google", "gsc-links-import"): + for cmd in ("crawl", "content_analysis", "report", "plot", "lighthouse", "keywords", "warnings", "enrich", "google", "gsc-links-import"): assert cmd in proc.stdout diff --git a/tests/test_config_schema_keys.py b/tests/test_config_schema_keys.py index 97c211a4..9395716c 100644 --- a/tests/test_config_schema_keys.py +++ b/tests/test_config_schema_keys.py @@ -18,6 +18,11 @@ "store_outlinks", "store_content_excerpt", "content_excerpt_max_chars", + "store_page_html", + "max_stored_html_bytes", + "run_content_analysis", + "content_analysis_strategy", + "content_analysis_workers", "preserve_crawl_history", "crawl_stream_to_db", "crawl_exclude_urls", diff --git a/tests/test_content_analysis.py b/tests/test_content_analysis.py new file mode 100644 index 00000000..c1fee8a6 --- /dev/null +++ b/tests/test_content_analysis.py @@ -0,0 +1,59 @@ +"""Tests for content_analysis package.""" +from __future__ import annotations + +import json + +from website_profiling.content_analysis.dom_cleanup import cleanup_dom +from website_profiling.content_analysis.html_loader import load_soup +from website_profiling.content_analysis.main_content import find_main_content +from website_profiling.content_analysis.page import analyze_page_html +from website_profiling.content_analysis.tokenize import tokenize_words + + +def test_analyze_page_html_counts_body_words() -> None: + html = "

    Hello world again.

    " + out = analyze_page_html(html, strategy="main_only") + assert out["word_count"] > 0 + assert out["reading_level"] >= 0 + assert json.loads(out["top_keywords"]) + + +def test_main_only_excludes_sidebar_noise() -> None: + html = """ + +
    Extra sidebar noise with many filler words here today.
    +

    Article about widgets and reviews for buyers.

    + + """ + main_only = analyze_page_html(html, strategy="main_only") + full_body = analyze_page_html(html, strategy="full_body") + assert main_only["word_count"] < full_body["word_count"] + + +def test_cleanup_dom_removes_scripts() -> None: + soup = load_soup("

    Visible

    ") + cleaned = cleanup_dom(soup) + assert "Visible" in cleaned.get_text() + assert cleaned.find("script") is None + + +def test_excerpt_truncation() -> None: + words = " ".join(f"word{i}" for i in range(80)) + html = f"

    {words}

    " + out = analyze_page_html(html, excerpt_max_chars=40, strategy="main_only") + assert out["content_excerpt"] + assert len(out["content_excerpt"]) <= 40 + + +def test_find_main_content_prefers_main_tag() -> None: + soup = load_soup("
    noise
    Primary copy here
    ") + root = find_main_content(cleanup_dom(soup), strategy="main_only") + text = root.get_text(strip=True) + assert "Primary" in text + assert "noise" not in text + + +def test_tokenize_words_min_length() -> None: + tokens = tokenize_words("I am ok testing") + assert "I" not in tokens + assert "am" in tokens diff --git a/tests/test_content_analysis_coverage.py b/tests/test_content_analysis_coverage.py new file mode 100644 index 00000000..e61be94b --- /dev/null +++ b/tests/test_content_analysis_coverage.py @@ -0,0 +1,163 @@ +"""Additional coverage for content_analysis batch, pipeline, and related paths.""" +from __future__ import annotations + +import argparse +from unittest.mock import MagicMock + +import pandas as pd +import pytest + +from website_profiling.content_analysis import batch as ca_batch +from website_profiling.content_analysis.excerpt import build_excerpt +from website_profiling.content_analysis.pipeline import run_content_analysis +from website_profiling.content_analysis.dom_cleanup import cleanup_dom +from website_profiling.content_analysis.html_loader import load_soup +from website_profiling.content_analysis.main_content import find_main_content +from website_profiling.crawl.html_capture import should_store_page_html +from website_profiling.db import crawl_store as cs +from website_profiling.parsing.content import _count_syllables, parse_content_text + + +def test_build_excerpt_returns_full_when_under_limit() -> None: + assert build_excerpt("short readable text", 100) == "short readable text" + + +def test_cleanup_dom_removes_aria_hidden() -> None: + soup = load_soup( + '

    Keep

    ' + ) + cleaned = cleanup_dom(soup) + text = cleaned.get_text() + assert "Keep" in text + assert "Hidden" not in text + + +def test_find_main_content_falls_back_to_body() -> None: + soup = load_soup("
    Only body copy here today.
    ") + root = find_main_content(cleanup_dom(soup), strategy="main_only") + assert "Only body copy" in root.get_text() + + +def test_should_store_page_html_rejects_invalid_status() -> None: + assert not should_store_page_html( + enabled=True, + status="not-a-number", + content_type="text/html", + html="", + max_bytes=1000, + ) + + +def test_parse_content_text_delegates_to_analyzer() -> None: + from bs4 import BeautifulSoup + + soup = BeautifulSoup("

    Hello content world.

    ", "lxml") + out = parse_content_text(soup, "

    Hello content world.

    ") + assert out["word_count"] > 0 + + +def test_count_syllables_wrapper() -> None: + assert _count_syllables("hello") >= 1 + + +def test_analyze_row_skips_missing_html() -> None: + assert ca_batch._analyze_row({"url": "https://x.com", "html": ""}, excerpt_max_chars=0, strategy="main_only") is None + + +def test_analyze_run_html_empty() -> None: + assert ca_batch.analyze_run_html(MagicMock(), 1) == [] + + +def test_analyze_run_html_single_worker(monkeypatch: pytest.MonkeyPatch) -> None: + rows = [ + { + "url": "https://example.com", + "html": "
    hello world content here
    ", + } + ] + monkeypatch.setattr(ca_batch, "iter_html_pages", lambda *_a, **_k: iter(rows)) + out = ca_batch.analyze_run_html(MagicMock(), 3, workers=1) + assert len(out) == 1 + assert out[0]["url"] == "https://example.com" + assert out[0]["word_count"] > 0 + + +def test_analyze_run_html_parallel_workers(monkeypatch: pytest.MonkeyPatch) -> None: + rows = [ + {"url": f"https://example.com/{i}", "html": f"
    page {i} content words
    "} + for i in range(2) + ] + monkeypatch.setattr(ca_batch, "iter_html_pages", lambda *_a, **_k: iter(rows)) + out = ca_batch.analyze_run_html(MagicMock(), 3, workers=2) + assert len(out) == 2 + + +def test_iter_html_pages_paginates(monkeypatch: pytest.MonkeyPatch) -> None: + chunks = [ + [{"url": "https://a.com", "html": "a"}] * 500, + [{"url": "https://b.com", "html": "b"}], + ] + calls: list[tuple] = [] + + def _fake_read(_conn, crawl_run_id, *, limit=5000, offset=0): + calls.append((crawl_run_id, limit, offset)) + idx = offset // 500 + return iter(chunks[idx] if idx < len(chunks) else []) + + monkeypatch.setattr(ca_batch, "read_page_html_for_run", _fake_read) + got = list(ca_batch.iter_html_pages(MagicMock(), 9)) + assert len(got) == 501 + assert calls[0] == (9, 500, 0) + assert calls[1] == (9, 500, 500) + + +def test_run_content_analysis_skips_when_no_crawl_run(monkeypatch: pytest.MonkeyPatch) -> None: + class _Ctx: + def __enter__(self): + return object() + + def __exit__(self, *_a): + return False + + monkeypatch.setattr("website_profiling.content_analysis.pipeline.db_session", lambda: _Ctx()) + monkeypatch.setattr( + "website_profiling.content_analysis.pipeline.get_latest_crawl_run_id", + lambda _c: None, + ) + summary = run_content_analysis() + assert summary["pages_analyzed"] == 0 + assert summary["crawl_run_id"] is None + + +def test_merge_crawl_result_fields_batch_skips_invalid_rows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(cs, "_executemany", lambda *_a, **_k: None) + conn = MagicMock() + assert cs.merge_crawl_result_fields_batch(conn, 1, []) == 0 + assert cs.merge_crawl_result_fields_batch(conn, 1, [{"url": ""}]) == 0 + assert cs.merge_crawl_result_fields_batch(conn, 1, [{"url": "https://x.com"}]) == 0 + + +def test_write_crawl_deletes_html_when_table_missing(monkeypatch: pytest.MonkeyPatch) -> None: + executed: list[str] = [] + + class _Tx: + def __enter__(self): + return conn + + def __exit__(self, *_a): + return False + + class _Conn: + def transaction(self): + return _Tx() + + def execute(self, sql, params=None): + executed.append(sql) + if "crawl_page_html" in sql: + raise RuntimeError("missing table") + + conn = _Conn() + monkeypatch.setattr(cs, "_crawl_rows_from_df", lambda df, run_id: []) + monkeypatch.setattr(cs, "_write_crawl_rows", lambda *_a, **_k: None) + cs.write_crawl(conn, pd.DataFrame([{"url": "https://a.com"}]), crawl_run_id=5) # type: ignore[arg-type] + assert any("crawl_page_html" in sql for sql in executed) diff --git a/tests/test_content_analysis_pipeline.py b/tests/test_content_analysis_pipeline.py new file mode 100644 index 00000000..bb328494 --- /dev/null +++ b/tests/test_content_analysis_pipeline.py @@ -0,0 +1,104 @@ +"""Tests for merge_crawl_result_fields_batch and content analysis pipeline.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from website_profiling.db import crawl_store as cs + + +def test_merge_crawl_result_fields_batch(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple] = [] + + def _fake_executemany(conn, sql, params, *, page_size=500): + calls.append((sql, params)) + + monkeypatch.setattr(cs, "_executemany", _fake_executemany) + conn = MagicMock() + n = cs.merge_crawl_result_fields_batch( + conn, + 3, + [{"url": "https://example.com/a", "word_count": 120, "top_keywords": "[]"}], + commit=True, + ) + assert n == 1 + conn.commit.assert_called_once() + assert len(calls) == 1 + sql, params = calls[0] + assert "crawl_results" in sql + assert params[0][1] == 3 + assert params[0][2] == "https://example.com/a" + + +def test_run_content_analysis_skips_without_database(capsys) -> None: + from website_profiling.commands import pipeline_cmd + + pipeline_cmd._run_content_analysis({"store_page_html": True}, False) + assert "database required" in capsys.readouterr().out.lower() + + +def test_run_content_analysis_skips_without_html_storage(capsys) -> None: + from website_profiling.commands import pipeline_cmd + + pipeline_cmd._run_content_analysis({"store_page_html": False}, True) + assert "store_page_html" in capsys.readouterr().out + + +def test_run_content_analysis_runs_when_html_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + from website_profiling.commands import pipeline_cmd + + calls: list[dict] = [] + + monkeypatch.setattr( + "website_profiling.content_analysis.run_content_analysis", + lambda **kwargs: calls.append(kwargs) or {"pages_analyzed": 2}, + ) + + pipeline_cmd._run_content_analysis( + { + "store_page_html": True, + "store_content_excerpt": True, + "content_excerpt_max_chars": "512", + "content_analysis_strategy": "full_body", + "content_analysis_workers": "2", + }, + True, + ) + assert calls[0]["strategy"] == "full_body" + assert calls[0]["excerpt_max_chars"] == 512 + assert calls[0]["workers"] == 2 + + + +def test_run_content_analysis_invokes_batch(monkeypatch: pytest.MonkeyPatch) -> None: + from website_profiling.content_analysis import pipeline as ca_pipeline + + calls: list[dict] = [] + + class _Ctx: + def __enter__(self): + return object() + + def __exit__(self, *_a): + return False + + monkeypatch.setattr(ca_pipeline, "db_session", lambda: _Ctx()) + monkeypatch.setattr(ca_pipeline, "get_latest_crawl_run_id", lambda _c: 9) + monkeypatch.setattr( + ca_pipeline, + "analyze_run_html", + lambda *_a, **_k: [{"url": "https://ex.com", "word_count": 10, "top_keywords": "[]"}], + ) + monkeypatch.setattr( + ca_pipeline, + "merge_crawl_result_fields_batch", + lambda _c, run_id, updates, commit=True: calls.append({"run_id": run_id, "n": len(updates)}), + ) + monkeypatch.setattr(ca_pipeline, "emit_phase_start", MagicMock()) + monkeypatch.setattr(ca_pipeline, "emit_phase_done", MagicMock()) + monkeypatch.setattr(ca_pipeline, "emit_progress", MagicMock()) + + summary = ca_pipeline.run_content_analysis(excerpt_max_chars=0, strategy="main_only", workers=1) + assert summary["pages_analyzed"] == 1 + assert calls[0]["run_id"] == 9 diff --git a/tests/test_crawl_db_writer_imports.py b/tests/test_crawl_db_writer_imports.py index c5b845a8..17c34596 100644 --- a/tests/test_crawl_db_writer_imports.py +++ b/tests/test_crawl_db_writer_imports.py @@ -5,6 +5,7 @@ def test_crawl_db_writer_enqueue_and_batch_flush(monkeypatch: pytest.MonkeyPatch from website_profiling.crawl.crawler import _CrawlDbWriter written: list[tuple[int, int, bool]] = [] + html_written: list[tuple[int, int, bool]] = [] class _FakeConn: pass @@ -25,15 +26,58 @@ def __exit__(self, _t, _v, _tb): "website_profiling.db.crawl_store.write_crawl_batch", lambda _conn, rows, run_id, commit=True: written.append((len(rows), run_id, commit)), ) + monkeypatch.setattr( + "website_profiling.db.html_store.write_page_html_batch", + lambda _conn, rows, run_id, commit=True: html_written.append((len(rows), run_id, commit)), + ) - writer = _CrawlDbWriter(crawl_run_id=5, batch_size=50) + writer = _CrawlDbWriter(crawl_run_id=5, batch_size=50, store_page_html=True) for i in range(51): writer.enqueue({"url": f"https://a.com/{i}"}) + writer.enqueue_html({"url": "https://a.com/html", "html": "", "status": "200"}) writer.finish() writer.run() writer.raise_if_failed() assert written == [(50, 5, True), (1, 5, True)] + assert html_written == [(1, 5, True)] + + +def test_crawl_db_writer_skips_html_when_disabled() -> None: + from website_profiling.crawl.crawler import _CrawlDbWriter + + writer = _CrawlDbWriter(crawl_run_id=1, batch_size=50, store_page_html=False) + writer.enqueue_html({"url": "https://a.com", "html": ""}) + writer.finish() + writer.run() + writer.raise_if_failed() + + +def test_crawl_db_writer_flushes_html_mid_batch(monkeypatch: pytest.MonkeyPatch) -> None: + from website_profiling.crawl.crawler import _CrawlDbWriter + + html_written: list[tuple[int, int, bool]] = [] + + class _FakeCtx: + def __enter__(self): + return object() + + def __exit__(self, _t, _v, _tb): + return False + + monkeypatch.setattr("website_profiling.db.db_session", lambda: _FakeCtx()) + monkeypatch.setattr( + "website_profiling.db.html_store.write_page_html_batch", + lambda _conn, rows, run_id, commit=True: html_written.append((len(rows), run_id, commit)), + ) + + writer = _CrawlDbWriter(crawl_run_id=2, batch_size=50, store_page_html=True) + for i in range(51): + writer.enqueue_html({"url": f"https://a.com/{i}", "html": f"{i}"}) + writer.finish() + writer.run() + writer.raise_if_failed() + assert html_written == [(50, 2, True), (1, 2, True)] def test_crawl_db_writer_records_run_errors(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_crawl_html_storage.py b/tests/test_crawl_html_storage.py new file mode 100644 index 00000000..00404e57 --- /dev/null +++ b/tests/test_crawl_html_storage.py @@ -0,0 +1,126 @@ +"""Tests for HTML capture during crawl.""" +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from website_profiling.crawl.html_capture import build_page_html_record, should_store_page_html + + +def test_should_store_page_html_accepts_200_html() -> None: + assert should_store_page_html( + enabled=True, + status=200, + content_type="text/html; charset=utf-8", + html="ok", + max_bytes=10_000, + ) + + +def test_should_store_page_html_rejects_when_disabled() -> None: + assert not should_store_page_html( + enabled=False, + status=200, + content_type="text/html", + html="", + max_bytes=10_000, + ) + + +def test_should_store_page_html_rejects_404() -> None: + assert not should_store_page_html( + enabled=True, + status=404, + content_type="text/html", + html="", + max_bytes=10_000, + ) + + +def test_should_store_page_html_rejects_non_html_mime() -> None: + assert not should_store_page_html( + enabled=True, + status=200, + content_type="application/json", + html='{"a":1}', + max_bytes=10_000, + ) + + +def test_should_store_page_html_rejects_oversized() -> None: + html = "x" * 100 + assert not should_store_page_html( + enabled=True, + status=200, + content_type="text/html", + html=html, + max_bytes=50, + ) + + +def test_build_page_html_record_normalizes_url() -> None: + rec = build_page_html_record( + url="https://example.com/page/", + html="hi", + status=200, + content_type="text/html", + fetch_method="rendered", + max_bytes=5000, + ) + assert rec is not None + assert rec["url"] == "https://example.com/page" + assert rec["fetch_method"] == "rendered" + assert rec["byte_length"] == len("hi".encode("utf-8")) + + +def test_crawler_capture_uses_buffer_when_no_db_writer(monkeypatch: pytest.MonkeyPatch) -> None: + from website_profiling.crawl.crawler import Crawler + + monkeypatch.setattr( + "website_profiling.crawl.crawler.build_fetcher", + lambda **kwargs: MagicMock(fetch=MagicMock()), + ) + monkeypatch.setattr( + "website_profiling.crawl.frontier.CrawlFrontier.seed_initial_urls", + lambda *args, **kwargs: None, + ) + + crawler = Crawler( + start_url="https://example.com", + max_pages=1, + store_page_html=True, + max_stored_html_bytes=50_000, + ) + crawler._capture_page_html( + "https://example.com/a", + "text", + 200, + "text/html", + "static", + ) + assert len(crawler._html_buffer) == 1 + assert crawler._html_buffer[0]["url"] == "https://example.com/a" + + +def test_crawler_skips_html_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + from website_profiling.crawl.crawler import Crawler + + monkeypatch.setattr( + "website_profiling.crawl.crawler.build_fetcher", + lambda **kwargs: MagicMock(fetch=MagicMock()), + ) + monkeypatch.setattr( + "website_profiling.crawl.frontier.CrawlFrontier.seed_initial_urls", + lambda *args, **kwargs: None, + ) + + crawler = Crawler(start_url="https://example.com", max_pages=1, store_page_html=False) + crawler._capture_page_html( + "https://example.com/a", + "text", + 200, + "text/html", + "static", + ) + assert crawler._html_buffer == [] diff --git a/tests/test_crawler_deep.py b/tests/test_crawler_deep.py index 75c560fa..1f8c58e1 100644 --- a/tests/test_crawler_deep.py +++ b/tests/test_crawler_deep.py @@ -359,9 +359,10 @@ def test_crawl_streams_rows_to_db_writer(monkeypatch): class FakeDbWriter: instances: list["FakeDbWriter"] = [] - def __init__(self, crawl_run_id: int, batch_size: int) -> None: + def __init__(self, crawl_run_id: int, batch_size: int, *, store_page_html: bool = False) -> None: self.crawl_run_id = crawl_run_id self.batch_size = batch_size + self.store_page_html = store_page_html self.enqueued: list[dict] = [] self.started = False self.finished = False @@ -540,3 +541,82 @@ def __exit__(self, _t, _v, _tb): df = mod.run_crawler("https://a.com", output_db=True, crawl_stream_to_db=True, show_progress=False) assert not df.empty + +def test_run_crawler_flushes_buffered_html(monkeypatch): + import website_profiling.crawl.crawler as mod + + class FakeCrawler: + def __init__(self, **_kwargs): + self.link_edges_accum = [] + self.store_page_html = True + self._html_buffer = [ + {"url": "https://a.com", "html": "", "status": "200"}, + ] + + def crawl(self, **_kwargs): + return pd.DataFrame([{"url": "https://a.com", "status": 200}]) + + class _Ctx: + def __enter__(self): + return object() + + def __exit__(self, _t, _v, _tb): + return False + + html_writes: list[tuple] = [] + + fake_db = types.SimpleNamespace( + backup_db_if_exists=lambda: None, + create_crawl_run=lambda *_a, **_k: 7, + db_session=lambda: _Ctx(), + read_historical_data=lambda: {}, + restore_historical_data=lambda *_a, **_k: None, + write_crawl=lambda *_a, **_k: None, + ) + fake_storage = types.SimpleNamespace(ensure_crawl_tables_cleared=lambda *_a, **_k: None) + monkeypatch.setattr(mod, "Crawler", FakeCrawler) + monkeypatch.setitem(__import__("sys").modules, "website_profiling.db", fake_db) + monkeypatch.setitem(__import__("sys").modules, "website_profiling.db.storage", fake_storage) + monkeypatch.setattr( + "website_profiling.db.html_store.write_page_html_batch", + lambda _conn, rows, run_id, commit=True: html_writes.append((len(rows), run_id)), + ) + + mod.run_crawler( + "https://a.com", + output_db=True, + preserve_crawl_history=False, + show_progress=False, + store_page_html=True, + ) + assert html_writes == [(1, 7)] + + +def test_capture_page_html_enqueues_to_stream_writer(): + import website_profiling.crawl.crawler as mod + + class _Writer: + def __init__(self): + self.records: list[dict] = [] + + def enqueue_html(self, record: dict) -> None: + self.records.append(record) + + crawler = mod.Crawler( + start_url="https://site.com", + ignore_robots=True, + store_page_html=True, + max_pages=1, + ) + writer = _Writer() + crawler._db_writer = writer + crawler._capture_page_html( + "https://site.com", + "Hello", + 200, + "text/html", + "static", + ) + assert len(writer.records) == 1 + assert writer.records[0]["url"] == "https://site.com" + diff --git a/tests/test_historical_keywords_crawl_store_unit.py b/tests/test_historical_keywords_crawl_store_unit.py index 48b7fb44..c9e86eb3 100644 --- a/tests/test_historical_keywords_crawl_store_unit.py +++ b/tests/test_historical_keywords_crawl_store_unit.py @@ -191,7 +191,7 @@ def test_ensure_crawl_tables_cleared_commits(): conn = _Conn() ensure_crawl_tables_cleared(conn) # type: ignore[arg-type] assert conn.commits == 1 - assert any("TRUNCATE crawl_results, edges, nodes" in s for s, _ in conn.executed) + assert any("TRUNCATE crawl_results, crawl_page_html, edges, nodes" in s for s, _ in conn.executed) def test_read_lh_runs_by_url_and_page_summaries(): diff --git a/tests/test_html_store.py b/tests/test_html_store.py new file mode 100644 index 00000000..5772abdc --- /dev/null +++ b/tests/test_html_store.py @@ -0,0 +1,151 @@ +"""Tests for crawl_page_html storage.""" +from __future__ import annotations + +import pytest + +from website_profiling.db import html_store as hs + + +class _Cursor: + def __init__(self, rows=None): + self._rows = rows or [] + self.executemany_calls: list[tuple] = [] + + def executemany(self, sql, params): + self.executemany_calls.append((sql, params)) + + def fetchall(self): + return list(self._rows) + + def fetchone(self): + return self._rows[0] if self._rows else None + + +class _Conn: + def __init__(self, select_rows=None): + self.select_rows = select_rows or [] + self.executed: list[tuple] = [] + self.commits = 0 + self.executemany_calls: list[tuple] = [] + self._cursor = _Cursor() + + def execute(self, sql, params=None): + self.executed.append((sql, params)) + if "SELECT" in sql.upper(): + self._cursor = _Cursor(self.select_rows) + return self._cursor + + def commit(self): + self.commits += 1 + + def cursor(self): + cur = _Cursor() + cur.executemany_calls = self.executemany_calls + + class _CM: + def __enter__(self_inner): + return cur + + def __exit__(self_inner, _t, _v, _tb): + return False + + return _CM() + + +def test_write_page_html_batch_builds_rows_and_commits() -> None: + conn = _Conn() + records = [ + { + "url": "https://example.com/page", + "html": "Hello", + "status": "200", + "content_type": "text/html", + "fetch_method": "rendered", + "byte_length": 32, + } + ] + hs.write_page_html_batch(conn, records, crawl_run_id=7, commit=True) + assert conn.commits == 1 + assert len(conn.executemany_calls) == 1 + _sql, params = conn.executemany_calls[0] + assert "crawl_page_html" in _sql + assert params[0][0] == 7 + assert params[0][1] == "https://example.com/page" + + +def test_write_page_html_batch_skips_empty_html() -> None: + conn = _Conn() + hs.write_page_html_batch(conn, [{"url": "https://x.com", "html": ""}], crawl_run_id=1) + assert conn.commits == 0 + assert not conn.executed + + +def test_read_page_html_returns_row() -> None: + row = { + "url": "https://example.com", + "html": "", + "status": "200", + "content_type": "text/html", + "fetch_method": "static", + "byte_length": 13, + "captured_at": "2026-01-01", + } + conn = _Conn(select_rows=[row]) + out = hs.read_page_html(conn, 3, "https://example.com/") + assert out is not None + assert out["html"] == "" + + +def test_read_page_html_for_run_yields_rows() -> None: + rows = [ + {"url": "https://a.com", "html": "a", "status": "200", + "content_type": "text/html", "fetch_method": "static", "byte_length": 1, "captured_at": "t"}, + ] + conn = _Conn(select_rows=rows) + got = list(hs.read_page_html_for_run(conn, 1, limit=10)) + assert len(got) == 1 + assert got[0]["url"] == "https://a.com" + + +def test_delete_page_html_for_run() -> None: + conn = _Conn() + hs.delete_page_html_for_run(conn, 9, commit=True) + assert conn.commits == 1 + assert any("DELETE FROM crawl_page_html" in sql for sql, _ in conn.executed) + + +def test_read_page_html_empty_url_returns_none() -> None: + conn = _Conn() + assert hs.read_page_html(conn, 1, "") is None + + +def test_read_page_html_returns_none_when_missing() -> None: + conn = _Conn(select_rows=[]) + assert hs.read_page_html(conn, 3, "https://missing.example") is None + + +def test_read_page_html_handles_db_error() -> None: + class _BadConn: + def execute(self, *_a, **_k): + raise RuntimeError("db down") + + assert hs.read_page_html(_BadConn(), 1, "https://example.com") is None # type: ignore[arg-type] + + +def test_read_page_html_for_run_handles_db_error() -> None: + class _BadConn: + def execute(self, *_a, **_k): + raise RuntimeError("db down") + + assert list(hs.read_page_html_for_run(_BadConn(), 1)) == [] # type: ignore[arg-type] + + +def test_delete_page_html_for_run_handles_db_error() -> None: + class _BadConn: + def execute(self, *_a, **_k): + raise RuntimeError("db down") + + def commit(self): + raise RuntimeError("should not commit") + + hs.delete_page_html_for_run(_BadConn(), 1, commit=True) # type: ignore[arg-type] diff --git a/tests/test_pipeline_cmd_run_unit.py b/tests/test_pipeline_cmd_run_unit.py index 872dae05..08b57a75 100644 --- a/tests/test_pipeline_cmd_run_unit.py +++ b/tests/test_pipeline_cmd_run_unit.py @@ -352,3 +352,49 @@ def test_lighthouse_main_prints_unicode_summary_on_cp1252(monkeypatch, tmp_path) output = buffer.getvalue().decode("utf-8", errors="replace") assert "2500" in output + +def test_pipeline_run_content_analysis_command(monkeypatch) -> None: + from website_profiling.commands import pipeline_cmd + + called = {"content": 0} + + monkeypatch.setattr(pipeline_cmd, "_run_crawl", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_report", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_plot", lambda *_a, **_k: None) + monkeypatch.setattr( + pipeline_cmd, + "_run_content_analysis", + lambda *_a, **_k: called.__setitem__("content", called["content"] + 1), + ) + + cfg = { + "start_url": "https://site.com", + "run_crawl": "false", + "run_content_analysis": "true", + "store_page_html": "true", + "run_report": "false", + "run_plot": "false", + } + pipeline_cmd.run(cfg, argparse.Namespace(command="content_analysis")) + assert called["content"] == 1 + + +def test_pipeline_lists_content_analysis_in_steps(monkeypatch, capsys) -> None: + from website_profiling.commands import pipeline_cmd + + monkeypatch.setattr(pipeline_cmd, "_run_crawl", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_content_analysis", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_report", lambda *_a, **_k: None) + monkeypatch.setattr(pipeline_cmd, "_run_plot", lambda *_a, **_k: None) + + cfg = { + "start_url": "https://site.com", + "run_crawl": "false", + "run_content_analysis": "true", + "store_page_html": "true", + "run_report": "false", + "run_plot": "false", + } + pipeline_cmd.run(cfg, argparse.Namespace(command=None)) + assert "content-analysis" in capsys.readouterr().out + diff --git a/tests/test_reporting_builder_modules.py b/tests/test_reporting_builder_modules.py index 209bbd52..97206895 100644 --- a/tests/test_reporting_builder_modules.py +++ b/tests/test_reporting_builder_modules.py @@ -837,3 +837,13 @@ def headers(self): 5, 0.0, ) == [] + + +def test_builder_exposes_llm_keyword_cluster_imports() -> None: + """Regression: LLM keyword cluster branch must not NameError after builder split.""" + import website_profiling.reporting.builder as builder_mod + from website_profiling.analysis.text_hygiene import is_junk_semantic_term + from website_profiling.llm.enrich import cluster_keywords_llm + + assert builder_mod.is_junk_semantic_term is is_junk_semantic_term + assert builder_mod.cluster_keywords_llm is cluster_keywords_llm diff --git a/web/app/api/crawl/page-html/route.ts b/web/app/api/crawl/page-html/route.ts new file mode 100644 index 00000000..71f34003 --- /dev/null +++ b/web/app/api/crawl/page-html/route.ts @@ -0,0 +1,75 @@ +import { NextResponse, type NextRequest } from 'next/server'; +import { forbiddenIfNotLocal } from '@/server/localOnly'; +import { withReportDb } from '@/server/reportDb'; +import { deletePageHtmlForRun, listCrawlPageHtmlRuns } from '@/lib/loadReportDb'; +import type { ApiRouteHandler } from '@/types/api'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +type DeleteBody = { + crawlRunId?: number | null; +}; + +/** + * GET /api/crawl/page-html?limit=30 + * Lists recent crawl runs with stored HTML stats. + */ +export const GET: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + + const limitRaw = Number(request.nextUrl.searchParams.get('limit') || '30'); + const limit = Number.isFinite(limitRaw) ? Math.min(100, Math.max(1, limitRaw)) : 30; + + try { + const runs = await withReportDb((client) => listCrawlPageHtmlRuns(client, { limit })); + return NextResponse.json({ runs }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg, runs: [] }, { status: 500 }); + } +}; + +/** + * DELETE /api/crawl/page-html + * Body: { crawlRunId: number } + * Removes raw HTML for one crawl run; crawl results and reports are kept. + */ +export const DELETE: ApiRouteHandler = async (request: NextRequest): Promise => { + const denied = forbiddenIfNotLocal(request); + if (denied) return denied; + + let body: DeleteBody = {}; + try { + body = (await request.json()) as DeleteBody; + } catch { + const crawlRunIdRaw = request.nextUrl.searchParams.get('crawlRunId'); + if (crawlRunIdRaw) body.crawlRunId = Number(crawlRunIdRaw); + } + + const crawlRunId = + body.crawlRunId != null && Number.isFinite(Number(body.crawlRunId)) + ? Number(body.crawlRunId) + : null; + + if (crawlRunId == null) { + return NextResponse.json({ error: 'crawlRunId is required' }, { status: 400 }); + } + + try { + const deletedPages = await withReportDb((client) => deletePageHtmlForRun(client, crawlRunId)); + if (deletedPages === 0) { + return NextResponse.json({ + ok: true, + crawlRunId, + deletedPages: 0, + message: 'No stored HTML found for this crawl run.', + }); + } + return NextResponse.json({ ok: true, crawlRunId, deletedPages }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: msg }, { status: 500 }); + } +}; diff --git a/web/src/components/pipeline/CrawlPageHtmlManager.tsx b/web/src/components/pipeline/CrawlPageHtmlManager.tsx new file mode 100644 index 00000000..a89f74f4 --- /dev/null +++ b/web/src/components/pipeline/CrawlPageHtmlManager.tsx @@ -0,0 +1,207 @@ +'use client'; + +import { useCallback, useEffect, useState } from 'react'; +import { Loader2, Trash2 } from 'lucide-react'; +import { apiUrl } from '@/lib/publicBase'; +import { strings, format } from '@/lib/strings'; +import { formatReportGeneratedAt } from '@/lib/reportTimestamps'; +import type { CrawlPageHtmlRunRow } from '@/types/report'; + +const sh = strings.pipelineRunner.storedHtml; + +function formatBytes(bytes: number): string { + if (!Number.isFinite(bytes) || bytes <= 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB'] as const; + let n = bytes; + let i = 0; + while (n >= 1024 && i < units.length - 1) { + n /= 1024; + i += 1; + } + const digits = i === 0 ? 0 : n >= 100 ? 0 : 1; + return `${n.toFixed(digits)} ${units[i]}`; +} + +export interface CrawlPageHtmlManagerProps { + disabled?: boolean; +} + +export default function CrawlPageHtmlManager({ disabled = false }: CrawlPageHtmlManagerProps) { + const [runs, setRuns] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [confirmRunId, setConfirmRunId] = useState(null); + const [deletingRunId, setDeletingRunId] = useState(null); + + const loadRuns = useCallback(async () => { + setLoading(true); + setError(null); + try { + const res = await fetch(apiUrl('/crawl/page-html?limit=30')); + const body = (await res.json()) as { runs?: CrawlPageHtmlRunRow[]; error?: string }; + if (!res.ok) { + setError(body.error || sh.loadFailed); + setRuns([]); + return; + } + setRuns(Array.isArray(body.runs) ? body.runs : []); + } catch { + setError(sh.loadFailed); + setRuns([]); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + void loadRuns(); + }, [loadRuns]); + + const handleDelete = async (crawlRunId: number) => { + setDeletingRunId(crawlRunId); + setError(null); + try { + const res = await fetch(apiUrl('/crawl/page-html'), { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ crawlRunId }), + }); + const body = (await res.json().catch(() => ({}))) as { error?: string }; + if (!res.ok) { + setError(body.error || sh.deleteFailed); + return; + } + setConfirmRunId(null); + setRuns((prev) => + prev.map((row) => + row.crawl_run_id === crawlRunId ? { ...row, page_count: 0, total_bytes: 0 } : row, + ), + ); + } catch { + setError(sh.deleteFailed); + } finally { + setDeletingRunId(null); + } + }; + + const storedRuns = runs.filter((r) => r.page_count > 0); + + return ( +
    +
    +

    {sh.title}

    +

    {sh.hint}

    +
    + + {error ? ( +

    + {error} +

    + ) : null} + + {loading ? ( +
    + + {sh.loading} +
    + ) : storedRuns.length === 0 ? ( +

    {sh.empty}

    + ) : ( +
    + + + + + + + + + + + + {storedRuns.map((row) => { + const isConfirm = confirmRunId === row.crawl_run_id; + const isDeleting = deletingRunId === row.crawl_run_id; + const createdLabel = row.created_at + ? formatReportGeneratedAt(row.created_at) + : format(sh.runIdLabel, { id: row.crawl_run_id }); + return ( + + + + + + + + ); + })} + +
    {sh.colRun}{sh.colSite}{sh.colPages}{sh.colSize}{sh.colAction}
    +
    #{row.crawl_run_id}
    +
    {createdLabel}
    + {row.render_mode ? ( +
    {row.render_mode}
    + ) : null} +     		+ + {row.start_url} + + + {row.page_count.toLocaleString()} + + {formatBytes(row.total_bytes)} + + {isConfirm ? ( +
    +

    + {format(sh.deleteConfirm, { id: row.crawl_run_id })} +

    +
    + + +
    +
    + ) : ( + + )} +
    +
    + )} + + {!loading && storedRuns.length > 0 ? ( + + ) : null} +
    + ); +} diff --git a/web/src/components/pipeline/PipelineSettingsPanel.tsx b/web/src/components/pipeline/PipelineSettingsPanel.tsx index d126fdc4..7676d9a1 100644 --- a/web/src/components/pipeline/PipelineSettingsPanel.tsx +++ b/web/src/components/pipeline/PipelineSettingsPanel.tsx @@ -13,6 +13,7 @@ import { useReadOnlySession } from '@/hooks/useReadOnlySession'; import Button from '@/components/Button'; import GoogleIntegrationsPanel from '@/components/GoogleIntegrationsPanel'; import ConfigField from './ConfigField'; +import CrawlPageHtmlManager from './CrawlPageHtmlManager'; import PipelineSettingsSectionTabs from './PipelineSettingsSectionTabs'; import { PIPELINE_SETTINGS_GROUPS, @@ -334,12 +335,15 @@ export default function PipelineSettingsPanel({ const pipelineSection = PIPELINE_CONFIG_SECTIONS.find((sec) => sec.id === sectionId); if (pipelineSection) { return ( - handlePipelineFieldChange(sectionId, key, value)} - /> + <> + handlePipelineFieldChange(sectionId, key, value)} + /> + {sectionId === 'crawl' ? : null} + ); } diff --git a/web/src/lib/formatPipelineLog.ts b/web/src/lib/formatPipelineLog.ts index bb2a1c83..72c24964 100644 --- a/web/src/lib/formatPipelineLog.ts +++ b/web/src/lib/formatPipelineLog.ts @@ -13,6 +13,7 @@ export type PipelineLogLineKind = export type PipelinePhase = | 'config' | 'crawl' + | 'content_analysis' | 'lighthouse' | 'report' | 'keywords' @@ -101,6 +102,7 @@ const ANSI_RE = /\x1b\[[0-9;]*m/g; export const PHASE_LABELS: Record = { config: 'Settings', crawl: 'Crawl', + content_analysis: 'Content analysis', lighthouse: 'Lighthouse', report: 'Site audit', keywords: 'Keywords', @@ -133,6 +135,7 @@ const STEP_LABELS: Record = { export const PIPELINE_STEPPER_PHASES: ProgressPhase[] = [ 'config', 'crawl', + 'content_analysis', 'lighthouse', 'report', 'keywords', @@ -148,6 +151,7 @@ export function phaseFromSectionTag(tag: string): PipelinePhase { const t = tag.toLowerCase(); if (t.includes('config')) return 'config'; if (t.includes('crawl')) return 'crawl'; + if (t.includes('content')) return 'content_analysis'; if (t.includes('lighthouse')) return 'lighthouse'; if (t.includes('report')) return 'report'; if (t.includes('plot')) return 'plot'; @@ -300,7 +304,7 @@ export function resolveActiveProgress( function phaseFromProgressEvent(phase: string): PipelinePhase { const p = phase.toLowerCase(); - if (p === 'config' || p === 'crawl' || p === 'lighthouse' || p === 'report' || p === 'keywords' || p === 'plot') { + if (p === 'config' || p === 'crawl' || p === 'content_analysis' || p === 'lighthouse' || p === 'report' || p === 'keywords' || p === 'plot') { return p; } if (p === 'optional') return 'optional'; @@ -561,6 +565,7 @@ export const PIPELINE_LOG_LINE_CLASS: Record = { export const PHASE_CHIP_CLASS: Record = { config: 'border-violet-500/40 bg-violet-500/10 text-violet-200', crawl: 'border-blue-500/40 bg-blue-500/10 text-blue-200', + content_analysis: 'border-teal-500/40 bg-teal-500/10 text-teal-200', lighthouse: 'border-amber-500/40 bg-amber-500/10 text-amber-200', report: 'border-green-500/40 bg-green-500/10 text-green-200', keywords: 'border-pink-500/40 bg-pink-500/10 text-pink-200', diff --git a/web/src/lib/loadReportDb.ts b/web/src/lib/loadReportDb.ts index 83dda6a9..0a67ec98 100644 --- a/web/src/lib/loadReportDb.ts +++ b/web/src/lib/loadReportDb.ts @@ -1,5 +1,6 @@ import type { PoolClient } from 'pg'; import type { + CrawlPageHtmlRunRow, CrawlRunRow, CrawlRunSummary, ReportListRow, @@ -39,6 +40,69 @@ export async function getCrawlRunsRows(client: PoolClient): Promise> { + const stats = new Map(); + if (!crawlRunIds.length) return stats; + try { + const { rows } = await client.query<{ + crawl_run_id: string | number; + page_count: string | number; + total_bytes: string | number; + }>( + `SELECT crawl_run_id, + COUNT(*)::int AS page_count, + COALESCE(SUM(byte_length), 0)::bigint AS total_bytes + FROM crawl_page_html + WHERE crawl_run_id = ANY($1::bigint[]) + GROUP BY crawl_run_id`, + [crawlRunIds], + ); + for (const row of rows) { + stats.set(Number(row.crawl_run_id), { + page_count: Number(row.page_count) || 0, + total_bytes: Number(row.total_bytes) || 0, + }); + } + } catch { + /* crawl_page_html may be missing */ + } + return stats; +} + +export async function listCrawlPageHtmlRuns( + client: PoolClient, + options: { limit?: number } = {}, +): Promise { + const limit = options.limit ?? 30; + const runs = await getCrawlRunsRows(client); + const slice = runs.slice(0, Math.max(1, limit)); + const ids = slice.map((r) => r.id); + const stats = await getPageHtmlStatsByRunIds(client, ids); + return slice.map((run) => { + const s = stats.get(run.id); + return { + crawl_run_id: run.id, + start_url: run.start_url, + created_at: run.created_at, + render_mode: run.render_mode, + page_count: s?.page_count ?? 0, + total_bytes: s?.total_bytes ?? 0, + }; + }); +} + +export async function deletePageHtmlForRun(client: PoolClient, crawlRunId: number): Promise { + try { + const res = await client.query('DELETE FROM crawl_page_html WHERE crawl_run_id = $1', [crawlRunId]); + return res.rowCount ?? 0; + } catch { + return 0; + } +} + export async function getCrawlRunSummaries(client: PoolClient): Promise { try { const { rows } = await client.query( diff --git a/web/src/lib/pipelineConfigSchema.ts b/web/src/lib/pipelineConfigSchema.ts index ca1f0144..1613012f 100644 --- a/web/src/lib/pipelineConfigSchema.ts +++ b/web/src/lib/pipelineConfigSchema.ts @@ -147,6 +147,45 @@ export const PIPELINE_CONFIG_SECTIONS: PipelineConfigSection[] = [ { key: 'store_outlinks', label: 'Store external links', type: 'bool', defaultValue: true }, { key: 'store_content_excerpt', label: 'Store page text excerpt', type: 'bool', defaultValue: true }, { key: 'content_excerpt_max_chars', label: 'Excerpt max chars', type: 'number', defaultValue: '4096' }, + { + key: 'store_page_html', + label: 'Store raw page HTML', + type: 'bool', + defaultValue: false, + help: 'Persist fetched HTML per URL in the database for later content analysis. Increases DB size; does not affect CSV exports.', + }, + { + key: 'max_stored_html_bytes', + label: 'Max stored HTML per page (bytes)', + type: 'number', + defaultValue: '2097152', + help: 'Skip pages whose HTML exceeds this size (default 2 MB). Only applies when store_page_html is enabled.', + }, + { + key: 'run_content_analysis', + label: 'Run content analysis step', + type: 'bool', + defaultValue: false, + help: 'After crawl, analyze stored HTML and update word counts/keywords in crawl results. Requires store_page_html.', + }, + { + key: 'content_analysis_strategy', + label: 'Content analysis strategy', + type: 'select', + defaultValue: 'main_only', + options: [ + { value: 'main_only', label: 'Main content only (main/article)' }, + { value: 'full_body', label: 'Full body text' }, + ], + help: 'How to select page text when analyzing stored HTML.', + }, + { + key: 'content_analysis_workers', + label: 'Content analysis workers', + type: 'number', + defaultValue: '4', + help: 'Parallel workers for the post-crawl content analysis step.', + }, { key: 'custom_extraction_regex', label: 'Custom extraction regex', diff --git a/web/src/lib/pipelineLiveEstimate.ts b/web/src/lib/pipelineLiveEstimate.ts index 1a2dc2fe..cfb0ad82 100644 --- a/web/src/lib/pipelineLiveEstimate.ts +++ b/web/src/lib/pipelineLiveEstimate.ts @@ -24,6 +24,7 @@ export interface LivePipelineEstimate { const PIPELINE_PHASE_ORDER: ProgressPhase[] = [ 'config', 'crawl', + 'content_analysis', 'lighthouse', 'report', 'keywords', diff --git a/web/src/server/crawlPageHtmlRoute.test.ts b/web/src/server/crawlPageHtmlRoute.test.ts new file mode 100644 index 00000000..693397fe --- /dev/null +++ b/web/src/server/crawlPageHtmlRoute.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; +import { NextRequest } from 'next/server'; +import { localRequest } from '@/server/testHelpers/routeTestUtils'; + +const withReportDbMock = vi.fn(); + +vi.mock('@/server/reportDb', () => ({ + withReportDb: (fn: (client: unknown) => Promise) => withReportDbMock(fn), +})); + +describe('/api/crawl/page-html', () => { + beforeEach(() => { + withReportDbMock.mockReset(); + vi.resetModules(); + }); + + it('GET returns runs with stats', async () => { + withReportDbMock.mockImplementation(async (fn) => + fn({ + query: vi.fn(), + }), + ); + const loadMod = await import('@/lib/loadReportDb'); + vi.spyOn(loadMod, 'listCrawlPageHtmlRuns').mockResolvedValue([ + { + crawl_run_id: 3, + start_url: 'https://example.com', + created_at: '2026-06-01', + page_count: 12, + total_bytes: 4096, + }, + ]); + + const { GET } = await import('../../app/api/crawl/page-html/route'); + const res = await GET(localRequest('/api/crawl/page-html')); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.runs).toHaveLength(1); + expect(body.runs[0].crawl_run_id).toBe(3); + expect(body.runs[0].page_count).toBe(12); + }); + + it('DELETE removes HTML for a crawl run', async () => { + withReportDbMock.mockImplementation(async (fn) => fn({})); + const loadMod = await import('@/lib/loadReportDb'); + vi.spyOn(loadMod, 'deletePageHtmlForRun').mockResolvedValue(5); + + const { DELETE } = await import('../../app/api/crawl/page-html/route'); + const res = await DELETE( + localRequest('/api/crawl/page-html', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ crawlRunId: 7 }), + }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.ok).toBe(true); + expect(body.deletedPages).toBe(5); + expect(body.crawlRunId).toBe(7); + }); + + it('DELETE requires crawlRunId', async () => { + const { DELETE } = await import('../../app/api/crawl/page-html/route'); + const res = await DELETE( + localRequest('/api/crawl/page-html', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }), + ); + expect(res.status).toBe(400); + }); + + it('rejects non-local hosts', async () => { + const { GET } = await import('../../app/api/crawl/page-html/route'); + const res = await GET(new NextRequest('http://192.168.1.5:3000/api/crawl/page-html')); + expect(res.status).toBe(403); + }); +}); diff --git a/web/src/strings.json b/web/src/strings.json index ddfc2687..0bd0f082 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -216,6 +216,27 @@ "browserCrawlBannerTitle": "Headless browser not available", "browserCrawlBannerHint": "JavaScript and Auto crawl modes need Playwright Python packages and Chromium. Run: pip install -r requirements.txt. Ensure Chrome or Chromium is on PATH or set CHROME_PATH.", "browserCrawlChecking": "Checking browser availability…", + "storedHtml": { + "title": "Stored raw HTML", + "hint": "Delete persisted page HTML for a specific crawl run to free database space. Crawl results, word counts, and audit reports are kept.", + "loading": "Loading stored HTML…", + "empty": "No stored HTML in recent crawl runs.", + "loadFailed": "Could not load stored HTML stats.", + "deleteFailed": "Could not delete stored HTML.", + "colRun": "Crawl run", + "colSite": "Start URL", + "colPages": "Pages", + "colSize": "Size", + "colAction": "Action", + "runIdLabel": "Run #{id}", + "delete": "Delete", + "deleteTitle": "Delete stored HTML for crawl run #{id}", + "deleteConfirm": "Delete all stored HTML for crawl run #{id}? Crawl metrics and reports stay intact.", + "confirmDelete": "Delete HTML", + "cancel": "Cancel", + "deleting": "Deleting…", + "refresh": "Refresh list" + }, "saveSettings": "Save settings", "saveAndClose": "Save & close", "saving": "Saving…", diff --git a/web/src/types/report.ts b/web/src/types/report.ts index ad940bcf..16352d9f 100644 --- a/web/src/types/report.ts +++ b/web/src/types/report.ts @@ -763,6 +763,16 @@ export interface CrawlRunSummary { discovery_mode?: string; } +/** Stored raw HTML footprint for one crawl run (crawl_page_html). */ +export interface CrawlPageHtmlRunRow { + crawl_run_id: number; + start_url: string; + created_at: string; + render_mode?: string; + page_count: number; + total_bytes: number; +} + export interface PortfolioCrawlConfig { pages_crawled?: number; max_pages_configured?: number; From 5275484a82ecc52ab2a13088ef562ec32fa2f086 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 12:54:45 +0530 Subject: [PATCH 4/5] better tools --- AGENT.md | 2 +- README.md | 6 +- docs/MCP.md | 55 +++- .../integrations/google/store.py | 26 ++ src/website_profiling/llm/agent.py | 97 ++++-- src/website_profiling/mcp/core_server.py | 8 + src/website_profiling/mcp/domain_server.py | 11 + src/website_profiling/mcp/server.py | 143 ++++---- .../tools/audit_tools/context.py | 10 +- .../tools/audit_tools/data_coverage.py | 94 ++++++ .../tools/audit_tools/google.py | 57 +++- .../tools/audit_tools/insight_helpers.py | 207 ++++++++++++ .../tools/audit_tools/insight_tools.py | 166 ++++++++++ .../tools/audit_tools/keywords.py | 30 ++ .../tools/audit_tools/registry.py | 147 ++++++++- .../tools/audit_tools/router_tools.py | 147 +++++++++ .../tools/audit_tools/tool_catalog.py | 23 ++ .../tools/audit_tools/tool_domains.py | 308 ++++++++++++++++++ .../tools/audit_tools/tool_selector.py | 147 +++++++++ tests/test_audit_tools_expanded.py | 2 +- tests/test_google_store_full.py | 22 ++ tests/test_mcp_registry.py | 26 +- tests/test_mcp_server_helpers.py | 95 +++++- tests/test_router_tools.py | 31 ++ tests/test_tool_selector.py | 101 ++++++ 25 files changed, 1822 insertions(+), 139 deletions(-) create mode 100644 src/website_profiling/mcp/core_server.py create mode 100644 src/website_profiling/mcp/domain_server.py create mode 100644 src/website_profiling/tools/audit_tools/data_coverage.py create mode 100644 src/website_profiling/tools/audit_tools/insight_helpers.py create mode 100644 src/website_profiling/tools/audit_tools/insight_tools.py create mode 100644 src/website_profiling/tools/audit_tools/router_tools.py create mode 100644 src/website_profiling/tools/audit_tools/tool_domains.py create mode 100644 src/website_profiling/tools/audit_tools/tool_selector.py create mode 100644 tests/test_google_store_full.py create mode 100644 tests/test_router_tools.py create mode 100644 tests/test_tool_selector.py diff --git a/AGENT.md b/AGENT.md index e59a6fa9..cdb903aa 100644 --- a/AGENT.md +++ b/AGENT.md @@ -25,7 +25,7 @@ - **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline. - **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch. - **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/chat` POST (SSE agent); `/api/chat/sessions` GET/POST; `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run -- **MCP:** `python -m website_profiling.mcp` (stdio, **221 read-only audit tools** + MCP resources). See `docs/MCP.md`. Requires `pip install -r requirements.txt`. +- **MCP:** `python -m website_profiling.mcp` (stdio, **240 read-only audit tools**, domain-scoped via `WP_MCP_DOMAIN`). See `docs/MCP.md`. Requires `pip install -r requirements.txt`. - **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages` tables, migration `012_chat_sessions`). - **Job store:** in-memory on `globalThis` in `web/src/server/pipelineJobs.ts` — job status/log is lost on server restart (single-process dev/Docker only). - **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`** diff --git a/README.md b/README.md index aa7dea34..efa69b90 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Repository: [codefrydev/WebsiteProfiling](https://github.com/codefrydev/WebsiteP -Also included: **AI chat** over audit data (optional), **221 MCP tools**, keyword explorer, backlinks, compare runs, and portfolio management for agencies. +Also included: **AI chat** over audit data (optional), **240 MCP tools** (domain-scoped servers), keyword explorer, backlinks, compare runs, and portfolio management for agencies.

    Site Audit preview @@ -91,7 +91,7 @@ WebsiteProfiling/ │ ├── integrations/ # Google Search Console, GA4, Bing, CrUX │ ├── llm/ # AI enrich + chat agent │ ├── tools/ # Exports, audit query tools, MCP helpers -│ ├── mcp/ # MCP server (221 read-only tools) +│ ├── mcp/ # MCP server (240 read-only tools, domain bundles) │ ├── db/ # PostgreSQL storage layer │ ├── commands/ # CLI subcommands │ ├── cli.py # Pipeline entrypoint @@ -186,7 +186,7 @@ Google Search Console / Analytics: connect via **Integrations** (gear icon) in t | **Ollama** | Local daemon at `http://127.0.0.1:11434`. Chat UI lists installed models plus the live Ollama cloud catalog (billing: free local, account free tier, Pro). Native tool calling when supported; otherwise ReAct fallback. Pick the model in-chat without leaving the page. | | **OpenAI** / **Anthropic** | API key in AI settings; native tool calling with streaming. | -The agent uses the same **221 read-only audit tools** as the MCP server (`docs/MCP.md`). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`). +The agent uses the same **240 read-only audit tools** as the MCP server (`docs/MCP.md`), with **dynamic routing** (~45 tools per turn plus router meta-tools). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`). Production: `docker-compose.prod.yml` (set `POSTGRES_PASSWORD`, `AUTH_SECRET`). diff --git a/docs/MCP.md b/docs/MCP.md index 2458bfa0..1ad0edc1 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -12,8 +12,47 @@ export PYTHONPATH=src ## Cursor configuration +Site Audit exposes **domain-scoped MCP servers** (like Cursor optional plugins). Connect only the bundles you need instead of loading all 240 tools in one server. + +| `WP_MCP_DOMAIN` | Typical tools | Use when | +|-----------------|---------------|----------| +| `core` (default) | Router, workflows, insight (~12) | General chat, tool search, coverage | +| `crawl` | Crawl, on-page, schema, accessibility | Technical crawl audits | +| `google` | Google, insight, CTR, keywords | GSC/GA4 analysis | +| `links` | Links, backlinks, indexation | Link architecture | +| `full` | All 240 tools | Debugging / legacy single-server setup | + Add to `.cursor/mcp.json` (or Cursor MCP settings): +```json +{ + "mcpServers": { + "site-audit-core": { + "command": "python", + "args": ["-m", "website_profiling.mcp"], + "env": { + "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling", + "PYTHONPATH": "src", + "WP_MCP_DOMAIN": "core", + "WP_PROPERTY_ID": "1" + } + }, + "site-audit-google": { + "command": "python", + "args": ["-m", "website_profiling.mcp"], + "env": { + "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling", + "PYTHONPATH": "src", + "WP_MCP_DOMAIN": "google", + "WP_PROPERTY_ID": "1" + } + } + } +} +``` + +Single-server legacy setup (all tools): + ```json { "mcpServers": { @@ -23,6 +62,7 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings): "env": { "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling", "PYTHONPATH": "src", + "WP_MCP_DOMAIN": "full", "WP_PROPERTY_ID": "1" } } @@ -41,9 +81,14 @@ Add to `.cursor/mcp.json` (or Cursor MCP settings): | `audit://property/{id}/report/latest` | Payload key index (counts, not full blob) | | `audit://property/{id}/report/{report_id}` | Payload key index for a specific report | | `audit://glossary` | Excerpt from `docs/GLOSSARY.md` | -| `audit://tools` | Tool catalog grouped by SEO domain | +| `audit://tools` | Tool catalog for the connected `WP_MCP_DOMAIN` server | +| `audit://domains` | Available MCP domain bundles and tool groupings | -## Tools (221 read-only + export) +## Tools (240 read-only + export) + +### Router and insight (Tier 0 — `WP_MCP_DOMAIN=core`) + +`search_audit_tools`, `list_tool_domains`, `get_data_coverage_report`, `run_insight_workflow`, `run_technical_workflow`, `run_keyword_workflow`, `run_domain_agent`, `get_landing_page_blended_table`, `get_opportunity_matrix`, `get_traffic_health_check`, `get_landing_page_full_diagnosis`, `get_issue_to_traffic_map` ### Export and deliverables @@ -99,11 +144,11 @@ Size-based tools require `probe_image_inventory=true` in pipeline config when bu ### Keywords -`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `get_serp_feature_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`, `list_keywords_ctr_opportunity`, `expand_keywords`, `generate_content_brief` +`get_keyword_summary`, `search_keywords`, `get_striking_distance_keywords`, `get_keyword_cannibalisation`, `get_query_page_misalignment`, `get_semantic_keyword_clusters`, `get_keyword_history`, `get_keyword_serp_overlay`, `get_serp_feature_overlay`, `list_keywords_by_action`, `list_keywords_by_position`, `list_keywords_by_impressions`, `list_keywords_ctr_opportunity`, `expand_keywords`, `generate_content_brief`, `get_brand_keyword_split`, `list_keywords_by_intent` ### Google and CTR -`get_google_summary`, `get_google_integration_status`, `get_gsc_top_queries`, `get_gsc_top_pages`, `get_gsc_ctr_opportunity_pages`, `get_ga4_summary`, `get_ga4_page_metrics`, `get_gsc_page_query_slice`, `get_gsc_url_inspection`, `get_gsc_index_coverage`, `analyze_serp_snippet_for_url` +`get_google_summary`, `get_google_integration_status`, `get_gsc_top_queries`, `get_gsc_top_pages`, `get_gsc_ctr_opportunity_pages`, `get_ga4_summary`, `get_ga4_page_metrics`, `get_gsc_page_query_slice`, `get_gsc_url_inspection`, `get_gsc_index_coverage`, `analyze_serp_snippet_for_url`, `get_gsc_daily_trend`, `get_ga4_daily_trend`, `get_ga4_by_device`, `get_ga4_by_channel`, `get_gsc_page_queries` ### Backlinks @@ -163,6 +208,8 @@ Already available: `validate_rich_results`, `get_gsc_url_inspection`, `export_si The same tools power **AI Chat** at [http://localhost:3000/chat](http://localhost:3000/chat). Enable AI in Run audit → AI settings. +In-app chat uses **dynamic tool routing**: each turn loads Tier 0 router tools plus a domain-scoped subset (~45 tools), not the full catalog. Set `CHAT_TOOL_MODE=full` to load all tools for debugging. + ## Ollama note When the local Ollama daemon supports native tools (most current models, including Ollama cloud refs like `minimax-m3:cloud`), chat uses Ollama’s `/api/chat` tool format. Older or tool-less models fall back to JSON ReAct parsing. OpenAI and Anthropic always use native tool calling with streaming in the chat UI. diff --git a/src/website_profiling/integrations/google/store.py b/src/website_profiling/integrations/google/store.py index 0a4da7ae..79a4f272 100644 --- a/src/website_profiling/integrations/google/store.py +++ b/src/website_profiling/integrations/google/store.py @@ -63,3 +63,29 @@ def read_latest_google_data( def _to_payload_shape(data: dict[str, Any]) -> dict[str, Any]: """Strip gsc_full/ga4_full keys from the payload.""" return {k: v for k, v in data.items() if k not in ("gsc_full", "ga4_full")} + + +def read_google_data_full( + conn: Connection, + property_id: int | None = None, +) -> Optional[dict[str, Any]]: + """Return latest google_data row including gsc_full and ga4_full blobs.""" + try: + if property_id is not None: + cur = conn.execute( + """ + SELECT data FROM google_data + WHERE property_id = %s + ORDER BY id DESC LIMIT 1 + """, + (property_id,), + ) + else: + cur = conn.execute("SELECT data FROM google_data ORDER BY id DESC LIMIT 1") + row = cur.fetchone() + if row is None: + return None + data = _parse_row_json(row) + return data if isinstance(data, dict) else None + except Exception: + return None diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py index 9194ddae..ecdcaa9f 100644 --- a/src/website_profiling/llm/agent.py +++ b/src/website_profiling/llm/agent.py @@ -8,6 +8,12 @@ from ..text_sanitize import sanitize_unicode_deep, strip_surrogates from ..tools.audit_tools import AuditToolContext from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, openai_tools_schema +from ..tools.audit_tools.tool_selector import ( + apply_tool_cap, + chat_tool_mode, + chat_tool_search_cap, + select_tools_for_turn, +) from .base import ChatResult, ToolCall, get_llm_client MAX_TOOL_ROUNDS = 10 @@ -15,24 +21,13 @@ SYSTEM_PROMPT = """You are Site Audit AI, a technical SEO assistant for a self-hosted site audit platform. You help users understand crawl results, audit issues, Lighthouse scores, keywords, and Search Console data. -Tool domains (prefer specific tools over generic list_issues): -- Portfolio/report: get_report_summary, get_category_scores, list_audit_categories, get_executive_summary, get_audit_recommendations, list_report_history, get_portfolio_summary -- Issues: list_issues, search_issues, list_top_impact_issues, prioritize_fix_roadmap, get_critical_issues, list_issues_by_category, get_category_issues, list_issues_with_ai_fixes, generate_issue_fix, list_issue_workflow -- On-page: list_pages_missing_title, list_pages_noindex, list_seo_onpage_issues, list_content_url_issues, list_pages_missing_canonical, list_canonical_mismatch, list_pages_with_missing_alt, list_pages_missing_viewport -- Crawl/pages: search_pages, search_pages_advanced, get_page_details, get_page_analysis, list_status_4xx_pages, list_pages_soft_404, list_dead_end_pages, list_duplicate_title_groups, list_heavy_pages_by_bytes, get_asset_weight_summary, get_readability_summary, get_status_code_breakdown, get_depth_distribution, list_long_redirect_chains, list_robots_blocked_urls, get_top_pages_by_pagerank -- Schema/technical: get_schema_coverage, get_seo_health, get_security_findings, get_security_findings_summary, get_tech_stack_summary, list_pages_by_technology -- Indexation: get_indexation_coverage, list_indexation_gaps, get_indexation_url_join -- Keywords: get_keyword_summary, get_striking_distance_keywords, list_keywords_ctr_opportunity, list_keywords_by_position, get_keyword_serp_overlay, get_serp_feature_overlay, expand_keywords, generate_content_brief -- Google: get_google_summary, get_gsc_top_queries, get_gsc_top_pages, get_gsc_ctr_opportunity_pages, get_google_integration_status, get_gsc_page_query_slice, get_gsc_url_inspection, get_gsc_index_coverage, get_ga4_page_metrics, analyze_serp_snippet_for_url -- Links/backlinks: get_gsc_sample_links, get_backlinks_velocity, get_third_party_links_overlay, list_broken_link_sources, get_page_coach -- Performance: get_lighthouse_summary, list_slow_pages, get_crux_summary, get_lighthouse_human_summary, list_lighthouse_poor_accessibility_pages, list_lighthouse_cwv_failures -- Content/charts: get_issue_priority_breakdown, get_mime_type_breakdown, get_title_length_distribution, get_domain_link_distribution, get_outlink_distribution, get_content_analytics, get_top_crawled_pages, get_duplicate_cluster -- Ops/logs: get_property_ops, list_crawl_runs, get_latest_log_analysis, get_log_top_paths, list_log_only_paths, list_crawl_only_paths, get_log_googlebot_stats -- Drift: compare_reports, compare_category_deltas, compare_issue_deltas, compare_indexation_deltas, compare_orphan_deltas, compare_url_set_diff, compare_google_metrics, compare_security_deltas, compare_health_score_delta, get_health_history, get_category_health_history -- GEO/AEO: get_geo_readiness_score, get_aeo_content_signals_for_url, get_llms_txt_status, draft_llms_txt, get_faq_schema_coverage, get_eeat_signals_summary, get_internal_link_suggestions, check_ai_citation_presence -- Accessibility/assets: list_pages_with_axe_violations, get_axe_audit_summary, list_pages_with_mixed_content, list_pages_poor_cache_headers, get_rich_results_summary, list_rich_results_failures -- Export/deliverables: export_audit_report, export_compare_csv, export_list_as_csv, compose_custom_report, export_custom_report, list_export_formats -- Images: get_image_audit_summary, list_pages_with_missing_alt, list_pages_without_lazy_images, list_pages_with_images_missing_dimensions, list_site_image_urls, list_lighthouse_image_opportunities, list_largest_images, list_unoptimized_images, list_images_needing_attention +Tool routing (only a subset of tools is loaded each turn): +- Always available: search_audit_tools, list_tool_domains, get_data_coverage_report, run_insight_workflow, run_technical_workflow, run_keyword_workflow, run_domain_agent, plus top insight tools (get_report_summary, get_opportunity_matrix, get_traffic_health_check, etc.) +- Use search_audit_tools(query) to discover specialized tools by topic (e.g. "broken links", "GSC CTR", "export PDF"). +- Use list_tool_domains to see domain groupings and example prompts. +- Use run_*_workflow for common multi-step analyses (insight, technical, keyword). +- Use run_domain_agent(task, domain) for deep exploration within one domain. +- Use get_data_coverage_report when tools return empty or missing data. Image playbook: - Overview: get_image_audit_summary first — the UI renders summary cards, page preview lists (alt/lazy/OG/dimensions), and Lighthouse image findings. Write only ### Power Insights and ### Recommended actions (interpretation). Never repeat counts, URL lists, or markdown tables of pages. @@ -120,9 +115,11 @@ def _react_step( return ChatResult(content=text) -def _tools_description(*, compact: bool = False) -> str: +def _tools_description(*, names: set[str] | None = None, compact: bool = False) -> str: lines = [] for t in TOOL_DEFINITIONS: + if names is not None and t["name"] not in names: + continue if compact: lines.append(f"- {t['name']}") else: @@ -130,6 +127,41 @@ def _tools_description(*, compact: bool = False) -> str: return "\n".join(lines) +def _last_user_message(messages: list[dict[str, str]]) -> str: + for msg in reversed(messages): + if msg.get("role") == "user": + return str(msg.get("content") or "") + return "" + + +def _expand_active_tools_from_result( + tc_name: str, + tool_result: dict[str, Any], + active: set[str], +) -> set[str]: + expanded = set(active) + pinned: set[str] = set() + + if tc_name == "search_audit_tools": + names = tool_result.get("tool_names") + if isinstance(names, list): + for name in names[:12]: + if isinstance(name, str) and name: + expanded.add(name) + pinned.add(name) + elif tc_name == "run_domain_agent": + names = tool_result.get("tools_used") + if isinstance(names, list): + for name in names: + if isinstance(name, str) and name: + expanded.add(name) + pinned.add(name) + + if chat_tool_mode() != "full" and pinned: + expanded = apply_tool_cap(expanded, chat_tool_search_cap(), pinned=pinned) + return expanded + + def _build_openai_messages(history: list[dict[str, str]]) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}] for msg in history: @@ -164,7 +196,9 @@ def run_agent_turn( return {"ok": False, "error": msg} openai_messages = _build_openai_messages(messages) - tools = openai_tools_schema() + last_user = _last_user_message(messages) + active_names = select_tools_for_turn(last_user, messages) + tools = openai_tools_schema(active_names) tool_events: list[dict[str, Any]] = [] final_message = "" @@ -182,7 +216,12 @@ def on_token(text: str) -> None: if _supports_native_tools(client): result = client.chat_with_tools(llm_messages, tools, on_token=on_token) else: - result = _react_step(client, llm_messages, _tools_description(compact=True), on_token) + result = _react_step( + client, + llm_messages, + _tools_description(names=active_names, compact=True), + on_token, + ) except Exception as e: msg = str(e).strip() or type(e).__name__ if "httpx" in msg.lower() or "requirements.txt" in msg.lower(): @@ -227,12 +266,22 @@ def on_token(text: str) -> None: for tc in result.tool_calls: _emit(on_event, {"type": "tool_start", "name": tc.name, "args": tc.arguments}) - tool_result = sanitize_unicode_deep( - dispatch_tool(tc.name, tc.arguments, context=context), - ) + if chat_tool_mode() != "full" and tc.name not in active_names: + tool_result = { + "error": f"tool not loaded this turn: {tc.name}", + "hint": "Call search_audit_tools to load specialized tools, or rephrase your request.", + } + else: + tool_result = sanitize_unicode_deep( + dispatch_tool(tc.name, tc.arguments, context=context), + ) _emit(on_event, {"type": "tool_end", "name": tc.name, "result": tool_result}) tool_events.append({"name": tc.name, "args": tc.arguments, "result": tool_result}) + active_names = _expand_active_tools_from_result(tc.name, tool_result, active_names) + if chat_tool_mode() != "full": + tools = openai_tools_schema(active_names) + tool_content = json.dumps(tool_result, default=str) if ollama_format: openai_messages.append({ diff --git a/src/website_profiling/mcp/core_server.py b/src/website_profiling/mcp/core_server.py new file mode 100644 index 00000000..7026c4af --- /dev/null +++ b/src/website_profiling/mcp/core_server.py @@ -0,0 +1,8 @@ +"""MCP core router server (Tier 0 + insight tools).""" +from __future__ import annotations + +from .domain_server import run_domain_server + + +def main() -> None: + run_domain_server("core") diff --git a/src/website_profiling/mcp/domain_server.py b/src/website_profiling/mcp/domain_server.py new file mode 100644 index 00000000..96f617dc --- /dev/null +++ b/src/website_profiling/mcp/domain_server.py @@ -0,0 +1,11 @@ +"""MCP domain server entry — set WP_MCP_DOMAIN before starting.""" +from __future__ import annotations + +import os + +from .server import main + + +def run_domain_server(domain: str) -> None: + os.environ["WP_MCP_DOMAIN"] = domain + main() diff --git a/src/website_profiling/mcp/server.py b/src/website_profiling/mcp/server.py index 5b150e61..4ca37356 100644 --- a/src/website_profiling/mcp/server.py +++ b/src/website_profiling/mcp/server.py @@ -9,7 +9,14 @@ from ..db.storage import db_session from ..tools.audit_tools import AuditToolContext -from ..tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool, tool_handler_names +from ..tools.audit_tools.registry import ( + TOOL_DEFINITIONS, + dispatch_tool, + list_domains_catalog, + mcp_tool_names, + tools_catalog_by_domain, +) +from ..tools.audit_tools.tool_domains import MCP_DOMAIN_BUNDLES _URI_PROPERTY = re.compile(r"^audit://property/(\d+)$") _URI_REPORT_LATEST = re.compile(r"^audit://property/(\d+)/report/latest$") @@ -64,90 +71,41 @@ def _read_glossary_excerpt() -> str: return text[:12000] +def _mcp_domain() -> str: + return (os.environ.get("WP_MCP_DOMAIN") or "core").strip().lower() + + +def _exposed_tool_names() -> set[str]: + return mcp_tool_names(_mcp_domain()) + + def _tools_catalog_json() -> str: - domains: dict[str, list[str]] = { - "portfolio": [], - "issues": [], - "crawl": [], - "schema": [], - "links": [], - "indexation": [], - "content": [], - "keywords": [], - "google": [], - "backlinks": [], - "performance": [], - "drift": [], - "security": [], - "ops": [], - "export": [], - "images": [], - "geo": [], - "accessibility": [], - "assets": [], - "ctr": [], - "integrations": [], - } - for tool in TOOL_DEFINITIONS: - name = tool["name"] - if name.startswith("export_") or name == "compose_custom_report" or name == "list_export_formats": - domains["export"].append(name) - elif name.startswith(("get_image_", "list_pages_without_lazy", "list_pages_with_images_missing", "list_site_image", "list_lighthouse_image", "list_largest_images", "list_unoptimized_images", "list_images_needing")): - domains["images"].append(name) - elif name.startswith(("list_propert", "get_propert", "get_report", "get_executive", "get_site", "list_report", "get_portfolio")) or name in ( - "get_ads_txt_status", - "get_security_txt_status", - "get_contact_intelligence", - "get_rich_results_summary", - "list_rich_results_failures", - "get_competitor_keyword_gap", - "get_pagination_audit_summary", - ): - domains["portfolio"].append(name) - elif name in ( - "list_top_impact_issues", - "prioritize_fix_roadmap", - "generate_issue_fix", - "summarize_category_for_client", - ) or "issue" in name or "category" in name or "workflow" in name: - domains["issues"].append(name) - elif name.startswith(("get_geo_", "get_aeo_", "get_llms_", "get_eeat_", "get_faq_", "list_pages_missing_faq", "draft_llms", "check_ai_citation")): - domains["geo"].append(name) - elif "axe" in name or "mixed_content" in name or name == "get_heading_outline_for_url": - domains["accessibility"].append(name) - elif name in ("get_asset_weight_summary", "get_readability_summary", "list_heavy_pages_by_bytes", "list_pages_poor_cache_headers", "list_pages_low_content_ratio"): - domains["assets"].append(name) - elif "ctr" in name or name in ("list_keywords_ctr_opportunity", "analyze_serp_snippet_for_url"): - domains["ctr"].append(name) - elif name in ("get_gsc_url_inspection", "get_gsc_index_coverage", "get_bing_index_status", "get_serp_feature_overlay"): - domains["integrations"].append(name) - elif name.startswith(("list_pages_", "list_canonical", "list_long_", "list_robots_", "get_top_pages_by", "search_pages", "get_page_", "list_redirects", "list_broken", "list_status_", "get_status_code", "get_response_time", "get_depth", "get_crawl_", "get_browser", "list_pages_with", "list_pages_by")): - domains["crawl"].append(name) - elif "schema" in name or name == "get_seo_health": - domains["schema"].append(name) - elif "orphan" in name or "link" in name or "fingerprint" in name or "pagerank" in name: - domains["links"].append(name) - elif "indexation" in name or "hreflang" in name or "language" in name or name == "list_subdomains": - domains["indexation"].append(name) - elif "content" in name or "social" in name or "ner" in name or "thin" in name or "opportunit" in name or "duplicate" in name: - domains["content"].append(name) - elif "keyword" in name or "cannibal" in name or "misalignment" in name or "striking" in name or "semantic" in name or name == "expand_keywords" or name == "generate_content_brief": - domains["keywords"].append(name) - elif "google" in name or "gsc" in name or "ga4" in name: - domains["google"].append(name) - elif "backlink" in name or "competitor" in name or "bing" in name or "gsc_links" in name: - domains["backlinks"].append(name) - elif "lighthouse" in name or "crux" in name or "slow" in name or "cwv" in name: - domains["performance"].append(name) - elif "health" in name or "compare" in name or "alert" in name or "tech_stack" in name or name == "list_pages_by_technology": - domains["drift"].append(name) - elif "security" in name: - domains["security"].append(name) - elif "log" in name or name in ("get_property_ops", "list_crawl_runs", "list_log_uploads", "get_page_coach"): - domains["ops"].append(name) - else: - domains["portfolio"].append(name) - return json.dumps({"tool_count": len(TOOL_DEFINITIONS), "handlers": sorted(tool_handler_names()), "domains": domains}, indent=2) + domain = _mcp_domain() + exposed = _exposed_tool_names() + by_domain = tools_catalog_by_domain() + scoped: dict[str, list[str]] = {} + for d, names in by_domain.items(): + filtered = [n for n in names if n in exposed] + if filtered: + scoped[d] = filtered + return json.dumps({ + "mcp_domain": domain, + "tool_count": len(exposed), + "handlers": sorted(exposed), + "domains": scoped, + "available_mcp_domains": sorted(MCP_DOMAIN_BUNDLES.keys()), + }, indent=2) + + +def _domains_resource_json() -> str: + return json.dumps({ + "current_mcp_domain": _mcp_domain(), + "bundles": { + key: sorted(domains) + for key, domains in MCP_DOMAIN_BUNDLES.items() + }, + "catalog": list_domains_catalog(), + }, indent=2) def _resolve_resource(uri: str) -> str: @@ -161,6 +119,9 @@ def _resolve_resource(uri: str) -> str: if uri == "audit://tools": return _tools_catalog_json() + if uri == "audit://domains": + return _domains_resource_json() + m = _URI_PROPERTY.match(uri) if m: pid = int(m.group(1)) @@ -202,13 +163,16 @@ def main() -> None: "MCP SDK not installed. Run: pip install -r requirements.txt", ) from e - server = Server("site-audit") + server = Server(f"site-audit-{_mcp_domain()}") default_pid = _default_property_id() + exposed = _exposed_tool_names() @server.list_tools() async def list_tools() -> list[Tool]: out: list[Tool] = [] for spec in TOOL_DEFINITIONS: + if spec["name"] not in exposed: + continue out.append( Tool( name=spec["name"], @@ -220,6 +184,12 @@ async def list_tools() -> list[Tool]: @server.call_tool() async def call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextContent]: + if name not in exposed: + result = { + "error": f"tool not exposed in MCP domain {_mcp_domain()}: {name}", + "hint": "Connect WP_MCP_DOMAIN=full or the domain server that includes this tool.", + } + return [TextContent(type="text", text=json.dumps(result, indent=2, default=str))] args = dict(arguments or {}) ctx = _merge_context(args) result = dispatch_tool(name, args, context=ctx) @@ -231,7 +201,8 @@ async def list_resources() -> list[Resource]: resources = [ Resource(uri="audit://properties", name="Properties", description="All configured site properties", mimeType="application/json"), Resource(uri="audit://glossary", name="Glossary", description="Site Audit field glossary excerpt", mimeType="text/markdown"), - Resource(uri="audit://tools", name="Tool catalog", description="MCP tool catalog grouped by domain", mimeType="application/json"), + Resource(uri="audit://tools", name="Tool catalog", description="MCP tool catalog for the connected domain server", mimeType="application/json"), + Resource(uri="audit://domains", name="MCP domain servers", description="Available WP_MCP_DOMAIN bundles and domain groupings", mimeType="application/json"), ] if default_pid: resources.extend([ diff --git a/src/website_profiling/tools/audit_tools/context.py b/src/website_profiling/tools/audit_tools/context.py index 9f09ec87..7b1f86bf 100644 --- a/src/website_profiling/tools/audit_tools/context.py +++ b/src/website_profiling/tools/audit_tools/context.py @@ -11,7 +11,7 @@ from ...db.report_store import read_report_payload from ...integrations.google.gsc_links_store import read_latest_gsc_links_data from ...integrations.google.keyword_store import read_latest_keyword_data -from ...integrations.google.store import read_latest_google_data +from ...integrations.google.store import read_latest_google_data, read_google_data_full @dataclass @@ -50,6 +50,14 @@ def load_keywords(self, conn: Connection) -> Optional[dict[str, Any]]: embedded = payload.get("keywords") return embedded if isinstance(embedded, dict) else None + def load_google_full(self, conn: Connection) -> Optional[dict[str, Any]]: + full = read_google_data_full(conn, self.property_id) + if full: + return full + payload = self.load_payload(conn) + embedded = payload.get("google") + return embedded if isinstance(embedded, dict) else None + def load_gsc_links(self, conn: Connection) -> Optional[dict[str, Any]]: links = read_latest_gsc_links_data(conn, self.property_id, for_report=False) if links: diff --git a/src/website_profiling/tools/audit_tools/data_coverage.py b/src/website_profiling/tools/audit_tools/data_coverage.py new file mode 100644 index 00000000..5a30fb47 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/data_coverage.py @@ -0,0 +1,94 @@ +"""Data coverage report — which integrations and optional audit data are populated.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...db.property_store import get_property_by_id +from .context import AuditToolContext + + +def _check(name: str, populated: bool, hint: str = "") -> dict[str, Any]: + return {"signal": name, "populated": populated, "config_hint": hint if not populated else ""} + + +def get_data_coverage_report(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required", "checks": []} + + prop = get_property_by_id(conn, scoped.property_id) + if not prop: + return {"error": "property not found", "checks": []} + + payload = scoped.load_payload(conn) + google = scoped.load_google(conn) + keywords = scoped.load_keywords(conn) + gsc_links = scoped.load_gsc_links(conn) + + checks: list[dict[str, Any]] = [] + checks.append(_check( + "google_oauth", + bool(prop.get("google_refresh_token")), + "Connect Google OAuth in Integrations.", + )) + checks.append(_check( + "gsc_data", + bool(google and isinstance(google.get("gsc"), dict) and (google.get("gsc") or {}).get("summary")), + "Map GSC site URL and re-run pipeline.", + )) + checks.append(_check( + "ga4_data", + bool(google and isinstance(google.get("ga4"), dict) and (google.get("ga4") or {}).get("summary")), + "Connect GA4 property in Integrations.", + )) + checks.append(_check( + "keyword_data", + bool(keywords and (keywords.get("rows") or keywords.get("total_keywords"))), + "Run keyword enrichment in pipeline.", + )) + checks.append(_check( + "gsc_links_import", + bool(gsc_links and (gsc_links.get("sample_links") or gsc_links.get("referring_domains"))), + "Import GSC Links CSV in Backlinks view.", + )) + overlays = (gsc_links or {}).get("third_party_overlays") if isinstance(gsc_links, dict) else None + checks.append(_check( + "moz_majestic_overlay", + bool(isinstance(overlays, list) and overlays), + "Upload Moz or Majestic CSV in Backlinks > third-party overlay.", + )) + checks.append(_check( + "image_inventory", + bool(payload.get("image_inventory")), + "Set probe_image_inventory=true in pipeline config and rebuild report.", + )) + checks.append(_check( + "axe_violations", + bool(payload.get("axe_audit_summary") or payload.get("axe_violations")), + "Set enable_axe=true and use javascript/auto crawl rendering.", + )) + checks.append(_check( + "rich_results_validation", + bool(payload.get("rich_results_validation") or payload.get("rich_results_meta")), + "Enable rich results validation on report build.", + )) + checks.append(_check( + "audit_report", + bool(payload), + "Run a site audit crawl and report build.", + )) + + missing = [c["signal"] for c in checks if not c["populated"]] + return { + "property_id": scoped.property_id, + "checks": checks, + "missing_count": len(missing), + "missing": missing, + "provenance": {"sources": ["property", "google_data", "report_payload"], "confidence": "high"}, + "insights": [ + f"{len(checks) - len(missing)}/{len(checks)} data signals populated.", + *(f"Enable: {m}" for m in missing[:5]), + ], + } diff --git a/src/website_profiling/tools/audit_tools/google.py b/src/website_profiling/tools/audit_tools/google.py index 14184fec..5043131b 100644 --- a/src/website_profiling/tools/audit_tools/google.py +++ b/src/website_profiling/tools/audit_tools/google.py @@ -95,7 +95,7 @@ def get_gsc_page_query_slice(conn: Connection, ctx: AuditToolContext, args: dict url = str(args.get("url") or "").strip() if not url: return {"error": "url is required"} - data = scoped.load_google(conn) + data = scoped.load_google_full(conn) or scoped.load_google(conn) if not data: return {"error": "no google data found"} slice_data = slice_from_google_row(data, url) @@ -187,3 +187,58 @@ def get_gsc_ctr_opportunity_pages(conn: Connection, ctx: AuditToolContext, args: "truncated": sliced["truncated"], "provenance": "Search Console", } + + +def _google_series(data: dict[str, Any] | None, section: str, key: str) -> dict[str, Any]: + if not data: + return {"error": "no google data found", "missing": True} + block = data.get(section) if isinstance(data.get(section), dict) else {} + series = block.get(key) or [] + return { + key: series if isinstance(series, list) else [], + "fetched_at": data.get("fetched_at"), + "date_range": data.get("date_range"), + "provenance": "Search Console" if section == "gsc" else "Google Analytics 4", + } + + +def get_gsc_daily_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + return _google_series(scoped.load_google(conn), "gsc", "daily") + + +def get_ga4_daily_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + return _google_series(scoped.load_google(conn), "ga4", "daily") + + +def get_ga4_by_device(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + return _google_series(scoped.load_google(conn), "ga4", "by_device") + + +def get_ga4_by_channel(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + return _google_series(scoped.load_google(conn), "ga4", "by_channel") + + +def get_gsc_page_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required"} + raw = scoped.load_google_full(conn) or scoped.load_google(conn) + if not raw: + return {"error": "no google data found", "missing": True} + slice_data = slice_from_google_row(raw, url) + gsc = slice_data.get("gsc") if isinstance(slice_data.get("gsc"), dict) else {} + queries = gsc.get("queries") or [] + limit = parse_limit(args.get("limit"), 25, 50) + sliced = cap_list(queries if isinstance(queries, list) else [], limit, max_cap=50) + return { + "url": url, + "queries": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "provenance": "Search Console", + } diff --git a/src/website_profiling/tools/audit_tools/insight_helpers.py b/src/website_profiling/tools/audit_tools/insight_helpers.py new file mode 100644 index 00000000..45431067 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/insight_helpers.py @@ -0,0 +1,207 @@ +"""Shared helpers for cross-platform GSC + GA4 + audit insight tools.""" +from __future__ import annotations + +from typing import Any + +from ...integrations.google.normalize import normalize_url, url_to_path + + +def provenance_block( + sources: list[str], + fetched_at: str | None = None, + *, + confidence: str = "high", +) -> dict[str, Any]: + return { + "sources": sources, + "fetched_at": fetched_at, + "confidence": confidence, + } + + +def _num(val: Any, default: float = 0.0) -> float: + try: + if val is None: + return default + return float(val) + except (TypeError, ValueError): + return default + + +def classify_opportunity_quadrant( + gsc_row: dict[str, Any] | None, + ga4_row: dict[str, Any] | None, + *, + site_median_sessions: float = 0.0, +) -> str: + position = _num((gsc_row or {}).get("position"), 99) + impressions = _num((gsc_row or {}).get("impressions")) + sessions = _num((ga4_row or {}).get("sessions")) + engagement = _num((ga4_row or {}).get("engagementRate")) + + rank_potential = impressions >= 100 and 4 <= position <= 20 + convert_potential = sessions >= max(site_median_sessions * 0.5, 5) or engagement >= 0.5 + + if rank_potential and convert_potential: + return "high_impact" + if rank_potential: + return "worth_optimizing" + if convert_potential: + return "good_but_capped" + return "low_priority" + + +def traffic_health_ratio( + gsc_summary: dict[str, Any] | None, + ga4_summary: dict[str, Any] | None, +) -> dict[str, Any]: + clicks = _num((gsc_summary or {}).get("clicks")) + sessions = _num((ga4_summary or {}).get("sessions")) + if clicks <= 0 and sessions <= 0: + return { + "gsc_clicks": clicks, + "ga4_sessions": sessions, + "ratio": None, + "diagnosis": "no_data", + "note": "Connect GSC and GA4 and re-run the pipeline.", + } + ratio = sessions / clicks if clicks > 0 else None + diagnosis = "healthy" + note = "GSC clicks and GA4 sessions are in a plausible range." + if ratio is not None: + if ratio < 0.3: + diagnosis = "tracking_gap" + note = "GA4 sessions are much lower than GSC clicks — check filters, consent mode, or landing page tagging." + elif ratio > 3.0: + diagnosis = "filter_issue" + note = "GA4 sessions exceed GSC clicks — GA4 may include non-organic traffic or GSC date range differs." + return { + "gsc_clicks": clicks, + "ga4_sessions": sessions, + "ratio": round(ratio, 3) if ratio is not None else None, + "diagnosis": diagnosis, + "note": note, + } + + +def blend_landing_pages( + gsc_by_page: dict[str, Any], + ga4_by_path: dict[str, Any], + *, + limit: int = 50, + min_impressions: int = 0, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + ga4_by_norm: dict[str, dict[str, Any]] = {} + for path, val in (ga4_by_path or {}).items(): + if not isinstance(val, dict): + continue + full = str(val.get("full_url") or path) + ga4_by_norm[normalize_url(full)] = val + ga4_by_norm[normalize_url(str(path))] = val + + session_vals = [_num(v.get("sessions")) for v in ga4_by_norm.values() if isinstance(v, dict)] + session_vals.sort() + median_sessions = session_vals[len(session_vals) // 2] if session_vals else 0.0 + + for page_url, gsc_row in (gsc_by_page or {}).items(): + if not isinstance(gsc_row, dict): + continue + impressions = _num(gsc_row.get("impressions")) + if impressions < min_impressions: + continue + norm = normalize_url(str(page_url)) + ga4_row = ga4_by_norm.get(norm) + if ga4_row is None: + path = url_to_path(str(page_url)) + ga4_row = ga4_by_norm.get(normalize_url(path)) + quadrant = classify_opportunity_quadrant( + gsc_row, ga4_row if isinstance(ga4_row, dict) else None, + site_median_sessions=median_sessions, + ) + rows.append({ + "url": page_url, + "gsc_clicks": int(_num(gsc_row.get("clicks"))), + "gsc_impressions": int(impressions), + "gsc_position": round(_num(gsc_row.get("position")), 1), + "gsc_ctr": round(_num(gsc_row.get("ctr")), 4), + "ga4_sessions": int(_num((ga4_row or {}).get("sessions"))) if ga4_row else 0, + "ga4_engagement_rate": round(_num((ga4_row or {}).get("engagementRate")), 3) if ga4_row else None, + "quadrant": quadrant, + }) + + rows.sort(key=lambda r: (-r["gsc_clicks"], -r["gsc_impressions"])) + return rows[: max(1, min(limit, 100))] + + +def page_issue_flags(url: str, payload: dict[str, Any]) -> list[dict[str, Any]]: + norm = normalize_url(url) + flags: list[dict[str, Any]] = [] + for cat in payload.get("categories") or []: + if not isinstance(cat, dict): + continue + for issue in cat.get("issues") or []: + if not isinstance(issue, dict): + continue + issue_url = str(issue.get("url") or "") + if issue_url and normalize_url(issue_url) != norm: + continue + flags.append({ + "priority": issue.get("priority"), + "category_id": cat.get("id"), + "message": issue.get("message"), + "url": issue_url or url, + }) + return flags[:30] + + +def composite_page_score( + gsc_page: dict[str, Any] | None, + ga4_page: dict[str, Any] | None, + gsc_site: dict[str, Any] | None, + ga4_site: dict[str, Any] | None, + issue_flags: list[dict[str, Any]], + lighthouse: dict[str, Any] | None, +) -> dict[str, Any]: + score = 70.0 + flags_out: list[str] = [] + + site_pos = _num((gsc_site or {}).get("position"), 10) + page_pos = _num((gsc_page or {}).get("position"), site_pos) + if page_pos > site_pos + 5: + score -= 10 + flags_out.append("below_avg_gsc_position") + + site_eng = _num((ga4_site or {}).get("engagementRate"), 0.5) + page_eng = _num((ga4_page or {}).get("engagementRate"), site_eng) + if ga4_page and page_eng < site_eng * 0.7: + score -= 10 + flags_out.append("low_engagement") + + crit = sum(1 for f in issue_flags if str(f.get("priority")) == "Critical") + high = sum(1 for f in issue_flags if str(f.get("priority")) == "High") + if crit: + score -= min(20, crit * 10) + flags_out.append("critical_issues") + elif high: + score -= min(10, high * 5) + flags_out.append("high_issues") + + if lighthouse: + perf = _num(lighthouse.get("performance"), 100) + seo = _num(lighthouse.get("seo"), 100) + if perf < 50: + score -= 8 + flags_out.append("poor_lighthouse_performance") + if seo < 70: + score -= 5 + flags_out.append("poor_lighthouse_seo") + + score = max(0, min(100, round(score))) + if score >= 75: + band = "green" + elif score >= 50: + band = "amber" + else: + band = "red" + return {"score": score, "band": band, "flags": flags_out} diff --git a/src/website_profiling/tools/audit_tools/insight_tools.py b/src/website_profiling/tools/audit_tools/insight_tools.py new file mode 100644 index 00000000..1d9357b2 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/insight_tools.py @@ -0,0 +1,166 @@ +"""Cross-platform insight audit tools (GSC + GA4 + crawl + issues).""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...integrations.google.page_lookup import slice_from_google_row +from ._slice import cap_list, parse_limit +from .context import AuditToolContext +from .insight_helpers import ( + blend_landing_pages, + composite_page_score, + page_issue_flags, + provenance_block, + traffic_health_ratio, +) +from .report import list_issues + + +def _gsc_ga4_blobs(raw: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + gsc = raw.get("gsc_full") if isinstance(raw.get("gsc_full"), dict) else raw.get("gsc") or {} + ga4 = raw.get("ga4_full") if isinstance(raw.get("ga4_full"), dict) else raw.get("ga4") or {} + return gsc if isinstance(gsc, dict) else {}, ga4 if isinstance(ga4, dict) else {} + + +def get_landing_page_blended_table(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + raw = scoped.load_google_full(conn) + if not raw: + return {"error": "no google data found", "missing": True, "rows": []} + gsc, ga4 = _gsc_ga4_blobs(raw) + by_page = gsc.get("by_page") if isinstance(gsc.get("by_page"), dict) else {} + by_path = ga4.get("by_path") if isinstance(ga4.get("by_path"), dict) else {} + if not by_page and gsc.get("top_pages"): + by_page = {str(r.get("page")): r for r in (gsc.get("top_pages") or []) if isinstance(r, dict) and r.get("page")} + limit = parse_limit(args.get("limit"), 30, 100) + min_impressions = parse_limit(args.get("min_impressions"), 0, 1_000_000) + rows = blend_landing_pages(by_page, by_path, limit=limit, min_impressions=min_impressions) + return { + "rows": rows, + "total": len(rows), + "truncated": len(by_page) > limit, + "provenance": provenance_block(["gsc", "ga4"], raw.get("fetched_at")), + "insights": [ + f"{sum(1 for r in rows if r['quadrant'] == 'high_impact')} high-impact landing pages", + f"{sum(1 for r in rows if r['quadrant'] == 'worth_optimizing')} worth optimizing for rank", + ], + } + + +def get_opportunity_matrix(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + blended = get_landing_page_blended_table(conn, ctx, args) + if blended.get("error"): + return blended + quadrants: dict[str, list[dict[str, Any]]] = { + "high_impact": [], + "worth_optimizing": [], + "good_but_capped": [], + "low_priority": [], + } + for row in blended.get("rows") or []: + q = str(row.get("quadrant") or "low_priority") + quadrants.setdefault(q, []).append(row) + counts = {k: len(v) for k, v in quadrants.items()} + return { + "quadrants": quadrants, + "counts": counts, + "provenance": blended.get("provenance"), + "insights": [ + f"Focus on {counts.get('high_impact', 0)} high-impact pages first.", + f"{counts.get('worth_optimizing', 0)} pages could rank higher with on-page work.", + ], + } + + +def get_traffic_health_check(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + raw = scoped.load_google_full(conn) or scoped.load_google(conn) + if not raw: + return {"error": "no google data found", "missing": True} + gsc, ga4 = _gsc_ga4_blobs(raw) + health = traffic_health_ratio( + gsc.get("summary") if isinstance(gsc.get("summary"), dict) else {}, + ga4.get("summary") if isinstance(ga4.get("summary"), dict) else {}, + ) + return { + **health, + "provenance": provenance_block(["gsc", "ga4"], raw.get("fetched_at")), + "insights": [health.get("note") or ""], + } + + +def get_landing_page_full_diagnosis(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or "").strip() + if not url: + return {"error": "url is required"} + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "missing": True} + raw = scoped.load_google_full(conn) or scoped.load_google(conn) or {} + slice_data = slice_from_google_row(raw, url) + gsc_page = (slice_data.get("gsc") or {}) if isinstance(slice_data.get("gsc"), dict) else None + ga4_page = (slice_data.get("ga4") or {}) if isinstance(slice_data.get("ga4"), dict) else None + benchmarks = slice_data.get("siteBenchmarks") or {} + flags = page_issue_flags(url, payload) + lh = (payload.get("lighthouse_by_url") or {}).get(url) or {} + if not lh: + norm_url = url.rstrip("/") + for k, v in (payload.get("lighthouse_by_url") or {}).items(): + if str(k).rstrip("/") == norm_url: + lh = v + break + score = composite_page_score( + gsc_page, ga4_page, + (benchmarks.get("gsc") or {}) if isinstance(benchmarks.get("gsc"), dict) else {}, + (benchmarks.get("ga4") or {}) if isinstance(benchmarks.get("ga4"), dict) else {}, + flags, + lh if isinstance(lh, dict) else None, + ) + crawl_row = None + for row in payload.get("top_pages") or []: + if isinstance(row, dict) and str(row.get("url") or "").rstrip("/") == url.rstrip("/"): + crawl_row = row + break + return { + "url": url, + "gsc_ga4": slice_data, + "issues": flags, + "lighthouse": lh or None, + "crawl": crawl_row, + "diagnosis": score, + "provenance": provenance_block( + ["gsc", "ga4", "crawl", "audit"], + raw.get("fetched_at") or payload.get("report_generated_at"), + ), + "insights": score.get("flags") or [], + } + + +def get_issue_to_traffic_map(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + result = list_issues(conn, ctx, {**args, "sort": "impact"}) + if result.get("error"): + return result + issues = result.get("issues") or [] + rows = [] + for issue in issues: + if not isinstance(issue, dict): + continue + rows.append({ + "url": issue.get("url"), + "priority": issue.get("priority"), + "category": issue.get("category"), + "message": issue.get("message"), + "impact_score": issue.get("impact_score"), + "gsc_clicks": issue.get("gsc_clicks"), + "ga4_sessions": issue.get("ga4_sessions"), + }) + return { + "issues": rows, + "total": result.get("total"), + "truncated": result.get("truncated"), + "provenance": provenance_block(["audit", "gsc", "ga4"]), + "insights": ["Issues sorted by traffic-weighted impact_score."], + } diff --git a/src/website_profiling/tools/audit_tools/keywords.py b/src/website_profiling/tools/audit_tools/keywords.py index 79b9a8ae..e6cd45c8 100644 --- a/src/website_profiling/tools/audit_tools/keywords.py +++ b/src/website_profiling/tools/audit_tools/keywords.py @@ -250,3 +250,33 @@ def _impressions(row: dict[str, Any]) -> int: conn, ctx, args, lambda r: _impressions(r) >= min_v, ) + + +def get_brand_keyword_split(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required"} + data = scoped.load_keywords(conn) + if not data: + return {"error": "no keyword data found", "missing": True} + rows = [r for r in (data.get("rows") or []) if isinstance(r, dict)] + branded = [r for r in rows if r.get("is_branded")] + non_branded = [r for r in rows if not r.get("is_branded")] + return { + "brand_name": data.get("brand_name"), + "branded_count": len(branded), + "non_branded_count": len(non_branded), + "branded_sample": branded[:10], + "non_branded_sample": non_branded[:10], + "provenance": "Keywords enrichment", + } + + +def list_keywords_by_intent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + intent = str(args.get("intent") or "").strip().lower() + if not intent: + return {"error": "intent is required"} + return _filter_keyword_rows( + conn, ctx, args, + lambda r: str(r.get("intent") or "").lower() == intent, + ) diff --git a/src/website_profiling/tools/audit_tools/registry.py b/src/website_profiling/tools/audit_tools/registry.py index faa8da99..4eb75f7e 100644 --- a/src/website_profiling/tools/audit_tools/registry.py +++ b/src/website_profiling/tools/audit_tools/registry.py @@ -25,6 +25,22 @@ get_title_length_distribution, get_top_crawled_pages, ) +from .data_coverage import get_data_coverage_report +from .insight_tools import ( + get_issue_to_traffic_map, + get_landing_page_blended_table, + get_landing_page_full_diagnosis, + get_opportunity_matrix, + get_traffic_health_check, +) +from .router_tools import ( + list_tool_domains, + run_domain_agent, + run_insight_workflow, + run_keyword_workflow, + run_technical_workflow, + search_audit_tools, +) from .compare import compare_reports from .compare_slices import ( compare_category_deltas, @@ -108,10 +124,15 @@ search_pages_advanced, ) from .google import ( + get_ga4_by_channel, + get_ga4_by_device, + get_ga4_daily_trend, get_ga4_page_metrics, get_ga4_summary, get_google_summary, get_gsc_ctr_opportunity_pages, + get_gsc_daily_trend, + get_gsc_page_queries, get_gsc_page_query_slice, get_gsc_top_pages, get_gsc_top_queries, @@ -128,6 +149,7 @@ from .international import get_hreflang_summary, get_language_summary from .issues import get_category_issues, list_issues_by_category from .keywords import ( + get_brand_keyword_split, get_keyword_cannibalisation, get_keyword_history, get_keyword_serp_overlay, @@ -137,6 +159,7 @@ get_striking_distance_keywords, list_keywords_by_action, list_keywords_by_impressions, + list_keywords_by_intent, list_keywords_by_position, list_keywords_ctr_opportunity, search_keywords, @@ -260,6 +283,16 @@ ) from .tech import get_tech_stack_summary, list_pages_by_technology from .tool_catalog import TOOL_DEFINITIONS +from .tool_domains import ( + TIER_0_TOOLS, + build_tool_meta, + classify_tool_domain, + domains_catalog, + tool_names_for_domain as _meta_tool_names_for_domain, + tool_names_for_mcp_bundle, + tool_names_for_tier as _meta_tool_names_for_tier, + tools_by_domain, +) from .workflow import list_issue_workflow ToolHandler = Callable[[Connection, AuditToolContext, dict[str, Any]], dict[str, Any]] @@ -486,6 +519,25 @@ "get_bing_index_status": get_bing_index_status, "get_serp_feature_overlay": get_serp_feature_overlay, "check_ai_citation_presence": check_ai_citation_presence, + "search_audit_tools": search_audit_tools, + "list_tool_domains": list_tool_domains, + "get_data_coverage_report": get_data_coverage_report, + "run_insight_workflow": run_insight_workflow, + "run_technical_workflow": run_technical_workflow, + "run_keyword_workflow": run_keyword_workflow, + "run_domain_agent": run_domain_agent, + "get_landing_page_blended_table": get_landing_page_blended_table, + "get_opportunity_matrix": get_opportunity_matrix, + "get_traffic_health_check": get_traffic_health_check, + "get_landing_page_full_diagnosis": get_landing_page_full_diagnosis, + "get_issue_to_traffic_map": get_issue_to_traffic_map, + "get_gsc_daily_trend": get_gsc_daily_trend, + "get_ga4_daily_trend": get_ga4_daily_trend, + "get_ga4_by_device": get_ga4_by_device, + "get_ga4_by_channel": get_ga4_by_channel, + "get_gsc_page_queries": get_gsc_page_queries, + "get_brand_keyword_split": get_brand_keyword_split, + "list_keywords_by_intent": list_keywords_by_intent, } @@ -512,10 +564,85 @@ def dispatch_tool( return handler(session, merged_ctx, payload_args) -def openai_tools_schema() -> list[dict[str, Any]]: - """Convert TOOL_DEFINITIONS to OpenAI function-calling format.""" +_TOOL_DEFINITIONS_BY_NAME: dict[str, dict[str, Any]] = {t["name"]: t for t in TOOL_DEFINITIONS} +_TOOL_META: dict[str, dict[str, Any]] = build_tool_meta(set(_TOOL_HANDLERS.keys())) + + +def tool_meta() -> dict[str, dict[str, Any]]: + return _TOOL_META + + +def tool_definition(name: str) -> dict[str, Any] | None: + return _TOOL_DEFINITIONS_BY_NAME.get(name) + + +def tool_names_for_domain(domain: str) -> list[str]: + return _meta_tool_names_for_domain(_TOOL_META, domain) + + +def tool_names_for_tier(tier: int) -> list[str]: + return _meta_tool_names_for_tier(_TOOL_META, tier) + + +def tier0_tool_names() -> set[str]: + return set(TIER_0_TOOLS) & set(_TOOL_HANDLERS.keys()) + + +def mcp_tool_names(bundle: str) -> set[str]: + return tool_names_for_mcp_bundle(_TOOL_META, bundle) & set(_TOOL_HANDLERS.keys()) + + +def tools_catalog_by_domain() -> dict[str, list[str]]: + return tools_by_domain(_TOOL_META) + + +def list_domains_catalog() -> list[dict[str, Any]]: + return domains_catalog(_TOOL_META) + + +def search_tools(query: str, limit: int = 10) -> list[dict[str, Any]]: + """Keyword search over tool name, description, tags, and domain.""" + q = (query or "").strip().lower() + if not q: + return [] + tokens = [t for t in q.replace("/", " ").split() if t] + scored: list[tuple[int, str, dict[str, Any]]] = [] + for tool in TOOL_DEFINITIONS: + name = tool["name"] + desc = str(tool.get("description") or "").lower() + meta = _TOOL_META.get(name) or {} + domain = str(meta.get("domain") or classify_tool_domain(name)) + tags = " ".join(str(t) for t in (meta.get("tags") or [])) + haystack = f"{name} {desc} {domain} {tags}".lower() + score = 0 + if q in name: + score += 100 + if q in haystack: + score += 40 + for tok in tokens: + if tok in name: + score += 30 + elif tok in haystack: + score += 10 + if score <= 0: + continue + scored.append((score, name, { + "name": name, + "description": tool.get("description", ""), + "domain": domain, + "tier": meta.get("tier", 1), + })) + scored.sort(key=lambda x: (-x[0], x[1])) + cap = max(1, min(int(limit or 10), 50)) + return [row for _, _, row in scored[:cap]] + + +def openai_tools_schema(names: set[str] | None = None) -> list[dict[str, Any]]: + """Convert TOOL_DEFINITIONS to OpenAI function-calling format (optional name filter).""" out: list[dict[str, Any]] = [] for tool in TOOL_DEFINITIONS: + if names is not None and tool["name"] not in names: + continue out.append({ "type": "function", "function": { @@ -529,3 +656,19 @@ def openai_tools_schema() -> list[dict[str, Any]]: def tool_handler_names() -> set[str]: return set(_TOOL_HANDLERS.keys()) + + +def validate_tool_registry() -> list[str]: + """Return validation errors for catalog/handler/meta parity.""" + errors: list[str] = [] + handler_names = tool_handler_names() + catalog_names = {t["name"] for t in TOOL_DEFINITIONS} + meta_names = set(_TOOL_META.keys()) + if handler_names != catalog_names: + errors.append(f"handler/catalog mismatch: handlers={len(handler_names)} catalog={len(catalog_names)}") + if handler_names != meta_names: + errors.append(f"handler/meta mismatch: handlers={len(handler_names)} meta={len(meta_names)}") + missing_t0 = TIER_0_TOOLS - handler_names + if missing_t0: + errors.append(f"tier0 tools missing handlers: {sorted(missing_t0)}") + return errors diff --git a/src/website_profiling/tools/audit_tools/router_tools.py b/src/website_profiling/tools/audit_tools/router_tools.py new file mode 100644 index 00000000..6eb0a941 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/router_tools.py @@ -0,0 +1,147 @@ +"""Router and workflow meta-tools (Tier 0).""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ._slice import parse_limit +from .context import AuditToolContext +from .tool_domains import classify_tool_domain + + +def search_audit_tools(_conn: Connection, _ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + from .registry import search_tools + + query = str(args.get("query") or args.get("q") or "").strip() + limit = parse_limit(args.get("limit"), 10, 50) + if not query: + return {"error": "query is required", "tools": [], "tool_names": []} + matches = search_tools(query, limit=limit) + return { + "query": query, + "tools": matches, + "tool_names": [m["name"] for m in matches], + "total": len(matches), + } + + +def list_tool_domains(_conn: Connection, _ctx: AuditToolContext, _args: dict[str, Any]) -> dict[str, Any]: + from .registry import list_domains_catalog, tools_catalog_by_domain + + catalog = list_domains_catalog() + by_domain = tools_catalog_by_domain() + return { + "domains": catalog, + "domain_tool_counts": {d: len(by_domain.get(d) or []) for d in by_domain}, + } + + +def _dispatch(name: str, conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + from .registry import dispatch_tool + + return dispatch_tool(name, args, context=ctx, conn=conn) + + +def run_insight_workflow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + wf_type = str(args.get("type") or "priorities").strip().lower() + base = {"property_id": scoped.property_id, "report_id": scoped.report_id} + steps: list[dict[str, Any]] = [] + + if wf_type in ("traffic", "health"): + r = _dispatch("get_traffic_health_check", conn, scoped, base) + steps.append({"tool": "get_traffic_health_check", "result": r}) + elif wf_type in ("landing_pages", "landing"): + r = _dispatch("get_landing_page_blended_table", conn, scoped, {**base, "limit": args.get("limit") or 30}) + steps.append({"tool": "get_landing_page_blended_table", "result": r}) + r2 = _dispatch("get_opportunity_matrix", conn, scoped, {**base, "limit": args.get("limit") or 30}) + steps.append({"tool": "get_opportunity_matrix", "result": r2}) + else: + r = _dispatch("get_opportunity_matrix", conn, scoped, {**base, "limit": args.get("limit") or 30}) + steps.append({"tool": "get_opportunity_matrix", "result": r}) + r2 = _dispatch("get_issue_to_traffic_map", conn, scoped, {**base, "limit": args.get("limit") or 20}) + steps.append({"tool": "get_issue_to_traffic_map", "result": r2}) + + return {"workflow": "insight", "type": wf_type, "steps": steps} + + +def run_technical_workflow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + base = {"property_id": scoped.property_id, "report_id": scoped.report_id} + steps = [ + {"tool": "get_report_summary", "result": _dispatch("get_report_summary", conn, scoped, base)}, + {"tool": "get_critical_issues", "result": _dispatch("get_critical_issues", conn, scoped, base)}, + {"tool": "get_issue_priority_breakdown", "result": _dispatch("get_issue_priority_breakdown", conn, scoped, base)}, + ] + baseline = args.get("baseline_report_id") + if baseline is not None: + steps.append({ + "tool": "compare_issue_deltas", + "result": _dispatch("compare_issue_deltas", conn, scoped, { + **base, + "baseline_report_id": baseline, + }), + }) + return {"workflow": "technical", "steps": steps} + + +def run_keyword_workflow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required"} + base = {"property_id": scoped.property_id, "limit": args.get("limit") or 20} + steps = [] + for tool_name in ("get_brand_keyword_split", "get_striking_distance_keywords", "list_keywords_ctr_opportunity"): + steps.append({"tool": tool_name, "result": _dispatch(tool_name, conn, scoped, base)}) + return {"workflow": "keyword", "steps": steps} + + +def run_domain_agent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Run a short scripted sequence of tools within one domain (subagent-style).""" + from .registry import search_tools, tool_names_for_domain, tool_meta + + scoped = ctx.with_args(args) + task = str(args.get("task") or "").strip() + domain = str(args.get("domain") or "").strip().lower() + max_steps = parse_limit(args.get("max_steps"), 5, 8) + if not task: + return {"error": "task is required"} + + meta = tool_meta() + if domain: + pool = set(tool_names_for_domain(domain)) + else: + pool = set(meta.keys()) + + matches = search_tools(task, limit=max_steps * 2) + picked: list[str] = [] + for m in matches: + name = m["name"] + if name in pool and name not in picked: + picked.append(name) + if len(picked) >= max_steps: + break + + if not picked: + for m in matches: + name = m["name"] + if name not in picked and name in meta: + picked.append(name) + if len(picked) >= max_steps: + break + + if not picked and domain: + picked = tool_names_for_domain(domain)[:max_steps] + + base = {"property_id": scoped.property_id, "report_id": scoped.report_id, "limit": 20} + steps = [] + for name in picked: + steps.append({"tool": name, "result": _dispatch(name, conn, scoped, base)}) + + return { + "task": task, + "domain": domain or classify_tool_domain(picked[0]) if picked else "", + "steps": steps, + "tools_used": picked, + } diff --git a/src/website_profiling/tools/audit_tools/tool_catalog.py b/src/website_profiling/tools/audit_tools/tool_catalog.py index f3f106f8..a85e89ef 100644 --- a/src/website_profiling/tools/audit_tools/tool_catalog.py +++ b/src/website_profiling/tools/audit_tools/tool_catalog.py @@ -378,4 +378,27 @@ def _tool(name: str, description: str, properties: dict[str, Any], required: lis _tool("get_bing_index_status", "Bing Webmaster URL info (requires bing_webmaster_api_key).", {"url": _URL, "property_id": _PID}, ["url", "property_id"]), _tool("get_serp_feature_overlay", "Keywords with SERP feature / competition overlay data.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]), _tool("check_ai_citation_presence", "On-site citation readiness estimate for brand/query (no live LLM API).", {"property_id": _PID, "query": {"type": "string"}, "brand": {"type": "string"}}), + # Router / Tier 0 (Cursor-style) + _tool("search_audit_tools", "Search the audit tool catalog by keyword. Returns matching tool names to call next.", {"query": {"type": "string"}, "limit": {"type": "integer", "maximum": 50}}, ["query"]), + _tool("list_tool_domains", "List SEO tool domains with counts and example prompts.", {}), + _tool("get_data_coverage_report", "Report which integrations and optional audit data are populated for a property.", {"property_id": _PID}, ["property_id"]), + _tool("run_insight_workflow", "Run insight workflow: type=traffic|landing_pages|priorities (default).", {"property_id": _PID, "report_id": _RID, "type": {"type": "string"}, "limit": _LIMIT}), + _tool("run_technical_workflow", "Run technical workflow: report summary, critical issues, priority chart; optional baseline compare.", {"property_id": _PID, "report_id": _RID, "baseline_report_id": {"type": "integer"}}), + _tool("run_keyword_workflow", "Run keyword workflow: brand split, striking distance, CTR opportunities.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]), + _tool("run_domain_agent", "Subagent-style: run up to max_steps tools in one domain for a task description.", {"property_id": _PID, "report_id": _RID, "task": {"type": "string"}, "domain": {"type": "string"}, "max_steps": {"type": "integer", "maximum": 8}}, ["task"]), + # Cross-platform insight (Tier 0) + _tool("get_landing_page_blended_table", "GSC clicks + GA4 sessions per landing page with opportunity quadrant.", {"property_id": _PID, "limit": {"type": "integer", "maximum": 100}, "min_impressions": {"type": "integer"}}, ["property_id"]), + _tool("get_opportunity_matrix", "Landing pages grouped by rank vs conversion opportunity quadrants.", {"property_id": _PID, "limit": _LIMIT}, ["property_id"]), + _tool("get_traffic_health_check", "GSC clicks vs GA4 sessions ratio and tracking health diagnosis.", {"property_id": _PID}, ["property_id"]), + _tool("get_landing_page_full_diagnosis", "One URL: GSC+GA4 slice, crawl, issues, Lighthouse, composite score.", {"url": _URL, "property_id": _PID, "report_id": _RID}, ["url"]), + _tool("get_issue_to_traffic_map", "Audit issues ranked by traffic-weighted impact with GSC/GA4 columns.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + # Google dimensions + _tool("get_gsc_daily_trend", "GSC daily clicks/impressions trend from latest fetch.", {"property_id": _PID}, ["property_id"]), + _tool("get_ga4_daily_trend", "GA4 daily sessions trend from latest fetch.", {"property_id": _PID}, ["property_id"]), + _tool("get_ga4_by_device", "GA4 sessions breakdown by device category.", {"property_id": _PID}, ["property_id"]), + _tool("get_ga4_by_channel", "GA4 sessions breakdown by channel.", {"property_id": _PID}, ["property_id"]), + _tool("get_gsc_page_queries", "GSC queries for a single page URL (full by_page blob).", {"url": _URL, "property_id": _PID, "limit": _LIMIT}, ["url"]), + # Keyword dimensions + _tool("get_brand_keyword_split", "Branded vs non-branded keyword counts and samples.", {"property_id": _PID}, ["property_id"]), + _tool("list_keywords_by_intent", "Keywords filtered by intent (informational, commercial, etc.).", {"property_id": _PID, "intent": {"type": "string"}, "limit": _LIMIT}, ["property_id", "intent"]), ] diff --git a/src/website_profiling/tools/audit_tools/tool_domains.py b/src/website_profiling/tools/audit_tools/tool_domains.py new file mode 100644 index 00000000..aecb51b4 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/tool_domains.py @@ -0,0 +1,308 @@ +"""Explicit tool domain/tier metadata for routing and MCP domain servers.""" +from __future__ import annotations + +from typing import Any + +CANONICAL_DOMAINS: tuple[str, ...] = ( + "core", + "portfolio", + "issues", + "crawl", + "onpage", + "schema", + "links", + "indexation", + "content", + "keywords", + "google", + "backlinks", + "performance", + "drift", + "security", + "ops", + "export", + "images", + "geo", + "accessibility", + "assets", + "ctr", + "integrations", + "insight", +) + +# Tier 0 — always included in chat dynamic routing (router + top insight tools). +TIER_0_TOOLS: frozenset[str] = frozenset({ + "search_audit_tools", + "list_tool_domains", + "get_data_coverage_report", + "run_insight_workflow", + "run_technical_workflow", + "run_keyword_workflow", + "run_domain_agent", + "get_report_summary", + "list_top_impact_issues", + "prioritize_fix_roadmap", + "get_landing_page_blended_table", + "get_opportunity_matrix", + "get_traffic_health_check", + "get_landing_page_full_diagnosis", + "get_issue_to_traffic_map", + "get_google_summary", +}) + +# Explicit domain overrides (name -> domain). +_DOMAIN_OVERRIDES: dict[str, str] = { + "search_audit_tools": "core", + "list_tool_domains": "core", + "get_data_coverage_report": "core", + "run_insight_workflow": "core", + "run_technical_workflow": "core", + "run_keyword_workflow": "core", + "run_domain_agent": "core", + "get_landing_page_blended_table": "insight", + "get_opportunity_matrix": "insight", + "get_traffic_health_check": "insight", + "get_landing_page_full_diagnosis": "insight", + "get_issue_to_traffic_map": "insight", + "get_gsc_daily_trend": "google", + "get_ga4_daily_trend": "google", + "get_ga4_by_device": "google", + "get_ga4_by_channel": "google", + "get_brand_keyword_split": "keywords", + "list_keywords_by_intent": "keywords", + "get_gsc_page_queries": "google", + "list_broken_links": "links", + "list_broken_link_sources": "links", + "get_gsc_sample_links": "backlinks", + "get_gsc_latest_links": "backlinks", + "get_gsc_links_summary": "backlinks", + "get_gsc_links_import_status": "backlinks", + "list_seo_onpage_issues": "onpage", + "list_content_url_issues": "onpage", + "list_pages_missing_title": "onpage", + "list_pages_missing_h1": "onpage", + "list_pages_multiple_h1": "onpage", + "list_pages_missing_meta_description": "onpage", + "list_pages_meta_desc_too_short": "onpage", + "list_pages_meta_desc_too_long": "onpage", + "list_pages_noindex": "onpage", + "list_pages_missing_canonical": "onpage", + "list_canonical_mismatch": "onpage", + "list_pages_with_missing_alt": "onpage", + "list_pages_skipped_headings": "onpage", + "list_pages_missing_viewport": "onpage", + "list_pages_missing_og_image": "onpage", + "get_report_summary": "portfolio", + "get_critical_issues": "issues", + "get_issue_priority_breakdown": "issues", + "list_top_impact_issues": "issues", + "prioritize_fix_roadmap": "issues", + "get_google_summary": "google", + "get_gsc_ctr_opportunity_pages": "ctr", + "list_keywords_ctr_opportunity": "ctr", + "analyze_serp_snippet_for_url": "ctr", + "compare_reports": "drift", +} + +_ONPAGE_PREFIXES = ( + "list_pages_missing_", + "list_pages_meta_desc_", + "list_pages_multiple_h1", + "list_pages_noindex", + "list_seo_onpage", + "list_content_url", +) + +_INSIGHT_PREFIXES = ( + "get_landing_page_", + "get_opportunity_", + "get_traffic_health", + "get_issue_to_traffic", +) + +# MCP server bundles (WP_MCP_DOMAIN env). +MCP_DOMAIN_BUNDLES: dict[str, frozenset[str]] = { + "core": frozenset({"core", "insight"}), + "crawl": frozenset({"crawl", "onpage", "schema", "accessibility", "assets"}), + "google": frozenset({"google", "insight", "ctr", "keywords", "integrations"}), + "links": frozenset({"links", "backlinks", "indexation"}), + "full": frozenset(CANONICAL_DOMAINS), +} + +DOMAIN_EXAMPLE_PROMPTS: dict[str, str] = { + "core": "What data do we have? Search for a specialized tool.", + "portfolio": "Give me an audit overview and health score.", + "issues": "What are the top critical issues to fix first?", + "crawl": "List 4xx pages and redirect chains.", + "onpage": "Which pages are missing title tags or meta descriptions?", + "google": "Top GSC queries and GA4 landing page performance.", + "insight": "High-click pages with low engagement — opportunity matrix.", + "drift": "Compare this audit to the previous report.", + "export": "Export the audit as PDF.", + "keywords": "Striking-distance keywords and CTR opportunities.", + "performance": "Lighthouse summary and Core Web Vitals failures.", + "links": "Orphan pages and broken internal links.", + "backlinks": "GSC backlinks sample and velocity.", + "images": "Image audit summary and largest unoptimized images.", + "geo": "GEO readiness score and llms.txt status.", +} + + +def classify_tool_domain(name: str) -> str: + """Return canonical domain for a tool name.""" + if name in _DOMAIN_OVERRIDES: + return _DOMAIN_OVERRIDES[name] + if name.startswith("compare_"): + return "drift" + if name in TIER_0_TOOLS: + return _DOMAIN_OVERRIDES.get(name, "core") + + if name.startswith("export_") or name in ("compose_custom_report", "list_export_formats"): + return "export" + if name.startswith(( + "get_image_", "list_pages_without_lazy", "list_pages_with_images_missing", + "list_site_image", "list_lighthouse_image", "list_largest_images", + "list_unoptimized_images", "list_images_needing", + )): + return "images" + if name.startswith(( + "get_landing_page_", "get_opportunity_", "get_traffic_health", "get_issue_to_traffic", + )): + return "insight" + if name.startswith(("get_geo_", "get_aeo_", "get_llms_", "get_eeat_", "get_faq_", + "list_pages_missing_faq", "draft_llms", "check_ai_citation")): + return "geo" + if "axe" in name or "mixed_content" in name or name == "get_heading_outline_for_url": + return "accessibility" + if name in ( + "get_asset_weight_summary", "get_readability_summary", "list_heavy_pages_by_bytes", + "list_pages_poor_cache_headers", "list_pages_low_content_ratio", + ): + return "assets" + if "ctr" in name or name in ("list_keywords_ctr_opportunity", "analyze_serp_snippet_for_url"): + return "ctr" + if name in ("get_gsc_url_inspection", "get_gsc_index_coverage", "get_bing_index_status", "get_serp_feature_overlay"): + return "integrations" + if any(name.startswith(p) for p in _ONPAGE_PREFIXES): + return "onpage" + if name.startswith(( + "list_propert", "get_propert", "get_report", "get_executive", "get_site", "list_report", + "get_portfolio", + )) or name in ( + "get_ads_txt_status", "get_security_txt_status", "get_contact_intelligence", + "get_rich_results_summary", "list_rich_results_failures", "get_competitor_keyword_gap", + "get_pagination_audit_summary", "get_portfolio_benchmark", + ): + return "portfolio" + if name in ( + "list_top_impact_issues", "prioritize_fix_roadmap", "generate_issue_fix", + "summarize_category_for_client", + ) or "issue" in name or "category" in name or "workflow" in name: + return "issues" + if name.startswith(( + "list_pages_", "list_canonical", "list_long_", "list_robots_", "get_top_pages_by", + "search_pages", "get_page_", "list_redirects", "list_broken", "list_status_", + "get_status_code", "get_response_time", "get_depth", "get_crawl_", "get_browser", + "list_pages_with", "list_pages_by", "list_pages_soft", "list_pages_poor", + "list_dead_end", "list_duplicate_title", "list_heavy_pages", + )): + return "crawl" + if "schema" in name or name == "get_seo_health": + return "schema" + if "orphan" in name or "link" in name or "fingerprint" in name or "pagerank" in name: + return "links" + if "indexation" in name or "hreflang" in name or "language" in name or name == "list_subdomains": + return "indexation" + if "content" in name or "social" in name or "ner" in name or "thin" in name or "opportunit" in name or "duplicate" in name: + return "content" + if "keyword" in name or "cannibal" in name or "misalignment" in name or "striking" in name or "semantic" in name or name in ("expand_keywords", "generate_content_brief"): + return "keywords" + if "google" in name or "gsc" in name or "ga4" in name: + return "google" + if "backlink" in name or "competitor" in name or "bing" in name or "gsc_links" in name: + return "backlinks" + if "lighthouse" in name or "crux" in name or "slow" in name or "cwv" in name: + return "performance" + if "health" in name or "compare" in name or "alert" in name or "tech_stack" in name or name == "list_pages_by_technology": + return "drift" + if "security" in name: + return "security" + if "log" in name or name in ("get_property_ops", "list_crawl_runs", "list_log_uploads", "get_page_coach"): + return "ops" + return "portfolio" + + +def _tags_for_tool(name: str, domain: str) -> list[str]: + tags = [domain] + parts = name.replace("_", " ").split() + tags.extend(p for p in parts if len(p) > 2 and p not in tags) + return tags[:8] + + +def build_tool_meta(tool_names: set[str] | frozenset[str]) -> dict[str, dict[str, Any]]: + """Build TOOL_META for all registered tool names.""" + meta: dict[str, dict[str, Any]] = {} + for name in sorted(tool_names): + domain = classify_tool_domain(name) + tier = 0 if name in TIER_0_TOOLS else 1 + meta[name] = { + "domain": domain, + "tier": tier, + "tags": _tags_for_tool(name, domain), + } + return meta + + +def tools_by_domain(meta: dict[str, dict[str, Any]]) -> dict[str, list[str]]: + out: dict[str, list[str]] = {d: [] for d in CANONICAL_DOMAINS} + for name, info in meta.items(): + domain = str(info.get("domain") or "portfolio") + if domain not in out: + out[domain] = [] + out[domain].append(name) + for domain in out: + out[domain].sort() + return out + + +def tool_names_for_domain(meta: dict[str, dict[str, Any]], domain: str) -> list[str]: + by_domain = tools_by_domain(meta) + return list(by_domain.get(domain) or []) + + +def tool_names_for_tier(meta: dict[str, dict[str, Any]], tier: int) -> list[str]: + return sorted(name for name, info in meta.items() if info.get("tier") == tier) + + +def tool_names_for_mcp_bundle(meta: dict[str, dict[str, Any]], bundle: str) -> set[str]: + """Return tool names exposed for an MCP domain bundle.""" + bundle_key = (bundle or "core").strip().lower() + allowed_domains = MCP_DOMAIN_BUNDLES.get(bundle_key, MCP_DOMAIN_BUNDLES["core"]) + if bundle_key == "full": + return set(meta.keys()) + names: set[str] = set() + by_domain = tools_by_domain(meta) + for domain in allowed_domains: + names.update(by_domain.get(domain) or []) + if bundle_key == "core": + names.update(TIER_0_TOOLS & set(meta.keys())) + return names + + +def domains_catalog(meta: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + by_domain = tools_by_domain(meta) + rows: list[dict[str, Any]] = [] + for domain in CANONICAL_DOMAINS: + tools = by_domain.get(domain) or [] + if not tools: + continue + rows.append({ + "domain": domain, + "tool_count": len(tools), + "example_prompt": DOMAIN_EXAMPLE_PROMPTS.get(domain, ""), + "mcp_bundle": next( + (b for b, domains in MCP_DOMAIN_BUNDLES.items() if domain in domains and b != "full"), + "full", + ), + }) + return rows diff --git a/src/website_profiling/tools/audit_tools/tool_selector.py b/src/website_profiling/tools/audit_tools/tool_selector.py new file mode 100644 index 00000000..69b87c0b --- /dev/null +++ b/src/website_profiling/tools/audit_tools/tool_selector.py @@ -0,0 +1,147 @@ +"""Dynamic tool selection for chat agent (Cursor-style subset loading).""" +from __future__ import annotations + +import os +import re +from typing import Any + +from .tool_domains import TIER_0_TOOLS +from .registry import tier0_tool_names, tool_meta, tool_names_for_domain + + +DOMAIN_KEYWORDS: dict[str, tuple[str, ...]] = { + "issues": ("issue", "issues", "critical issues", "fix", "priority", "roadmap", "impact"), + "crawl": ("crawl", "404", "500", "redirect", "status code", "orphan", "soft 404", "robots"), + "onpage": ("title tag", "meta description", "h1", "canonical", "noindex", "on-page", "onpage"), + "google": ("gsc", "search console", "ga4", "analytics", "clicks", "impressions", "queries"), + "insight": ("opportunity", "engagement", "landing page", "blended", "traffic health", "diagnosis"), + "keywords": ("keyword", "striking", "cannibal", "brand", "intent"), + "performance": ("lighthouse", "cwv", "core web vitals", "slow page", "crux", "page speed"), + "links": ("broken link", "internal link", "inlink", "outlink", "anchor text", "pagerank"), + "backlinks": ("backlink", "referring domain", "gsc links", "moz", "majestic"), + "drift": ("compare", "baseline", "delta", "history", "trend", "drift"), + "export": ("export", "pdf", "csv", "download"), + "images": ("image", "alt text", "lazy load", "webp", "lcp image"), + "geo": ("geo", "aeo", "llms.txt", "faq schema", "eeat"), + "accessibility": ("axe", "accessibility", "a11y", "mixed content"), + "security": ("security", "tls", "hsts", "ssl"), + "indexation": ("indexation", "sitemap", "hreflang", "indexed"), + "content": ("duplicate content", "thin content", "word count", "readability"), + "ops": ("access log", "log analysis", "log upload", "crawl run", "integration status"), + "portfolio": ("overview", "health score", "category scores", "executive", "portfolio", "audit summary"), + "ctr": ("ctr", "snippet", "title meta ctr"), +} + +# High-value tools referenced in the chat system prompt playbooks — pinned when domain matches. +PLAYBOOK_ANCHORS: dict[str, tuple[str, ...]] = { + "images": ("get_image_audit_summary",), + "export": ("export_audit_report", "export_list_as_csv"), + "issues": ("get_critical_issues", "get_issue_priority_breakdown", "list_issues"), + "portfolio": ("get_category_scores", "list_audit_categories"), + "performance": ("get_lighthouse_summary",), + "drift": ("compare_reports", "compare_issue_deltas"), + "google": ("get_gsc_top_queries", "get_ga4_page_metrics"), +} + + +def chat_tool_mode() -> str: + return (os.environ.get("CHAT_TOOL_MODE") or "dynamic").strip().lower() + + +def chat_tool_max() -> int: + floor = len(TIER_0_TOOLS) + 1 + try: + return max(floor, min(int(os.environ.get("CHAT_TOOL_MAX") or 45), 120)) + except (TypeError, ValueError): + return max(floor, 45) + + +def chat_tool_search_cap() -> int: + """Soft cap after search/domain-agent expansion (Tier 0 + pinned results).""" + return min(chat_tool_max() + 15, 75) + + +def _keyword_in_text(keyword: str, text: str) -> bool: + if " " in keyword: + return keyword in text + return re.search(rf"\b{re.escape(keyword)}\b", text) is not None + + +def _score_domains(text: str) -> list[tuple[int, str]]: + lower = text.lower() + scores: list[tuple[int, str]] = [] + for domain, keywords in DOMAIN_KEYWORDS.items(): + score = sum(3 if _keyword_in_text(kw, lower) else 0 for kw in keywords) + if score > 0: + scores.append((score, domain)) + scores.sort(key=lambda x: (-x[0], x[1])) + return scores + + +def apply_tool_cap( + selected: set[str], + cap: int, + *, + pinned: set[str] | None = None, + max_pinned: int = 12, +) -> set[str]: + """Trim tool set while preserving Tier 0 and optionally pinned names.""" + pinned = pinned or set() + tier0 = set(TIER_0_TOOLS) & selected + pinned_keep = sorted(pinned & selected)[: max(0, max_pinned)] + must_keep = tier0 | set(pinned_keep) + if len(selected) <= cap: + return selected + rest = sorted(selected - must_keep) + room = max(0, cap - len(must_keep)) + return must_keep | set(rest[:room]) + + +def select_tools_for_turn( + user_message: str, + history: list[dict[str, Any]] | None = None, + *, + max_tools: int | None = None, + extra_names: set[str] | None = None, +) -> set[str]: + """Return tool names to expose to the LLM this turn (Tier 0 + relevant Tier 1).""" + if chat_tool_mode() == "full": + from .registry import tool_handler_names + return tool_handler_names() + + cap = max_tools if max_tools is not None else chat_tool_max() + selected: set[str] = set(tier0_tool_names()) + if extra_names: + selected |= extra_names + + texts = [user_message or ""] + if history: + for msg in reversed(history): + if msg.get("role") == "user": + prior = str(msg.get("content") or "") + if prior and prior != (user_message or ""): + texts.append(prior) + break + combined = " ".join(texts) + domain_scores = _score_domains(combined) + scored_domains = [domain for _, domain in domain_scores[:4]] + + meta = tool_meta() + if not domain_scores: + for fallback in ("portfolio", "issues", "insight"): + selected.update(tool_names_for_domain(fallback)) + else: + for domain in scored_domains: + selected.update(tool_names_for_domain(domain)) + for anchor in PLAYBOOK_ANCHORS.get(domain, ()): + if anchor in meta: + selected.add(anchor) + + selected = apply_tool_cap(selected, cap) + selected = {n for n in selected if n in meta or n in tier0_tool_names()} + return selected + + +def compact_tool_list(names: set[str]) -> str: + lines = sorted(names) + return "\n".join(f"- {n}" for n in lines) diff --git a/tests/test_audit_tools_expanded.py b/tests/test_audit_tools_expanded.py index a0ffdcc7..8cc772e8 100644 --- a/tests/test_audit_tools_expanded.py +++ b/tests/test_audit_tools_expanded.py @@ -178,7 +178,7 @@ def conn() -> MagicMock: def test_handler_schema_parity() -> None: names = {t["name"] for t in TOOL_DEFINITIONS} assert names == tool_handler_names() - assert len(TOOL_DEFINITIONS) == 221 + assert len(TOOL_DEFINITIONS) == 240 def test_slice_helpers() -> None: diff --git a/tests/test_google_store_full.py b/tests/test_google_store_full.py new file mode 100644 index 00000000..6c6c802a --- /dev/null +++ b/tests/test_google_store_full.py @@ -0,0 +1,22 @@ +"""Tests for full Google data blob reader.""" +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +from website_profiling.integrations.google.store import read_google_data_full + + +def test_read_google_data_full_returns_blob() -> None: + blob = {"gsc_full": {"by_page": []}, "ga4_full": {"by_path": []}} + conn = MagicMock() + conn.execute.return_value.fetchone.return_value = {"data": blob} + result = read_google_data_full(conn, property_id=1) + assert result == blob + + +def test_read_google_data_full_none_when_missing() -> None: + conn = MagicMock() + conn.execute.return_value.fetchone.return_value = None + result = read_google_data_full(conn, property_id=1) + assert result is None diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py index 67d8fded..4fa53225 100644 --- a/tests/test_mcp_registry.py +++ b/tests/test_mcp_registry.py @@ -3,11 +3,17 @@ from unittest.mock import MagicMock, patch -from website_profiling.tools.audit_tools.registry import TOOL_DEFINITIONS, dispatch_tool +from website_profiling.tools.audit_tools.registry import ( + TOOL_DEFINITIONS, + dispatch_tool, + search_tools, + validate_tool_registry, +) +from website_profiling.tools.audit_tools.tool_domains import TIER_0_TOOLS def test_tool_definitions_schema() -> None: - assert len(TOOL_DEFINITIONS) == 221 + assert len(TOOL_DEFINITIONS) == 240 for tool in TOOL_DEFINITIONS: assert tool.get("name") assert tool.get("description") @@ -16,6 +22,22 @@ def test_tool_definitions_schema() -> None: assert schema.get("type") == "object" +def test_validate_tool_registry() -> None: + assert validate_tool_registry() == [] + + +def test_tier0_tools_have_handlers() -> None: + from website_profiling.tools.audit_tools.registry import tool_handler_names + + assert TIER_0_TOOLS <= tool_handler_names() + + +def test_search_tools_finds_broken_links() -> None: + matches = search_tools("broken links", limit=5) + names = {m["name"] for m in matches} + assert "list_broken_links" in names or "list_broken_link_sources" in names + + def test_dispatch_list_properties_roundtrip() -> None: conn = MagicMock() props = [{"id": 1, "name": "ex.com", "canonical_domain": "ex.com"}] diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py index d092518b..eddb9f76 100644 --- a/tests/test_mcp_server_helpers.py +++ b/tests/test_mcp_server_helpers.py @@ -60,23 +60,26 @@ def test_read_glossary_excerpt_missing(monkeypatch) -> None: def test_tools_catalog_json_includes_security_tools() -> None: - catalog = json.loads(mcp_server._tools_catalog_json()) - assert catalog["tool_count"] >= 221 + with patch.dict(os.environ, {"WP_MCP_DOMAIN": "full"}): + catalog = json.loads(mcp_server._tools_catalog_json()) + assert catalog["tool_count"] >= 240 assert "get_security_findings" in catalog["domains"]["security"] assert "get_geo_readiness_score" in catalog["domains"]["geo"] assert "get_gsc_url_inspection" in catalog["domains"]["integrations"] + assert catalog["mcp_domain"] == "full" def test_tools_catalog_json_backlinks_domain() -> None: - fake_tools = [ - { - "name": "get_bing_overview", - "description": "Bing overview without link in name.", - "inputSchema": {"type": "object", "properties": {}}, - }, - ] - with patch("website_profiling.mcp.server.TOOL_DEFINITIONS", fake_tools): - catalog = json.loads(mcp_server._tools_catalog_json()) + with patch.dict(os.environ, {"WP_MCP_DOMAIN": "full"}): + with patch( + "website_profiling.mcp.server.mcp_tool_names", + return_value={"get_bing_overview"}, + ): + with patch( + "website_profiling.mcp.server.tools_catalog_by_domain", + return_value={"backlinks": ["get_bing_overview"]}, + ): + catalog = json.loads(mcp_server._tools_catalog_json()) assert catalog["domains"]["backlinks"] == ["get_bing_overview"] @@ -164,15 +167,16 @@ async def __aexit__(self, *_args): monkeypatch.setitem(sys.modules, "mcp.server.stdio", fake_stdio_mod) monkeypatch.setitem(sys.modules, "mcp.types", fake_types_mod) - with patch.dict(os.environ, {"WP_PROPERTY_ID": "7"}, clear=False): + with patch.dict(os.environ, {"WP_PROPERTY_ID": "7", "WP_MCP_DOMAIN": "full"}, clear=False): mcp_server.main() - assert captured["name"] == "site-audit" + assert captured["name"] == "site-audit-full" assert captured["ran"] is True tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type] - assert len(tools) >= 221 + assert len(tools) >= 240 resources = asyncio.run(captured["list_resources"]()) # type: ignore[arg-type] assert any(r["uri"] == "audit://property/7" for r in resources) + assert any(r["uri"] == "audit://domains" for r in resources) with patch("website_profiling.mcp.server.dispatch_tool", return_value={"ok": True}): content = asyncio.run(captured["call_tool"]("list_properties", {"property_id": 1})) # type: ignore[arg-type] @@ -181,6 +185,69 @@ async def __aexit__(self, *_args): assert read_text.startswith("{") +def test_mcp_call_tool_rejects_tools_outside_domain(monkeypatch) -> None: + captured: dict[str, object] = {} + + class FakeServer: + def __init__(self, name: str) -> None: + captured["name"] = name + + def list_tools(self): + def decorator(fn): + captured["list_tools"] = fn + return fn + return decorator + + def call_tool(self): + def decorator(fn): + captured["call_tool"] = fn + return fn + return decorator + + def list_resources(self): + def decorator(fn): + return fn + return decorator + + def read_resource(self): + def decorator(fn): + return fn + return decorator + + def create_initialization_options(self): + return {} + + async def run(self, *_args, **_kwargs) -> None: + return None + + class FakeStdioCM: + async def __aenter__(self): + return (MagicMock(), MagicMock()) + + async def __aexit__(self, *_args): + return False + + fake_server_mod = MagicMock() + fake_server_mod.Server = FakeServer + fake_stdio_mod = MagicMock() + fake_stdio_mod.stdio_server = MagicMock(return_value=FakeStdioCM()) + fake_types_mod = MagicMock() + fake_types_mod.TextContent = lambda **kwargs: kwargs + fake_types_mod.Resource = lambda **kwargs: kwargs + fake_types_mod.Tool = lambda **kwargs: kwargs + + monkeypatch.setitem(sys.modules, "mcp", MagicMock()) + monkeypatch.setitem(sys.modules, "mcp.server", fake_server_mod) + monkeypatch.setitem(sys.modules, "mcp.server.stdio", fake_stdio_mod) + monkeypatch.setitem(sys.modules, "mcp.types", fake_types_mod) + + with patch.dict(os.environ, {"WP_MCP_DOMAIN": "core"}, clear=False): + mcp_server.main() + + blocked = asyncio.run(captured["call_tool"]("export_audit_report", {"format": "pdf"})) # type: ignore[arg-type] + assert "not exposed" in blocked[0]["text"] + + def test_mcp_package_main(monkeypatch) -> None: with patch("website_profiling.mcp.server.main") as mock_main: runpy.run_module("website_profiling.mcp", run_name="__main__") diff --git a/tests/test_router_tools.py b/tests/test_router_tools.py new file mode 100644 index 00000000..61c4187f --- /dev/null +++ b/tests/test_router_tools.py @@ -0,0 +1,31 @@ +"""Tests for run_domain_agent fallback behavior.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from website_profiling.tools.audit_tools import AuditToolContext +from website_profiling.tools.audit_tools.router_tools import run_domain_agent + + +def test_run_domain_agent_falls_back_to_global_search() -> None: + conn = MagicMock() + ctx = AuditToolContext(property_id=1) + fake_matches = [ + {"name": "list_broken_links", "description": "", "domain": "links", "tier": 1}, + {"name": "get_schema_coverage", "description": "", "domain": "schema", "tier": 1}, + ] + with patch( + "website_profiling.tools.audit_tools.registry.search_tools", + return_value=fake_matches, + ): + with patch( + "website_profiling.tools.audit_tools.registry.tool_names_for_domain", + return_value=["get_unrelated_tool"], + ): + result = run_domain_agent(conn, ctx, { + "task": "broken links audit", + "domain": "schema", + "max_steps": 2, + }) + + assert result["tools_used"] == ["list_broken_links", "get_schema_coverage"] diff --git a/tests/test_tool_selector.py b/tests/test_tool_selector.py new file mode 100644 index 00000000..8378d77e --- /dev/null +++ b/tests/test_tool_selector.py @@ -0,0 +1,101 @@ +"""Tests for dynamic chat tool selection.""" +from __future__ import annotations + +import os +from unittest.mock import patch + +from website_profiling.tools.audit_tools.registry import mcp_tool_names, tier0_tool_names, tool_handler_names +from website_profiling.tools.audit_tools.tool_selector import chat_tool_max, select_tools_for_turn + + +def test_select_tools_always_includes_tier0() -> None: + names = select_tools_for_turn("hello") + assert tier0_tool_names() <= names + assert len(names) <= chat_tool_max() + + +def test_select_tools_google_domain_boost() -> None: + names = select_tools_for_turn("Show me GSC clicks and GA4 landing pages") + assert "get_google_summary" in names or "get_gsc_top_queries" in names + + +def test_select_tools_full_mode() -> None: + with patch.dict(os.environ, {"CHAT_TOOL_MODE": "full"}): + names = select_tools_for_turn("anything") + assert names == tool_handler_names() + + +def test_select_tools_broken_links_in_subset() -> None: + names = select_tools_for_turn("list broken internal links") + assert "list_broken_links" in names + + +def test_broken_link_tools_classified_as_links() -> None: + from website_profiling.tools.audit_tools.tool_domains import classify_tool_domain + + assert classify_tool_domain("list_broken_links") == "links" + assert classify_tool_domain("list_broken_link_sources") == "links" + + +def test_mcp_core_includes_tier0_tools() -> None: + core = mcp_tool_names("core") + assert tier0_tool_names() <= core + + +def test_compare_tools_classified_as_drift() -> None: + from website_profiling.tools.audit_tools.tool_domains import classify_tool_domain + + assert classify_tool_domain("compare_issue_deltas") == "drift" + assert classify_tool_domain("compare_reports") == "drift" + + +def test_audit_report_overview_prefers_portfolio_not_export() -> None: + names = select_tools_for_turn("show me the audit report overview") + assert "get_report_summary" in names + assert "export_audit_report" not in names + + +def test_compare_prompt_loads_compare_issue_deltas() -> None: + names = select_tools_for_turn("compare issue deltas since last crawl") + assert "compare_issue_deltas" in names + + +def test_playbook_anchors_load_specialized_tools() -> None: + assert "get_image_audit_summary" in select_tools_for_turn("image alt audit") + assert "export_audit_report" in select_tools_for_turn("export as pdf") + assert "get_category_scores" in select_tools_for_turn("show category scores") + assert "get_critical_issues" in select_tools_for_turn("top critical issues") + assert "get_lighthouse_summary" in select_tools_for_turn("lighthouse scores") + + +def test_catalog_does_not_match_ops_domain() -> None: + from website_profiling.tools.audit_tools.tool_selector import _score_domains + + assert _score_domains("show catalog") == [] + + +def test_search_expansion_applies_soft_cap() -> None: + from website_profiling.llm.agent import _expand_active_tools_from_result + from website_profiling.tools.audit_tools.tool_selector import chat_tool_search_cap, tier0_tool_names + + active = tier0_tool_names() + many = [f"tool_{i}" for i in range(100)] + expanded = _expand_active_tools_from_result( + "search_audit_tools", + {"tool_names": many}, + active, + ) + assert len(expanded) <= chat_tool_search_cap() + + +def test_search_audit_tools_expansion_names() -> None: + from website_profiling.llm.agent import _expand_active_tools_from_result + + active = tier0_tool_names() + expanded = _expand_active_tools_from_result( + "search_audit_tools", + {"tool_names": ["list_broken_links", "get_schema_coverage"]}, + active, + ) + assert "list_broken_links" in expanded + assert "get_schema_coverage" in expanded From be646bc425965124590e3b2e608e06a4435c2a17 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 13 Jun 2026 14:31:33 +0530 Subject: [PATCH 5/5] hello --- .github/workflows/ci.yml | 6 +- AGENT.md | 2 +- README.md | 6 +- docs/MCP.md | 6 +- scripts/local-test.ps1 | 1 + scripts/local-test.sh | 4 + src/website_profiling/analysis/log_parser.py | 9 + .../integrations/google/keyword_store.py | 29 + .../integrations/google/store.py | 74 + src/website_profiling/reporting/builder.py | 144 ++ .../tools/audit_tools/backlink_lists.py | 133 ++ .../tools/audit_tools/compare_list_tools.py | 174 ++ .../tools/audit_tools/content_lists.py | 249 +++ .../tools/audit_tools/context.py | 16 +- .../tools/audit_tools/crawl.py | 119 +- .../tools/audit_tools/data_coverage.py | 38 + .../tools/audit_tools/export_tools.py | 91 + .../tools/audit_tools/geo_list_tools.py | 242 +++ .../tools/audit_tools/google_lists.py | 474 +++++ .../tools/audit_tools/indexation_lists.py | 345 ++++ .../tools/audit_tools/insight_helpers.py | 2 +- .../tools/audit_tools/issue_lists.py | 526 +++++ .../tools/audit_tools/keyword_lists.py | 558 ++++++ .../tools/audit_tools/link_lists.py | 208 ++ .../tools/audit_tools/registry.py | 219 +++ .../tools/audit_tools/tool_catalog.py | 102 + .../tools/audit_tools/tool_domains.py | 31 + .../tools/audit_tools/tool_selector.py | 12 +- tests/test_audit_tools_batch100_coverage.py | 1686 +++++++++++++++++ tests/test_audit_tools_expanded.py | 2 +- tests/test_log_parser.py | 8 + tests/test_mcp_registry.py | 2 +- tests/test_mcp_server_helpers.py | 23 +- tests/test_tools_gate100_coverage.py | 556 ++++++ 34 files changed, 6073 insertions(+), 24 deletions(-) create mode 100644 src/website_profiling/tools/audit_tools/backlink_lists.py create mode 100644 src/website_profiling/tools/audit_tools/compare_list_tools.py create mode 100644 src/website_profiling/tools/audit_tools/content_lists.py create mode 100644 src/website_profiling/tools/audit_tools/geo_list_tools.py create mode 100644 src/website_profiling/tools/audit_tools/google_lists.py create mode 100644 src/website_profiling/tools/audit_tools/indexation_lists.py create mode 100644 src/website_profiling/tools/audit_tools/issue_lists.py create mode 100644 src/website_profiling/tools/audit_tools/keyword_lists.py create mode 100644 src/website_profiling/tools/audit_tools/link_lists.py create mode 100644 tests/test_audit_tools_batch100_coverage.py create mode 100644 tests/test_tools_gate100_coverage.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a515a0a0..134057fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,11 +53,13 @@ jobs: tests/test_export_audit_coverage.py tests/test_audit_tools.py tests/test_audit_tools_expanded.py \ tests/test_audit_tools_coverage.py tests/test_audit_tools_dispatch_coverage.py \ tests/test_audit_tools_links_extras.py tests/test_audit_tools_expansion.py \ - tests/test_audit_tools_expansion_coverage.py tests/test_export_custom_coverage.py \ + tests/test_audit_tools_expansion_coverage.py tests/test_audit_tools_batch100_coverage.py tests/test_export_custom_coverage.py \ tests/test_export_artifacts_coverage.py tests/test_export_compare_coverage.py \ tests/test_export_tools_coverage.py tests/test_image_tools.py tests/test_export_custom.py \ tests/test_export_artifacts.py tests/test_export_compare.py tests/test_export_workbook.py \ - tests/test_export_sitemap.py tests/test_mcp_registry.py tests/test_mcp_resources.py \ + tests/test_export_sitemap.py tests/test_mcp_registry.py tests/test_mcp_resources.py \ + tests/test_router_tools.py tests/test_tool_selector.py \ + tests/test_tools_gate100_coverage.py \ tests/test_tools_branch_coverage.py \ --cov=website_profiling.tools --cov-config=.coveragerc.tools \ --cov-report=term-missing --cov-fail-under=100 -q -o addopts= diff --git a/AGENT.md b/AGENT.md index cdb903aa..ec06cbe5 100644 --- a/AGENT.md +++ b/AGENT.md @@ -25,7 +25,7 @@ - **Pipeline data** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) is stored in **PostgreSQL only** — no JSON/CSV/HTML exports from the main pipeline. - **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python), `PGPOOL_MAX` (Node). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch. - **`web/`:** `/api/report/*` (PostgreSQL); `/api/run` spawns Python (localhost only); `/api/crawl/browser-status` GET (localhost, Playwright/Chromium preflight); `/api/pipeline-config` GET/PUT; `/api/llm-config` GET/PUT (AI only); `/api/chat` POST (SSE agent); `/api/chat/sessions` GET/POST; `/api/properties/{id}/google/links/import` POST (GSC Links CSV); `PipelineRunnerFab` saves pipeline + LLM state before each run -- **MCP:** `python -m website_profiling.mcp` (stdio, **240 read-only audit tools**, domain-scoped via `WP_MCP_DOMAIN`). See `docs/MCP.md`. Requires `pip install -r requirements.txt`. +- **MCP:** `python -m website_profiling.mcp` (stdio, **340 read-only audit tools**, domain-scoped via `WP_MCP_DOMAIN`). See `docs/MCP.md`. Requires `pip install -r requirements.txt`. - **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages` tables, migration `012_chat_sessions`). - **Job store:** in-memory on `globalThis` in `web/src/server/pipelineJobs.ts` — job status/log is lost on server restart (single-process dev/Docker only). - **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`** diff --git a/README.md b/README.md index efa69b90..9059ecc6 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Repository: [codefrydev/WebsiteProfiling](https://github.com/codefrydev/WebsiteP -Also included: **AI chat** over audit data (optional), **240 MCP tools** (domain-scoped servers), keyword explorer, backlinks, compare runs, and portfolio management for agencies. +Also included: **AI chat** over audit data (optional), **340 MCP tools** (domain-scoped servers), keyword explorer, backlinks, compare runs, and portfolio management for agencies.

    Site Audit preview @@ -91,7 +91,7 @@ WebsiteProfiling/ │ ├── integrations/ # Google Search Console, GA4, Bing, CrUX │ ├── llm/ # AI enrich + chat agent │ ├── tools/ # Exports, audit query tools, MCP helpers -│ ├── mcp/ # MCP server (240 read-only tools, domain bundles) +│ ├── mcp/ # MCP server (340 read-only tools, domain bundles) │ ├── db/ # PostgreSQL storage layer │ ├── commands/ # CLI subcommands │ ├── cli.py # Pipeline entrypoint @@ -186,7 +186,7 @@ Google Search Console / Analytics: connect via **Integrations** (gear icon) in t | **Ollama** | Local daemon at `http://127.0.0.1:11434`. Chat UI lists installed models plus the live Ollama cloud catalog (billing: free local, account free tier, Pro). Native tool calling when supported; otherwise ReAct fallback. Pick the model in-chat without leaving the page. | | **OpenAI** / **Anthropic** | API key in AI settings; native tool calling with streaming. | -The agent uses the same **240 read-only audit tools** as the MCP server (`docs/MCP.md`), with **dynamic routing** (~45 tools per turn plus router meta-tools). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`). +The agent uses the same **340 read-only audit tools** as the MCP server (`docs/MCP.md`), with **dynamic routing** (~45 tools per turn plus router meta-tools). Responses stream over SSE (`POST /api/chat`) with status, tool activity, and tokens. Sessions are saved per property (`chat_sessions` / `chat_messages`). Production: `docker-compose.prod.yml` (set `POSTGRES_PASSWORD`, `AUTH_SECRET`). diff --git a/docs/MCP.md b/docs/MCP.md index 1ad0edc1..b39b8bab 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -12,7 +12,7 @@ export PYTHONPATH=src ## Cursor configuration -Site Audit exposes **domain-scoped MCP servers** (like Cursor optional plugins). Connect only the bundles you need instead of loading all 240 tools in one server. +Site Audit exposes **domain-scoped MCP servers** (like Cursor optional plugins). Connect only the bundles you need instead of loading all 340 tools in one server. | `WP_MCP_DOMAIN` | Typical tools | Use when | |-----------------|---------------|----------| @@ -20,7 +20,7 @@ Site Audit exposes **domain-scoped MCP servers** (like Cursor optional plugins). | `crawl` | Crawl, on-page, schema, accessibility | Technical crawl audits | | `google` | Google, insight, CTR, keywords | GSC/GA4 analysis | | `links` | Links, backlinks, indexation | Link architecture | -| `full` | All 240 tools | Debugging / legacy single-server setup | +| `full` | All 340 tools | Debugging / legacy single-server setup | Add to `.cursor/mcp.json` (or Cursor MCP settings): @@ -84,7 +84,7 @@ Single-server legacy setup (all tools): | `audit://tools` | Tool catalog for the connected `WP_MCP_DOMAIN` server | | `audit://domains` | Available MCP domain bundles and tool groupings | -## Tools (240 read-only + export) +## Tools (340 read-only + export) ### Router and insight (Tier 0 — `WP_MCP_DOMAIN=core`) diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1 index db9e8dc4..adc865d2 100644 --- a/scripts/local-test.ps1 +++ b/scripts/local-test.ps1 @@ -276,6 +276,7 @@ function Invoke-PytestTools { tests/test_export_sitemap.py ` tests/test_mcp_registry.py ` tests/test_mcp_resources.py ` + tests/test_tools_gate100_coverage.py ` tests/test_tools_branch_coverage.py ` --cov=website_profiling.tools ` --cov-config=.coveragerc.tools ` diff --git a/scripts/local-test.sh b/scripts/local-test.sh index b28d24bc..357f1d53 100755 --- a/scripts/local-test.sh +++ b/scripts/local-test.sh @@ -153,6 +153,7 @@ run_pytest_tools() { tests/test_audit_tools_links_extras.py \ tests/test_audit_tools_expansion.py \ tests/test_audit_tools_expansion_coverage.py \ + tests/test_audit_tools_batch100_coverage.py \ tests/test_export_custom_coverage.py \ tests/test_export_artifacts_coverage.py \ tests/test_export_compare_coverage.py \ @@ -165,6 +166,9 @@ run_pytest_tools() { tests/test_export_sitemap.py \ tests/test_mcp_registry.py \ tests/test_mcp_resources.py \ + tests/test_router_tools.py \ + tests/test_tool_selector.py \ + tests/test_tools_gate100_coverage.py \ tests/test_tools_branch_coverage.py \ --cov=website_profiling.tools \ --cov-config=.coveragerc.tools \ diff --git a/src/website_profiling/analysis/log_parser.py b/src/website_profiling/analysis/log_parser.py index 63999bc0..0ec4d519 100644 --- a/src/website_profiling/analysis/log_parser.py +++ b/src/website_profiling/analysis/log_parser.py @@ -15,6 +15,8 @@ def parse_access_log_lines(lines: list[str]) -> dict[str, Any]: """Return hit counts and URL sets from access log lines.""" url_hits: Counter[str] = Counter() status_hits: Counter[str] = Counter() + paths_5xx: Counter[str] = Counter() + googlebot_path_hits: Counter[str] = Counter() googlebot_hits = 0 parsed_lines = 0 @@ -29,16 +31,23 @@ def parse_access_log_lines(lines: list[str]) -> dict[str, Any]: path, status, ua = m.group(1), m.group(2), m.group(3).lower() url_hits[path] += 1 status_hits[status] += 1 + if status.startswith("5"): + paths_5xx[path] += 1 if "googlebot" in ua: googlebot_hits += 1 + googlebot_path_hits[path] += 1 top_urls = [{"path": p, "hits": c} for p, c in url_hits.most_common(100)] + paths_5xx_rows = [{"path": p, "hits": c} for p, c in paths_5xx.most_common(100)] + googlebot_paths = [{"path": p, "hits": c} for p, c in googlebot_path_hits.most_common(100)] return { "parsed_lines": parsed_lines, "unique_paths": len(url_hits), "googlebot_hits": googlebot_hits, "status_counts": dict(status_hits), "top_paths": top_urls, + "paths_5xx": paths_5xx_rows, + "googlebot_paths": googlebot_paths, } diff --git a/src/website_profiling/integrations/google/keyword_store.py b/src/website_profiling/integrations/google/keyword_store.py index c1d9b71b..6a6b5f4a 100644 --- a/src/website_profiling/integrations/google/keyword_store.py +++ b/src/website_profiling/integrations/google/keyword_store.py @@ -97,6 +97,35 @@ def append_keyword_history( conn.commit() +def read_keyword_snapshots_for_property( + conn: Connection, + property_id: int | None, + *, + limit: int = 2, +) -> list[dict[str, Any]]: + """Return the most recent keyword_data snapshots for rank delta tools.""" + if property_id is None: + return [] + try: + cur = conn.execute( + """ + SELECT fetched_at, data FROM keyword_data + WHERE property_id = %s + ORDER BY id DESC + LIMIT %s + """, + (property_id, max(1, int(limit))), + ) + out: list[dict[str, Any]] = [] + for row in cur.fetchall(): + data = _parse_row_json(row) + if isinstance(data, dict): + out.append({"fetched_at": row["fetched_at"], **data}) + return out + except Exception: + return [] + + def read_keyword_history( conn: Connection, keyword: str, diff --git a/src/website_profiling/integrations/google/store.py b/src/website_profiling/integrations/google/store.py index 79a4f272..ba5fdece 100644 --- a/src/website_profiling/integrations/google/store.py +++ b/src/website_profiling/integrations/google/store.py @@ -89,3 +89,77 @@ def read_google_data_full( return data if isinstance(data, dict) else None except Exception: return None + + +def read_prior_google_snapshot( + conn: Connection, + property_id: int | None = None, + *, + skip: int = 1, +) -> Optional[dict[str, Any]]: + """Return the Nth-most-recent google_data row (skip=1 → prior snapshot).""" + try: + offset = max(0, int(skip)) + if property_id is not None: + cur = conn.execute( + """ + SELECT data FROM google_data + WHERE property_id = %s + ORDER BY id DESC + OFFSET %s LIMIT 1 + """, + (property_id, offset), + ) + else: + cur = conn.execute( + """ + SELECT data FROM google_data + ORDER BY id DESC + OFFSET %s LIMIT 1 + """, + (offset,), + ) + row = cur.fetchone() + if row is None: + return None + data = _parse_row_json(row) + return data if isinstance(data, dict) else None + except Exception: + return None + + +def gsc_row_deltas( + current_rows: list[dict[str, Any]], + prior_rows: list[dict[str, Any]], + *, + key_field: str, +) -> list[dict[str, Any]]: + """Compute click/impression/position deltas keyed by page or query field.""" + prior_by_key: dict[str, dict[str, Any]] = {} + for row in prior_rows: + if not isinstance(row, dict): + continue + key = str(row.get(key_field) or "").strip().lower() + if key: + prior_by_key[key] = row + + deltas: list[dict[str, Any]] = [] + for row in current_rows: + if not isinstance(row, dict): + continue + key = str(row.get(key_field) or "").strip().lower() + if not key: + continue + prior = prior_by_key.get(key) or {} + cur_clicks = float(row.get("clicks") or 0) + pri_clicks = float(prior.get("clicks") or 0) + cur_impr = float(row.get("impressions") or 0) + pri_impr = float(prior.get("impressions") or 0) + cur_pos = float(row.get("position") or row.get("avg_position") or 0) + pri_pos = float(prior.get("position") or prior.get("avg_position") or 0) + out = dict(row) + out["clicks_delta"] = cur_clicks - pri_clicks + out["impressions_delta"] = cur_impr - pri_impr + out["position_delta"] = cur_pos - pri_pos if pri_pos else None + deltas.append(out) + return deltas diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py index b5e0fc75..f0febd25 100644 --- a/src/website_profiling/reporting/builder.py +++ b/src/website_profiling/reporting/builder.py @@ -51,6 +51,8 @@ META_DESC_LEN_MAX, META_DESC_LEN_MIN, THIN_CONTENT_CHARS, + TITLE_LEN_MAX, + TITLE_LEN_MIN, _compute_summary_seo_issues, ) from .site_level import _fetch_site_level @@ -631,6 +633,79 @@ def _bool_col(col): "images_total": int(pd.to_numeric(row.get("images_total"), errors="coerce") or 0), }) + title_short: list[dict[str, Any]] = [] + title_long: list[dict[str, Any]] = [] + if "title" in df.columns: + titles = df["title"].fillna("").astype(str) + tl = titles.str.len() + for i, row in df.iterrows(): + u = row.get("url") + if pd.isna(u) or not u: + continue + u = str(u).strip() + title_str = titles.iloc[i].strip() + n = int(tl.iloc[i]) + if n == 0: + continue + if n < TITLE_LEN_MIN: + title_short.append({"url": u, "title": title_str, "title_length": n}) + elif n > TITLE_LEN_MAX: + title_long.append({"url": u, "title": title_str, "title_length": n}) + + slow_response: list[dict[str, Any]] = [] + if "response_time_ms" in df.columns: + rt = pd.to_numeric(df["response_time_ms"], errors="coerce") + for i, row in df.iterrows(): + ms = rt.iloc[i] + if pd.isna(ms) or float(ms) <= 2000: + continue + u = row.get("url") + if pd.isna(u) or not u: + continue + slow_response.append({"url": str(u).strip(), "response_time_ms": int(ms)}) + + missing_html_lang: list[dict[str, Any]] = [] + invalid_viewport: list[dict[str, Any]] = [] + if "html_lang" in success_df_urls.columns: + for _, row in success_df_urls.iterrows(): + u = row.get("url") + if pd.isna(u) or not u: + continue + lang = str(row.get("html_lang") or "").strip() + if not lang: + missing_html_lang.append({"url": str(u).strip()}) + if "viewport_present" in success_df_urls.columns: + vp = success_df_urls["viewport_present"] + for _, row in success_df_urls.iterrows(): + u = row.get("url") + if pd.isna(u) or not u: + continue + if not bool(row.get("viewport_present")): + invalid_viewport.append({"url": str(u).strip()}) + + high_reading_level: list[dict[str, Any]] = [] + very_thin_content: list[dict[str, Any]] = [] + if "reading_level" in success_df_urls.columns: + rl = pd.to_numeric(success_df_urls["reading_level"], errors="coerce") + for i, row in success_df_urls.iterrows(): + val = rl.loc[i] + if pd.isna(val) or float(val) <= 12: + continue + u = row.get("url") + if pd.isna(u) or not u: + continue + high_reading_level.append({"url": str(u).strip(), "reading_level": float(val)}) + if "word_count" in success_df_urls.columns: + wc = pd.to_numeric(success_df_urls["word_count"], errors="coerce").fillna(0).astype(int) + for i, row in success_df_urls.iterrows(): + w = int(wc.loc[i]) + if w <= 0 or w >= 100: + continue + u = row.get("url") + if pd.isna(u) or not u: + continue + very_thin_content.append({"url": str(u).strip(), "word_count": w}) + content_urls = { "missing_h1": missing_h1, "missing_title": missing_title, @@ -644,6 +719,13 @@ def _bool_col(col): "missing_alt": missing_alt, "missing_lazy": missing_lazy, "missing_dimensions": missing_dimensions, + "title_short": title_short, + "title_long": title_long, + "slow_response": slow_response, + "missing_html_lang": missing_html_lang, + "invalid_viewport": invalid_viewport, + "high_reading_level": high_reading_level, + "very_thin_content": very_thin_content, } emit_progress("report", "content_analytics", message="Building content analytics") @@ -675,6 +757,65 @@ def _bool_col(col): depth_distribution = _build_depth_distribution(df) image_inventory, image_inventory_summary = _build_image_inventory(links, config) + hreflang_issue_urls: list[dict[str, Any]] = [] + try: + from .categories._helpers import _hreflang_issues + + for issue in _hreflang_issues(success_df_urls if len(success_df_urls) else df): + hreflang_issue_urls.append({ + "url": issue.get("url") or "", + "message": issue.get("message") or "", + "priority": issue.get("priority") or "Medium", + }) + except Exception: + hreflang_issue_urls = [] + + lighthouse_failure_urls: dict[str, list[dict[str, Any]]] = { + "lcp": [], "inp": [], "cls": [], "seo": [], + } + if lighthouse_by_url: + audit_map = { + "lcp": "largest-contentful-paint", + "inp": "interaction-to-next-paint", + "cls": "cumulative-layout-shift", + "seo": "seo", + } + for url, lh in lighthouse_by_url.items(): + if not isinstance(lh, dict): + continue + audits = lh.get("audits") if isinstance(lh.get("audits"), dict) else {} + for bucket, audit_id in audit_map.items(): + audit = audits.get(audit_id) if isinstance(audits, dict) else None + if not isinstance(audit, dict): + continue + score = audit.get("score") + if score is not None and float(score) < 0.9: + lighthouse_failure_urls[bucket].append({ + "url": str(url), + "score": score, + "displayValue": audit.get("displayValue"), + }) + + optional_audit_urls: dict[str, list[dict[str, Any]]] = { + "spell": [], "html": [], "amp": [], "pagination": [], + } + for cat in categories: + if not isinstance(cat, dict): + continue + for issue in cat.get("issues") or []: + if not isinstance(issue, dict): + continue + msg = str(issue.get("message") or "").lower() + rec = {"url": issue.get("url") or "", "message": issue.get("message") or ""} + if "spell" in msg: + optional_audit_urls["spell"].append(rec) + elif "html" in msg and "validation" in msg: + optional_audit_urls["html"].append(rec) + elif "amp" in msg: + optional_audit_urls["amp"].append(rec) + elif "pagination" in msg or "rel=prev" in msg or "rel=next" in msg: + optional_audit_urls["pagination"].append(rec) + report_data = { "site_name": site_display, "report_title": report_display_title, @@ -703,6 +844,9 @@ def _bool_col(col): "top_pages": top_pages, "links": links, "content_urls": content_urls, + "hreflang_issue_urls": hreflang_issue_urls, + "lighthouse_failure_urls": lighthouse_failure_urls, + "optional_audit_urls": optional_audit_urls, "security_findings": security_findings, "content_analytics": content_analytics, "text_content_analysis": text_content_analysis, diff --git a/src/website_profiling/tools/audit_tools/backlink_lists.py b/src/website_profiling/tools/audit_tools/backlink_lists.py new file mode 100644 index 00000000..de3db319 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/backlink_lists.py @@ -0,0 +1,133 @@ +"""Backlink list tools from GSC Links import data.""" +from __future__ import annotations + +from collections import Counter +from typing import Any +from urllib.parse import urlparse + +from psycopg import Connection + +from ._slice import cap_list, parse_limit +from .context import AuditToolContext + + +def _load_links(scoped: AuditToolContext, conn: Connection) -> dict[str, Any] | None: + if scoped.property_id is None: + return None + return scoped.load_gsc_links(conn) + + +def _all_link_rows(data: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for key in ("sample_links", "latest_links"): + chunk = data.get(key) or [] + if isinstance(chunk, list): + rows.extend([r for r in chunk if isinstance(r, dict)]) + return rows + + +def _norm_domain(url: str) -> str: + try: + host = urlparse(str(url or "")).netloc.lower() + return host[4:] if host.startswith("www.") else host + except Exception: + return str(url or "").lower() + + +def list_referring_domains(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required", "domains": [], "total": 0, "truncated": False} + data = _load_links(scoped, conn) + if not data: + return {"error": "no GSC links data", "missing": True, "domains": [], "total": 0, "truncated": False} + domains = list(data.get("top_linking_sites") or []) + if not domains: + counts: Counter[str] = Counter() + for row in _all_link_rows(data): + site = row.get("linking_site") or _norm_domain(str(row.get("source_page") or "")) + if site: + counts[site] += 1 + domains = [{"site": s, "link_count": c} for s, c in counts.most_common()] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(domains, limit, max_cap=50) + return {"domains": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_backlinks_by_anchor_text(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required", "links": [], "total": 0, "truncated": False} + data = _load_links(scoped, conn) + if not data: + return {"error": "no GSC links data", "missing": True, "links": [], "total": 0, "truncated": False} + anchor = str(args.get("anchor_text") or args.get("anchor") or "").strip().lower() + rows = _all_link_rows(data) + if anchor: + rows = [r for r in rows if anchor in str(r.get("anchor_text") or "").lower()] + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(rows, limit, max_cap=100) + return {"links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_backlinks_to_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required", "links": [], "total": 0, "truncated": False} + target = str(args.get("url") or args.get("target_page") or "").strip().lower().rstrip("/") + if not target: + return {"error": "url is required", "links": [], "total": 0, "truncated": False} + data = _load_links(scoped, conn) + if not data: + return {"error": "no GSC links data", "missing": True, "links": [], "total": 0, "truncated": False} + rows = _all_link_rows(data) + matched = [ + r for r in rows + if target in str(r.get("target_page") or "").lower().rstrip("/") + or target in str(r.get("target_url_on_linking_page") or "").lower().rstrip("/") + ] + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(matched, limit, max_cap=100) + return {"url": target, "links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_backlinks_from_domain(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required", "links": [], "total": 0, "truncated": False} + domain = str(args.get("domain") or args.get("linking_site") or "").strip().lower().lstrip("www.") + if not domain: + return {"error": "domain is required", "links": [], "total": 0, "truncated": False} + data = _load_links(scoped, conn) + if not data: + return {"error": "no GSC links data", "missing": True, "links": [], "total": 0, "truncated": False} + rows = _all_link_rows(data) + matched = [ + r for r in rows + if domain in str(r.get("linking_site") or _norm_domain(str(r.get("source_page") or ""))) + ] + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(matched, limit, max_cap=100) + return {"domain": domain, "links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def get_anchor_text_distribution(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return {"error": "property_id is required", "missing": True} + data = _load_links(scoped, conn) + if not data: + return {"error": "no GSC links data", "missing": True, "anchors": []} + top_text = data.get("top_linking_text") or [] + if isinstance(top_text, list) and top_text: + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(top_text, limit, max_cap=100) + return {"anchors": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "source": "top_linking_text"} + counts: Counter[str] = Counter() + for row in _all_link_rows(data): + text = str(row.get("anchor_text") or "").strip() or "(empty)" + counts[text] += 1 + anchors = [{"anchor_text": t, "link_count": c} for t, c in counts.most_common()] + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(anchors, limit, max_cap=100) + return {"anchors": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "source": "sample_links"} diff --git a/src/website_profiling/tools/audit_tools/compare_list_tools.py b/src/website_profiling/tools/audit_tools/compare_list_tools.py new file mode 100644 index 00000000..b95492f6 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/compare_list_tools.py @@ -0,0 +1,174 @@ +"""Report compare list tools using compare_helpers and compare_payload builders.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...reporting.compare_payload import ( + build_issue_deltas, + build_lighthouse_url_deltas, + build_url_set_diff, +) +from ._slice import cap_list, parse_limit +from .compare_helpers import load_compare_pair +from .context import AuditToolContext +from .google_lists import _gsc_rows, _index_gsc_rows, _num + + +def _compare_meta(current_rid: int | None, baseline_rid: int | None, current: dict, baseline: dict) -> dict[str, Any]: + return { + "current_report_id": current_rid, + "baseline_report_id": baseline_rid, + "current_generated_at": current.get("report_generated_at"), + "baseline_generated_at": baseline.get("report_generated_at"), + } + + +def list_compare_new_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return {**err, "issues": [], "total": 0, "truncated": False} + assert current is not None and baseline is not None + deltas = [d for d in build_issue_deltas(current, baseline) if d.get("kind") == "new"] + limit = parse_limit(args.get("limit"), 50, 100) + sliced = cap_list(deltas, limit, max_cap=100) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "issues": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + } + + +def list_compare_resolved_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return {**err, "issues": [], "total": 0, "truncated": False} + assert current is not None and baseline is not None + deltas = [d for d in build_issue_deltas(current, baseline) if d.get("kind") == "resolved"] + limit = parse_limit(args.get("limit"), 50, 100) + sliced = cap_list(deltas, limit, max_cap=100) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "issues": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + } + + +def list_compare_new_urls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return {**err, "urls": [], "total": 0, "truncated": False} + assert current is not None and baseline is not None + diff = build_url_set_diff(current, baseline) + new_urls = diff.get("new_urls") or [] + limit = parse_limit(args.get("limit"), 50, 200) + sliced = cap_list(new_urls, limit, max_cap=200) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "urls": sliced["items"], + "total": diff.get("new_count", sliced["total"]), + "truncated": sliced["truncated"], + } + + +def list_compare_removed_urls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return {**err, "urls": [], "total": 0, "truncated": False} + assert current is not None and baseline is not None + diff = build_url_set_diff(current, baseline) + removed_urls = diff.get("removed_urls") or [] + limit = parse_limit(args.get("limit"), 50, 200) + sliced = cap_list(removed_urls, limit, max_cap=200) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "urls": sliced["items"], + "total": diff.get("removed_count", sliced["total"]), + "truncated": sliced["truncated"], + } + + +def list_compare_lighthouse_regressions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return {**err, "pages": [], "total": 0, "truncated": False} + assert current is not None and baseline is not None + try: + min_drop = float(args.get("min_regression") or 5) + except (TypeError, ValueError): + min_drop = 5.0 + deltas = build_lighthouse_url_deltas(current, baseline) + regressions: list[dict[str, Any]] = [] + for row in deltas: + perf_delta = row.get("performance_delta") + seo_delta = row.get("seo_delta") + perf_drop = perf_delta is not None and perf_delta <= -min_drop + seo_drop = seo_delta is not None and seo_delta <= -min_drop + if perf_drop or seo_drop: + regressions.append({**row, "regression_type": "performance" if perf_drop else "seo"}) + regressions.sort(key=lambda r: min(r.get("performance_delta") or 0, r.get("seo_delta") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(regressions, limit, max_cap=50) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "min_regression": min_drop, + } + + +def list_compare_traffic_losers(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + current, baseline, cur_rid, base_rid, err = load_compare_pair(conn, ctx, args) + if err: + return {**err, "pages": [], "total": 0, "truncated": False} + assert current is not None and baseline is not None + + cur_google = current.get("google") if isinstance(current.get("google"), dict) else None + base_google = baseline.get("google") if isinstance(baseline.get("google"), dict) else None + if not cur_google: + cur_google = scoped.load_google_full(conn) or scoped.load_google(conn) + if not cur_google or not base_google: + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "error": "google data missing on current or baseline report", + "missing": True, + "pages": [], + "total": 0, + "truncated": False, + } + cur_pages = _index_gsc_rows(_gsc_rows(cur_google, "pages"), ("page", "url")) + base_pages = _index_gsc_rows(_gsc_rows(base_google, "pages"), ("page", "url")) + + losers: list[dict[str, Any]] = [] + for key, cur_row in cur_pages.items(): + base_row = base_pages.get(key) + if not base_row: + continue + cur_clicks = _num(cur_row.get("clicks")) + base_clicks = _num(base_row.get("clicks")) + delta = cur_clicks - base_clicks + if delta >= 0: + continue + url = str(cur_row.get("page") or cur_row.get("url") or key) + losers.append({ + "url": url, + "clicks_current": cur_clicks, + "clicks_baseline": base_clicks, + "click_delta": delta, + "impressions_current": _num(cur_row.get("impressions")), + "impressions_baseline": _num(base_row.get("impressions")), + }) + losers.sort(key=lambda r: r.get("click_delta", 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(losers, limit, max_cap=50) + return { + **_compare_meta(cur_rid, base_rid, current, baseline), + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + } diff --git a/src/website_profiling/tools/audit_tools/content_lists.py b/src/website_profiling/tools/audit_tools/content_lists.py new file mode 100644 index 00000000..3fdf7ff0 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/content_lists.py @@ -0,0 +1,249 @@ +"""Content quality, optional audit, and schema list tools.""" +from __future__ import annotations + +import re +from typing import Any + +from psycopg import Connection + +from ._slice import _parse_page_analysis, _row_schema_types_list, cap_list, parse_limit, payload_dict_slice +from .context import AuditToolContext + +_ARTICLE_TYPES = frozenset({"article", "newsarticle", "blogposting", "scholarlyarticle"}) +_ARTICLE_URL_HINTS = ("/blog/", "/news/", "/article/", "/post/", "/posts/") + + +def _optional_audit_urls( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + audit_type: str, + *, + item_key: str = "issues", +) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", item_key: [], "total": 0, "truncated": False} + optional = payload.get("optional_audit_urls") if isinstance(payload.get("optional_audit_urls"), dict) else {} + items = optional.get(audit_type) or [] + if isinstance(items, list) and items: + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(items, limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + categories = payload.get("categories") or [] + needle = audit_type.replace("_", " ") + issues: list[dict[str, Any]] = [] + for cat in categories: + if not isinstance(cat, dict): + continue + for issue in cat.get("issues") or []: + if not isinstance(issue, dict): + continue + msg = str(issue.get("message") or "").lower() + if audit_type == "spell" and "spell" in msg: + issues.append(issue) + elif audit_type == "html" and ("html" in msg or "markup" in msg): + issues.append(issue) + elif audit_type == "amp" and "amp" in msg: + issues.append(issue) + elif audit_type == "pagination" and ("pagination" in msg or "rel=prev" in msg or "rel=next" in msg): + issues.append(issue) + elif needle in msg: + issues.append(issue) + if not issues: + return {"missing": True, item_key: [], "total": 0, "truncated": False, "note": f"enable optional {audit_type} audit in pipeline config"} + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(issues, limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def get_text_content_analysis(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "missing": True} + result = payload_dict_slice(payload, "text_content_analysis") + if result.get("missing"): + content = payload.get("content_analytics") + if isinstance(content, dict) and content.get("keyword_index"): + return {"data": content, "missing": False, "note": "from content_analytics fallback"} + return result + + +def list_pages_containing_keyword(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + keyword = str(args.get("keyword") or args.get("query") or "").strip().lower() + if not keyword: + return {"error": "keyword is required", "pages": [], "total": 0, "truncated": False} + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + tca = payload.get("text_content_analysis") if isinstance(payload.get("text_content_analysis"), dict) else {} + index = tca.get("keyword_index") or [] + pages: list[dict[str, Any]] = [] + if isinstance(index, list): + for entry in index: + if not isinstance(entry, dict): + continue + word = str(entry.get("word") or "").lower() + if keyword not in word and word not in keyword: + continue + for page in entry.get("top_pages") or []: + if isinstance(page, dict): + pages.append({"url": page.get("url"), "keyword": entry.get("word"), "count": page.get("count")}) + elif isinstance(page, (list, tuple)) and page: + pages.append({"url": page[0], "keyword": entry.get("word"), "count": page[1] if len(page) > 1 else 1}) + if not pages: + df = scoped.load_crawl_df(conn) + if df is not None and not df.empty: + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + text = " ".join([ + str(rec.get("title") or ""), + str(rec.get("h1") or ""), + str(rec.get("content_excerpt") or ""), + ]).lower() + if keyword in text: + pages.append({"url": str(rec.get("url") or ""), "keyword": keyword}) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"keyword": keyword, "pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_pages_by_word_count_band(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + try: + min_wc = int(args.get("min_word_count") or 0) + max_wc = int(args.get("max_word_count") or 10_000) + except (TypeError, ValueError): + min_wc, max_wc = 0, 10_000 + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "missing": True} + pages: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + try: + wc = int(rec.get("word_count") or 0) + except (TypeError, ValueError): + wc = 0 + if min_wc <= wc <= max_wc: + pages.append({"url": str(rec.get("url") or ""), "word_count": wc, "title": str(rec.get("title") or "")}) + pages.sort(key=lambda p: p.get("word_count", 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return { + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "band": {"min_word_count": min_wc, "max_word_count": max_wc}, + } + + +def list_duplicate_content_pairs(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pairs": [], "total": 0, "truncated": False} + clusters = payload.get("content_duplicates") or [] + if not isinstance(clusters, list): + clusters = [] + pairs: list[dict[str, Any]] = [] + for cluster in clusters: + if not isinstance(cluster, dict): + continue + members = cluster.get("member_urls") or [] + if not isinstance(members, list): + continue + rep = str(cluster.get("representative_url") or members[0] if members else "") + for url in members: + u = str(url or "") + if u and u != rep: + pairs.append({ + "url_a": rep, + "url_b": u, + "cluster_id": cluster.get("id"), + "similarity": cluster.get("similarity"), + }) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pairs, limit, max_cap=50) + return {"pairs": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_spell_check_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _optional_audit_urls(conn, ctx, args, "spell") + + +def list_html_validation_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _optional_audit_urls(conn, ctx, args, "html") + + +def list_amp_validation_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _optional_audit_urls(conn, ctx, args, "amp") + + +def list_pagination_issues(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _optional_audit_urls(conn, ctx, args, "pagination") + + +def list_schema_errors_by_type(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "errors": [], "total": 0, "truncated": False} + schema_type = str(args.get("schema_type") or args.get("type") or "").strip().lower() + validation = payload.get("rich_results_validation") or [] + if not isinstance(validation, list): + validation = [] + errors = [ + r for r in validation + if isinstance(r, dict) and str(r.get("status") or "").lower() != "pass" + ] + if schema_type: + errors = [ + r for r in errors + if schema_type in str(r.get("type") or r.get("schema_type") or "").lower() + ] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(errors, limit, max_cap=50) + return {"errors": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _has_article_schema(row: dict[str, Any]) -> bool: + types = [t.lower() for t in _row_schema_types_list(row)] + return any(t in _ARTICLE_TYPES or "article" in t for t in types) + + +def list_pages_missing_article_schema(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "missing": True} + pages: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + url = str(rec.get("url") or "").lower() + path = url.split("://", 1)[-1] + looks_article = any(h in path for h in _ARTICLE_URL_HINTS) + pa = _parse_page_analysis(rec) + types = pa.get("json_ld_types") or pa.get("schema_types") or [] + if isinstance(types, str): + types = [types] + if not looks_article and not any("article" in str(t).lower() for t in types): + excerpt = str(rec.get("content_excerpt") or "") + if len(excerpt.split()) < 200: + continue + looks_article = bool(re.search(r"\b(posted|published|author)\b", excerpt, re.I)) + if not looks_article or _has_article_schema(rec): + continue + pages.append({"url": str(rec.get("url") or ""), "title": str(rec.get("title") or ""), "reason": "article_heuristic_no_schema"}) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Estimated"} diff --git a/src/website_profiling/tools/audit_tools/context.py b/src/website_profiling/tools/audit_tools/context.py index 7b1f86bf..f3ee260f 100644 --- a/src/website_profiling/tools/audit_tools/context.py +++ b/src/website_profiling/tools/audit_tools/context.py @@ -11,7 +11,11 @@ from ...db.report_store import read_report_payload from ...integrations.google.gsc_links_store import read_latest_gsc_links_data from ...integrations.google.keyword_store import read_latest_keyword_data -from ...integrations.google.store import read_latest_google_data, read_google_data_full +from ...integrations.google.store import ( + read_google_data_full, + read_latest_google_data, + read_prior_google_snapshot, +) @dataclass @@ -58,6 +62,16 @@ def load_google_full(self, conn: Connection) -> Optional[dict[str, Any]]: embedded = payload.get("google") return embedded if isinstance(embedded, dict) else None + def load_google_pair(self, conn: Connection) -> tuple[Optional[dict[str, Any]], Optional[dict[str, Any]]]: + """Return (current, prior) full Google snapshots for decay/compare tools.""" + current = read_google_data_full(conn, self.property_id) + prior = read_prior_google_snapshot(conn, self.property_id, skip=1) + if current is None: + payload = self.load_payload(conn) + embedded = payload.get("google") + current = embedded if isinstance(embedded, dict) else None + return current, prior + def load_gsc_links(self, conn: Connection) -> Optional[dict[str, Any]]: links = read_latest_gsc_links_data(conn, self.property_id, for_report=False) if links: diff --git a/src/website_profiling/tools/audit_tools/crawl.py b/src/website_profiling/tools/audit_tools/crawl.py index 66d924ca..1bfc536f 100644 --- a/src/website_profiling/tools/audit_tools/crawl.py +++ b/src/website_profiling/tools/audit_tools/crawl.py @@ -320,6 +320,46 @@ def _flag(val: Any) -> bool: return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} +def _console_error_entries(pa: dict[str, Any]) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + browser = pa.get("browser") if isinstance(pa.get("browser"), dict) else {} + for msg in browser.get("console") or []: + if isinstance(msg, dict): + entries.append({ + "error_type": str(msg.get("type") or msg.get("level") or "console"), + "message": str(msg.get("text") or msg.get("message") or ""), + "source": "console", + }) + for msg in browser.get("page_errors") or []: + if isinstance(msg, dict): + entries.append({ + "error_type": "page_error", + "message": str(msg.get("message") or msg.get("name") or ""), + "source": "page_error", + }) + for msg in browser.get("failed_requests") or []: + if isinstance(msg, dict): + entries.append({ + "error_type": "failed_request", + "message": str(msg.get("url") or msg.get("failure") or ""), + "source": "failed_request", + }) + raw = pa.get("console_errors") or pa.get("js_errors") or [] + if isinstance(raw, str): + raw = [raw] + if isinstance(raw, list): + for item in raw: + if isinstance(item, dict): + entries.append({ + "error_type": str(item.get("type") or item.get("level") or "console"), + "message": str(item.get("text") or item.get("message") or item), + "source": "console_errors", + }) + elif item: + entries.append({"error_type": "console", "message": str(item), "source": "console_errors"}) + return entries + + def list_pages_with_console_errors(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: scoped = ctx.with_args(args) df = scoped.load_crawl_df(conn) @@ -328,21 +368,90 @@ def list_pages_with_console_errors(conn: Connection, ctx: AuditToolContext, args pages = [] for _, row in df.iterrows(): pa = _parse_page_analysis(row.to_dict()) - errors = pa.get("console_errors") or pa.get("js_errors") or [] + errors = _console_error_entries(pa) if not errors: continue - if isinstance(errors, str): - errors = [errors] pages.append({ "url": str(row.get("url") or ""), - "error_count": len(errors) if isinstance(errors, list) else 1, - "errors": (errors[:5] if isinstance(errors, list) else [errors]), + "error_count": len(errors), + "errors": errors[:5], + }) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_pages_console_errors_by_type(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + error_type = str(args.get("error_type") or "").strip().lower() + if not error_type: + return {"error": "error_type is required", "pages": [], "total": 0, "truncated": False} + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False} + pages: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + pa = _parse_page_analysis(row.to_dict()) + matched = [ + e for e in _console_error_entries(pa) + if error_type in str(e.get("error_type") or "").lower() + or error_type in str(e.get("source") or "").lower() + ] + if not matched: + continue + pages.append({ + "url": str(row.get("url") or ""), + "error_type": error_type, + "error_count": len(matched), + "errors": matched[:5], }) limit = parse_limit(args.get("limit"), 30, 50) sliced = cap_list(pages, limit, max_cap=50) return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} +def list_pages_js_rendering_delta(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty or "fetch_method" not in df.columns: + return {"pages": [], "total": 0, "truncated": False, "note": "fetch_method not in crawl — use javascript or auto render mode"} + by_url: dict[str, dict[str, dict[str, Any]]] = {} + for _, row in df.iterrows(): + url = str(row.get("url") or "").rstrip("/").lower() + method = str(row.get("fetch_method") or "static").lower() + if not url: + continue + try: + word_count = int(row.get("word_count") or 0) + except (TypeError, ValueError): + word_count = 0 + by_url.setdefault(url, {})[method] = { + "title": str(row.get("title") or ""), + "word_count": word_count, + "h1": str(row.get("h1") or ""), + } + pages: list[dict[str, Any]] = [] + for url, methods in by_url.items(): + static = methods.get("static") + rendered = methods.get("rendered") or methods.get("javascript") + if not static or not rendered: + continue + title_diff = static.get("title") != rendered.get("title") + wc_diff = abs(int(static.get("word_count") or 0) - int(rendered.get("word_count") or 0)) + h1_diff = static.get("h1") != rendered.get("h1") + if title_diff or wc_diff > 50 or h1_diff: + pages.append({ + "url": url, + "title_differs": title_diff, + "word_count_delta": wc_diff, + "h1_differs": h1_diff, + }) + pages.sort(key=lambda p: -int(p.get("word_count_delta") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Crawl"} + + def list_pages_by_fetch_method(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: method = str(args.get("fetch_method") or "").strip().lower() if not method: diff --git a/src/website_profiling/tools/audit_tools/data_coverage.py b/src/website_profiling/tools/audit_tools/data_coverage.py index 5a30fb47..773663e5 100644 --- a/src/website_profiling/tools/audit_tools/data_coverage.py +++ b/src/website_profiling/tools/audit_tools/data_coverage.py @@ -26,6 +26,7 @@ def get_data_coverage_report(conn: Connection, ctx: AuditToolContext, args: dict google = scoped.load_google(conn) keywords = scoped.load_keywords(conn) gsc_links = scoped.load_gsc_links(conn) + google_full = scoped.load_google_full(conn) checks: list[dict[str, Any]] = [] checks.append(_check( @@ -79,6 +80,43 @@ def get_data_coverage_report(conn: Connection, ctx: AuditToolContext, args: dict bool(payload), "Run a site audit crawl and report build.", )) + checks.append(_check( + "gsc_full_blob", + bool(google_full and isinstance(google_full.get("gsc_full"), dict)), + "Re-run Google fetch to populate gsc_full for list/decay tools.", + )) + checks.append(_check( + "ga4_full_blob", + bool(google_full and isinstance(google_full.get("ga4_full"), dict)), + "Re-run GA4 fetch to populate ga4_full for landing-page list tools.", + )) + checks.append(_check( + "keyword_history", + bool(keywords and keywords.get("fetched_at")), + "Run keyword enrichment twice for rank delta tools.", + )) + checks.append(_check( + "text_content_analysis", + bool(payload.get("text_content_analysis")), + "Report build includes text_content_analysis from crawl.", + )) + checks.append(_check( + "semantic_keyword_clusters", + bool(payload.get("semantic_keyword_clusters")), + "Enable llm_enable_keyword_clusters for cluster list tools.", + )) + checks.append(_check( + "access_log", + bool(payload.get("log_analysis") or payload.get("access_log_summary")), + "Upload access logs in Integrations for log list tools.", + )) + from ...integrations.google.store import read_prior_google_snapshot + prior_google = read_prior_google_snapshot(conn, scoped.property_id, skip=1) if scoped.property_id else None + checks.append(_check( + "prior_google_snapshot", + bool(prior_google), + "Run at least two Google data fetches for decay/compare period tools.", + )) missing = [c["signal"] for c in checks if not c["populated"]] return { diff --git a/src/website_profiling/tools/audit_tools/export_tools.py b/src/website_profiling/tools/audit_tools/export_tools.py index 253401a2..cc738ea1 100644 --- a/src/website_profiling/tools/audit_tools/export_tools.py +++ b/src/website_profiling/tools/audit_tools/export_tools.py @@ -101,6 +101,97 @@ "search_keywords", "search_pages_by_schema_type", "list_pages_without_schema", + "list_pages_title_too_short", + "list_pages_title_too_long", + "list_pages_slow_response", + "list_pages_missing_html_lang", + "list_pages_invalid_viewport", + "list_pages_color_contrast_failures", + "list_pages_high_reading_level", + "list_pages_very_thin_content", + "list_hreflang_issue_pages", + "list_pages_missing_og_tags", + "list_pages_missing_twitter_cards", + "list_pages_invalid_json_ld", + "list_pages_mixed_language", + "list_orphan_hub_suggestions", + "list_lighthouse_failure_lcp", + "list_lighthouse_failure_inp", + "list_lighthouse_failure_cls", + "list_lighthouse_failure_seo", + "list_pages_console_errors_by_type", + "list_pages_js_rendering_delta", + "list_gsc_pages_by_impressions", + "list_gsc_pages_by_clicks", + "list_gsc_queries_by_impressions", + "list_gsc_queries_by_clicks", + "list_gsc_ctr_underperformers", + "list_gsc_decaying_pages", + "list_gsc_decaying_queries", + "list_gsc_new_queries", + "list_ga4_landing_pages", + "list_ga4_pages_by_bounce_rate", + "list_ga4_pages_by_engagement_rate", + "list_gsc_ga4_mismatch_pages", + "list_gsc_pages_by_position_band", + "list_gsc_branded_queries", + "list_gsc_non_branded_queries", + "list_keyword_rank_improvements", + "list_keyword_rank_declines", + "list_keywords_new_to_top_10", + "list_keywords_fell_out_of_top_10", + "list_cannibalisation_queries", + "list_cannibalisation_urls", + "list_misaligned_queries", + "list_keywords_by_recommended_action", + "list_keywords_by_serp_feature", + "list_semantic_cluster_pages", + "list_semantic_cluster_queries", + "list_keywords_near_page_one", + "list_keywords_high_impression_zero_click", + "list_keywords_by_competition_band", + "list_keywords_with_ai_overview", + "list_keywords_local_pack", + "list_keywords_question_intent", + "list_keywords_commercial_intent", + "list_referring_domains", + "list_backlinks_by_anchor_text", + "list_backlinks_to_url", + "list_backlinks_from_domain", + "list_outbound_links", + "list_internal_links_from_url", + "list_internal_links_to_url", + "list_links_by_rel_nofollow", + "list_pagerank_low_pages", + "list_indexation_submitted_not_indexed", + "list_indexation_indexed_not_submitted", + "list_sitemap_urls_not_in_crawl", + "list_crawl_urls_not_in_sitemap", + "list_log_paths_by_hits", + "list_log_5xx_paths", + "list_log_googlebot_low_crawl", + "list_log_orphan_high_traffic", + "list_redirect_chains_by_length", + "list_hreflang_reciprocal_gaps", + "list_pages_containing_keyword", + "list_pages_by_word_count_band", + "list_duplicate_content_pairs", + "list_spell_check_issues", + "list_html_validation_issues", + "list_amp_validation_issues", + "list_pagination_issues", + "list_schema_errors_by_type", + "list_pages_missing_article_schema", + "list_pages_missing_howto_schema", + "list_pages_ai_citation_signals", + "list_pages_missing_llms_txt_reference", + "list_robots_blocked_ai_crawlers", + "list_compare_new_issues", + "list_compare_resolved_issues", + "list_compare_new_urls", + "list_compare_removed_urls", + "list_compare_lighthouse_regressions", + "list_compare_traffic_losers", }) _EXPORT_TOOL_NAMES = frozenset({ diff --git a/src/website_profiling/tools/audit_tools/geo_list_tools.py b/src/website_profiling/tools/audit_tools/geo_list_tools.py new file mode 100644 index 00000000..4aaaa240 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/geo_list_tools.py @@ -0,0 +1,242 @@ +"""GEO/AEO page-level list tools.""" +from __future__ import annotations + +import re +from typing import Any +from urllib.parse import urljoin + +import requests +from psycopg import Connection + +from ._slice import _parse_page_analysis, _row_schema_types_list, cap_list, parse_limit +from .context import AuditToolContext +from .geo_tools import _fetch_llms_txt, _has_faq_schema + +_HOWTO_TYPES = frozenset({"howto", "how-to"}) +_HOWTO_URL_HINTS = ("/how-to", "/howto", "/guide/", "/tutorial/", "/recipes/") +_AI_CRAWLER_AGENTS = ( + "GPTBot", + "ChatGPT-User", + "ClaudeBot", + "anthropic-ai", + "Google-Extended", + "PerplexityBot", + "Bytespider", + "CCBot", +) + + +def _has_howto_schema(row: dict[str, Any]) -> bool: + types = [t.lower() for t in _row_schema_types_list(row)] + return any(t in _HOWTO_TYPES or "howto" in t for t in types) + + +def _looks_like_howto_page(rec: dict[str, Any]) -> bool: + url = str(rec.get("url") or "").lower() + heading = str(rec.get("heading_text") or rec.get("h1") or "").lower() + title = str(rec.get("title") or "").lower() + if any(h in url for h in _HOWTO_URL_HINTS): + return True + return any(k in heading or k in title for k in ("how to", "step-by-step", "tutorial", "guide")) + + +def _aeo_score(rec: dict[str, Any]) -> dict[str, Any]: + excerpt = str(rec.get("content_excerpt") or "") + words = excerpt.split() + lead = " ".join(words[:80]) + has_list = bool(re.search(r"^\s*[-*•]\s", excerpt, re.M)) or "

  • " in str(rec.get("html") or "").lower() + has_definition = bool(re.search(r"\b(is|are|means|refers to)\b", lead[:400], re.I)) + try: + wc = int(rec.get("word_count") or 0) + except (TypeError, ValueError): + wc = 0 + quotability = 0 + if wc >= 200: + quotability += 25 + if has_list: + quotability += 20 + if has_definition: + quotability += 25 + if _has_faq_schema(rec): + quotability += 30 + schema_types = _row_schema_types_list(rec) + if schema_types: + quotability += 10 + return { + "word_count": wc, + "has_lists": has_list, + "has_definition_pattern": has_definition, + "quotability_score": min(100, quotability), + "schema_types": schema_types[:5], + } + + +def _parse_robots_txt(domain: str) -> str: + if not domain: + return "" + base = f"https://{domain.lstrip('https://').lstrip('http://').split('/')[0]}" + url = urljoin(base + "/", "robots.txt") + try: + resp = requests.get(url, timeout=8, headers={"User-Agent": "SiteAudit/1.0"}) + if resp.status_code == 200: + return resp.text + except requests.RequestException: + return "" + return "" + + +def _agent_blocked(robots_text: str, agent: str) -> bool: + blocks: dict[str, bool] = {} + current_agent = "*" + for line in robots_text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + lower = line.lower() + if lower.startswith("user-agent:"): + current_agent = line.split(":", 1)[1].strip() + continue + if lower.startswith("disallow:"): + path = line.split(":", 1)[1].strip() + if path == "/": + blocks[current_agent.lower()] = True + agent_lower = agent.lower() + return bool(blocks.get(agent_lower) or blocks.get("*")) + + +def _llms_urls(llms_preview: str, llms_url: str) -> set[str]: + urls: set[str] = set() + for line in (llms_preview or "").splitlines(): + for match in re.findall(r"https?://[^\s)>]+", line): + urls.add(match.rstrip("/").lower()) + if llms_url: + urls.add(llms_url.rstrip("/").lower()) + return urls + + +def list_pages_missing_howto_schema(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "missing": True} + pages: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + if not _looks_like_howto_page(rec) or _has_howto_schema(rec): + continue + pages.append({ + "url": str(rec.get("url") or ""), + "title": str(rec.get("title") or ""), + "reason": "howto_heuristic_no_schema", + }) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Estimated"} + + +def list_pages_ai_citation_signals(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "missing": True} + try: + min_score = int(args.get("min_score") or 0) + except (TypeError, ValueError): + min_score = 0 + scored: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + signals = _aeo_score(rec) + if signals["quotability_score"] < min_score: + continue + scored.append({ + "url": str(rec.get("url") or ""), + "title": str(rec.get("title") or ""), + **signals, + }) + scored.sort(key=lambda p: -int(p.get("quotability_score") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(scored, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "provenance": "Estimated"} + + +def list_pages_missing_llms_txt_reference(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + domain = scoped.resolve_property_domain(conn) + llms = _fetch_llms_txt(domain) + if not llms.get("found"): + return { + "pages": [], + "total": 0, + "truncated": False, + "missing": True, + "note": "llms.txt not found on domain", + "domain": domain, + } + listed = _llms_urls(str(llms.get("preview") or ""), str(llms.get("url") or "")) + payload = scoped.load_payload(conn) + candidates: list[str] = [] + if payload: + for page in payload.get("top_pages") or payload.get("links") or []: + if isinstance(page, dict) and page.get("url"): + candidates.append(str(page["url"])) + df = scoped.load_crawl_df(conn) + if df is not None and not df.empty: + for _, row in df.iterrows(): + rec = row.to_dict() + if str(rec.get("status") or "").startswith("2") and rec.get("url"): + candidates.append(str(rec["url"])) + seen: set[str] = set() + missing: list[dict[str, Any]] = [] + for url in candidates: + norm = url.rstrip("/").lower() + if norm in seen: + continue + seen.add(norm) + if norm in listed or url in listed: + continue + missing.append({"url": url, "llms_txt_url": llms.get("url")}) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(missing, limit, max_cap=50) + return { + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "llms_txt_url": llms.get("url"), + "provenance": "Estimated", + } + + +def list_robots_blocked_ai_crawlers(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + domain = scoped.resolve_property_domain(conn) + if not domain: + return {"error": "domain unknown", "agents": [], "total": 0, "truncated": False} + robots_text = _parse_robots_txt(domain) + if not robots_text.strip(): + return { + "domain": domain, + "agents": [], + "total": 0, + "truncated": False, + "missing": True, + "note": "robots.txt not reachable", + } + blocked: list[dict[str, Any]] = [] + for agent in _AI_CRAWLER_AGENTS: + if _agent_blocked(robots_text, agent): + blocked.append({"agent": agent, "blocked": True, "scope": "disallow: /"}) + limit = parse_limit(args.get("limit"), 10, 20) + sliced = cap_list(blocked, limit, max_cap=20) + return { + "domain": domain, + "agents": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "robots_txt_checked": True, + "provenance": "Crawl", + } diff --git a/src/website_profiling/tools/audit_tools/google_lists.py b/src/website_profiling/tools/audit_tools/google_lists.py new file mode 100644 index 00000000..fac019aa --- /dev/null +++ b/src/website_profiling/tools/audit_tools/google_lists.py @@ -0,0 +1,474 @@ +"""Google Search Console and GA4 list/delta tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...integrations.google.keyword_enrich import ctr_as_fraction, industry_ctr +from ...integrations.google.normalize import normalize_url, url_to_path +from ._slice import cap_list, parse_limit +from .context import AuditToolContext +from .insight_helpers import blend_landing_pages, provenance_block, traffic_health_ratio, _num + + +def _gsc_ga4_blobs(raw: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + gsc = raw.get("gsc_full") if isinstance(raw.get("gsc_full"), dict) else raw.get("gsc") or {} + ga4 = raw.get("ga4_full") if isinstance(raw.get("ga4_full"), dict) else raw.get("ga4") or {} + return gsc if isinstance(gsc, dict) else {}, ga4 if isinstance(ga4, dict) else {} + + +def _load_google_pair(ctx: AuditToolContext, conn: Connection) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Current + prior Google snapshots (read_prior_google_snapshot, else 2nd google_data row).""" + current, prior = ctx.load_google_pair(conn) + if prior is not None or ctx.property_id is None: + return current, prior + try: + from ...integrations.google.store import read_prior_google_snapshot + + prior = read_prior_google_snapshot(conn, ctx.property_id, skip=1) + except Exception: + prior = None + if prior is None: + try: + cur = conn.execute( + """ + SELECT data FROM google_data + WHERE property_id = %s + ORDER BY id DESC LIMIT 2 + """, + (int(ctx.property_id),), + ) + rows = cur.fetchall() or [] + if len(rows) >= 2: + from ...db.storage import _parse_row_json + + prior_data = _parse_row_json(rows[1]) + prior = prior_data if isinstance(prior_data, dict) else None + except Exception: + prior = None + return current, prior + + +def _gsc_rows(data: dict[str, Any] | None, key: str) -> list[dict[str, Any]]: + if not data: + return [] + gsc, _ = _gsc_ga4_blobs(data) + rows = gsc.get(key) or gsc.get(f"top_{key}") or [] + if isinstance(rows, list): + return [r for r in rows if isinstance(r, dict)] + return [] + + +def _sort_gsc_rows(rows: list[dict[str, Any]], field: str, limit: int) -> dict[str, Any]: + sorted_rows = sorted(rows, key=lambda r: -_num(r.get(field))) + sliced = cap_list(sorted_rows, limit, max_cap=50) + return {"items": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_gsc_pages_by_impressions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + result = _sort_gsc_rows(_gsc_rows(data, "pages"), "impressions", limit) + return {"pages": result["items"], "total": result["total"], "truncated": result["truncated"]} + + +def list_gsc_pages_by_clicks(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + result = _sort_gsc_rows(_gsc_rows(data, "pages"), "clicks", limit) + return {"pages": result["items"], "total": result["total"], "truncated": result["truncated"]} + + +def list_gsc_queries_by_impressions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + result = _sort_gsc_rows(_gsc_rows(data, "queries"), "impressions", limit) + return {"queries": result["items"], "total": result["total"], "truncated": result["truncated"]} + + +def list_gsc_queries_by_clicks(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + result = _sort_gsc_rows(_gsc_rows(data, "queries"), "clicks", limit) + return {"queries": result["items"], "total": result["total"], "truncated": result["truncated"]} + + +def list_gsc_ctr_underperformers(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + pages = _gsc_rows(data, "pages") + ctrs = [ctr_as_fraction(r.get("ctr")) for r in pages if 1 <= _num(r.get("position"), 99) <= 10] + ctrs = [c for c in ctrs if c > 0] + site_median = sorted(ctrs)[len(ctrs) // 2] if ctrs else 0.05 + under: list[dict[str, Any]] = [] + for row in pages: + pos = _num(row.get("position"), 99) + if pos < 1 or pos > 10: + continue + ctr = ctr_as_fraction(row.get("ctr")) + expected = industry_ctr(pos) + if ctr > 0 and ctr < min(site_median * 0.7, expected * 0.7): + under.append({ + "page": row.get("page") or row.get("url"), + "clicks": row.get("clicks"), + "impressions": row.get("impressions"), + "ctr": row.get("ctr"), + "position": pos, + "site_median_ctr": round(site_median, 4), + }) + under.sort(key=lambda r: -_num(r.get("impressions"))) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(under, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _index_gsc_rows(rows: list[dict[str, Any]], key_fields: tuple[str, ...]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for row in rows: + key = "" + for field in key_fields: + key = str(row.get(field) or "").strip() + if key: + break + if key: + out[key] = row + return out + + +def _gsc_deltas( + current_rows: list[dict[str, Any]], + prior_rows: list[dict[str, Any]], + key_fields: tuple[str, ...], + *, + decay: bool = True, +) -> list[dict[str, Any]]: + curr = _index_gsc_rows(current_rows, key_fields) + prev = _index_gsc_rows(prior_rows, key_fields) + deltas: list[dict[str, Any]] = [] + for key, row in curr.items(): + old = prev.get(key) + if not old: + continue + click_delta = _num(row.get("clicks")) - _num(old.get("clicks")) + imp_delta = _num(row.get("impressions")) - _num(old.get("impressions")) + pos_delta = _num(row.get("position")) - _num(old.get("position")) + if decay: + if click_delta >= 0 and imp_delta >= 0 and pos_delta <= 0: + continue + else: + if click_delta <= 0 and imp_delta <= 0: + continue + entry = dict(row) + entry["key"] = key + entry["click_delta"] = int(click_delta) + entry["impression_delta"] = int(imp_delta) + entry["position_delta"] = round(pos_delta, 2) + deltas.append(entry) + deltas.sort(key=lambda r: (r.get("click_delta", 0), r.get("impression_delta", 0))) + return deltas + + +def list_gsc_decaying_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + current, prior = _load_google_pair(scoped, conn) + if not current: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + if not prior: + return {"error": "no prior google snapshot for decay comparison", "missing": True, "pages": [], "total": 0, "truncated": False} + deltas = _gsc_deltas(_gsc_rows(current, "pages"), _gsc_rows(prior, "pages"), ("page", "url"), decay=True) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(deltas, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_gsc_decaying_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + current, prior = _load_google_pair(scoped, conn) + if not current: + return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False} + if not prior: + return {"error": "no prior google snapshot for decay comparison", "missing": True, "queries": [], "total": 0, "truncated": False} + deltas = _gsc_deltas(_gsc_rows(current, "queries"), _gsc_rows(prior, "queries"), ("query",), decay=True) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(deltas, limit, max_cap=50) + return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_gsc_new_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + current, prior = _load_google_pair(scoped, conn) + if not current: + return {"error": "no google data found", "missing": True, "queries": [], "total": 0, "truncated": False} + if not prior: + return {"error": "no prior google snapshot", "missing": True, "queries": [], "total": 0, "truncated": False} + curr = _index_gsc_rows(_gsc_rows(current, "queries"), ("query",)) + prev = _index_gsc_rows(_gsc_rows(prior, "queries"), ("query",)) + new_rows = [row for key, row in curr.items() if key not in prev] + new_rows.sort(key=lambda r: -_num(r.get("impressions"))) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(new_rows, limit, max_cap=50) + return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_ga4_landing_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + _, ga4 = _gsc_ga4_blobs(data) + pages = ga4.get("top_pages") or [] + if not pages and isinstance(ga4.get("by_path"), dict): + pages = [{"path": k, **v} for k, v in ga4["by_path"].items() if isinstance(v, dict)] + if not isinstance(pages, list): + pages = [] + pages = sorted([p for p in pages if isinstance(p, dict)], key=lambda r: -_num(r.get("sessions"))) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_ga4_pages_by_bounce_rate(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + _, ga4 = _gsc_ga4_blobs(data) + pages = list(ga4.get("top_pages") or []) + if not pages and isinstance(ga4.get("by_path"), dict): + pages = [{"path": k, **v} for k, v in ga4["by_path"].items() if isinstance(v, dict)] + pages = [p for p in pages if isinstance(p, dict) and p.get("bounceRate") is not None] + pages.sort(key=lambda r: -_num(r.get("bounceRate"))) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_ga4_pages_by_engagement_rate(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + _, ga4 = _gsc_ga4_blobs(data) + pages = list(ga4.get("top_pages") or []) + if not pages and isinstance(ga4.get("by_path"), dict): + pages = [{"path": k, **v} for k, v in ga4["by_path"].items() if isinstance(v, dict)] + pages = [p for p in pages if isinstance(p, dict) and p.get("engagementRate") is not None] + pages.sort(key=lambda r: -_num(r.get("engagementRate"))) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _daily_series(data: dict[str, Any] | None, section: str, match_key: str, match_val: str) -> list[dict[str, Any]]: + if not data: + return [] + block = data.get(section) if isinstance(data.get(section), dict) else {} + daily = block.get("daily") or [] + if not isinstance(daily, list): + return [] + needle = match_val.strip().lower() + out: list[dict[str, Any]] = [] + for row in daily: + if not isinstance(row, dict): + continue + for field in ("query", "page", "path", "url"): + if field in row and str(row.get(field) or "").strip().lower() == needle: + out.append(row) + break + else: + dims = row.get("dimensions") if isinstance(row.get("dimensions"), dict) else {} + if any(str(dims.get(k) or "").strip().lower() == needle for k in dims): + out.append(row) + return out + + +def get_gsc_query_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + query = str(args.get("query") or "").strip() + if not query: + return {"error": "query is required"} + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True} + series = _daily_series(data, "gsc", "query", query) + if not series: + gsc, _ = _gsc_ga4_blobs(data) + for row in gsc.get("queries") or []: + if isinstance(row, dict) and str(row.get("query") or "").lower() == query.lower(): + return {"query": query, "snapshot": row, "daily": [], "missing": True, "note": "daily series not stored"} + return {"query": query, "daily": series, "fetched_at": data.get("fetched_at")} + + +def get_gsc_page_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + url = str(args.get("url") or args.get("page") or "").strip() + if not url: + return {"error": "url is required"} + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True} + series = _daily_series(data, "gsc", "page", url) + norm = normalize_url(url) + if not series: + series = _daily_series(data, "gsc", "page", norm) + return {"url": url, "daily": series, "fetched_at": data.get("fetched_at"), "missing": not bool(series)} + + +def get_ga4_path_trend(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + path = str(args.get("path") or args.get("url") or "").strip() + if not path: + return {"error": "path is required"} + if path.startswith(("http://", "https://")): + path = url_to_path(path) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True} + series = _daily_series(data, "ga4", "path", path) + return {"path": path, "daily": series, "fetched_at": data.get("fetched_at"), "missing": not bool(series)} + + +def list_gsc_ga4_mismatch_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + gsc, ga4 = _gsc_ga4_blobs(data) + by_page = gsc.get("by_page") if isinstance(gsc.get("by_page"), dict) else {} + by_path = ga4.get("by_path") if isinstance(ga4.get("by_path"), dict) else {} + if not by_page and gsc.get("pages"): + by_page = {str(r.get("page")): r for r in (gsc.get("pages") or []) if isinstance(r, dict) and r.get("page")} + rows = blend_landing_pages(by_page, by_path, limit=200, min_impressions=0) + mismatches: list[dict[str, Any]] = [] + for row in rows: + clicks = _num(row.get("gsc_clicks")) + sessions = _num(row.get("ga4_sessions")) + if clicks >= 10 and sessions == 0: + mismatches.append({**row, "mismatch": "gsc_clicks_no_ga4_sessions"}) + elif sessions >= 10 and clicks == 0: + mismatches.append({**row, "mismatch": "ga4_sessions_no_gsc_clicks"}) + elif clicks > 0 and sessions / clicks > 3: + mismatches.append({**row, "mismatch": "ga4_sessions_high_vs_clicks"}) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(mismatches, limit, max_cap=50) + return { + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "provenance": provenance_block(["gsc", "ga4"], data.get("fetched_at")), + } + + +def list_gsc_pages_by_position_band(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True, "pages": [], "total": 0, "truncated": False} + try: + min_pos = float(args.get("min_position") or 1) + max_pos = float(args.get("max_position") or 20) + except (TypeError, ValueError): + min_pos, max_pos = 1.0, 20.0 + pages = [ + r for r in _gsc_rows(data, "pages") + if min_pos <= _num(r.get("position"), 99) <= max_pos + ] + pages.sort(key=lambda r: _num(r.get("position"))) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def get_gsc_site_benchmarks(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_google_full(conn) or scoped.load_google(conn) + if not data: + return {"error": "no google data found", "missing": True} + gsc, ga4 = _gsc_ga4_blobs(data) + pages = _gsc_rows(data, "pages") + ctrs = [ctr_as_fraction(r.get("ctr")) for r in pages if _num(r.get("impressions")) > 0] + positions = [_num(r.get("position")) for r in pages if _num(r.get("position")) > 0] + ctrs.sort() + positions.sort() + return { + "median_ctr": round(ctrs[len(ctrs) // 2], 4) if ctrs else None, + "median_position": round(positions[len(positions) // 2], 2) if positions else None, + "page_count": len(pages), + "gsc_summary": gsc.get("summary") if isinstance(gsc.get("summary"), dict) else {}, + "ga4_summary": ga4.get("summary") if isinstance(ga4.get("summary"), dict) else {}, + "fetched_at": data.get("fetched_at"), + } + + +def list_gsc_branded_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_keywords(conn) + if not data: + return {"error": "no keyword data found", "missing": True, "queries": [], "total": 0, "truncated": False} + branded = [r for r in (data.get("rows") or []) if isinstance(r, dict) and r.get("is_branded")] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(branded, limit, max_cap=50) + return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_gsc_non_branded_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + data = scoped.load_keywords(conn) + if not data: + return {"error": "no keyword data found", "missing": True, "queries": [], "total": 0, "truncated": False} + non_branded = [r for r in (data.get("rows") or []) if isinstance(r, dict) and not r.get("is_branded")] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(non_branded, limit, max_cap=50) + return {"queries": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def compare_gsc_periods(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + current, prior = _load_google_pair(scoped, conn) + if not current: + return {"error": "no google data found", "missing": True} + if not prior: + return {"error": "no prior google snapshot for period comparison", "missing": True} + gsc_curr, ga4_curr = _gsc_ga4_blobs(current) + gsc_prev, ga4_prev = _gsc_ga4_blobs(prior) + curr_summary = gsc_curr.get("summary") if isinstance(gsc_curr.get("summary"), dict) else {} + prev_summary = gsc_prev.get("summary") if isinstance(gsc_prev.get("summary"), dict) else {} + ga4_curr_summary = ga4_curr.get("summary") if isinstance(ga4_curr.get("summary"), dict) else {} + ga4_prev_summary = ga4_prev.get("summary") if isinstance(ga4_prev.get("summary"), dict) else {} + + def _delta(key: str, cur: dict[str, Any], prev_d: dict[str, Any]) -> dict[str, Any]: + c = _num(cur.get(key)) + p = _num(prev_d.get(key)) + return {"current": c, "prior": p, "delta": round(c - p, 2)} + + return { + "gsc": { + "clicks": _delta("clicks", curr_summary, prev_summary), + "impressions": _delta("impressions", curr_summary, prev_summary), + "ctr": _delta("ctr", curr_summary, prev_summary), + "position": _delta("position", curr_summary, prev_summary), + }, + "ga4": { + "sessions": _delta("sessions", ga4_curr_summary, ga4_prev_summary), + "users": _delta("users", ga4_curr_summary, ga4_prev_summary), + }, + "traffic_health": traffic_health_ratio(curr_summary, ga4_curr_summary), + "current_fetched_at": current.get("fetched_at"), + "prior_fetched_at": prior.get("fetched_at"), + "provenance": provenance_block(["gsc", "ga4"], current.get("fetched_at")), + } diff --git a/src/website_profiling/tools/audit_tools/indexation_lists.py b/src/website_profiling/tools/audit_tools/indexation_lists.py new file mode 100644 index 00000000..bdca54be --- /dev/null +++ b/src/website_profiling/tools/audit_tools/indexation_lists.py @@ -0,0 +1,345 @@ +"""Indexation, log analysis, redirect chain, and hreflang list tools.""" +from __future__ import annotations + +from typing import Any + +from psycopg import Connection + +from ...integrations.google.normalize import normalize_url, url_to_path +from ...reporting.categories import REDIRECT_CHAIN_LONG +from ._slice import _parse_page_analysis, cap_list, parse_limit +from .context import AuditToolContext +from .ops import _load_log_analysis + +_REDIRECT_CHAIN_MIN = REDIRECT_CHAIN_LONG + + +def _norm_path(url: str) -> str: + try: + return url_to_path(str(url or "")) or "/" + except Exception: + return str(url or "") + + +def _indexation_cov(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return None, {"error": "no report found", "urls": [], "total": 0, "truncated": False} + cov = payload.get("indexation_coverage") + if not isinstance(cov, dict): + return payload, { + "error": "indexation_coverage not in report", + "missing": True, + "urls": [], + "total": 0, + "truncated": False, + } + return payload, cov + + +def _cap_indexation_urls( + urls: list[Any], + args: dict[str, Any], + *, + gap_type: str = "", + totals: dict[str, Any] | None = None, + total_key: str = "", +) -> dict[str, Any]: + if not isinstance(urls, list): + urls = [] + items = [{"url": str(u)} if not isinstance(u, dict) else u for u in urls if u] + limit = parse_limit(args.get("limit"), 50, 200) + sliced = cap_list(items, limit, max_cap=200) + total_all = None + if totals and total_key: + total_all = totals.get(total_key) + return { + "gap_type": gap_type, + "urls": sliced["items"], + "total": int(total_all or sliced["total"]), + "truncated": sliced["truncated"] or (int(total_all or 0) > limit), + } + + +def list_indexation_submitted_not_indexed(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """URLs in sitemap but not successfully crawled (submitted, not indexed in crawl).""" + _, cov = _indexation_cov(conn, ctx, args) + if cov.get("error"): + return cov + assert isinstance(cov, dict) + lists = cov.get("lists") or {} + urls = lists.get("sitemap_only") or [] + totals = cov.get("lists_total") or {} + return _cap_indexation_urls(urls, args, gap_type="sitemap_only", totals=totals, total_key="sitemap_only") + + +def list_indexation_indexed_not_submitted(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Crawled URLs not present in sitemap (found/indexable but not submitted).""" + _, cov = _indexation_cov(conn, ctx, args) + if cov.get("error"): + return cov + assert isinstance(cov, dict) + lists = cov.get("lists") or {} + urls = lists.get("crawled_not_in_sitemap") or [] + totals = cov.get("lists_total") or {} + return _cap_indexation_urls(urls, args, gap_type="crawled_not_in_sitemap", totals=totals, total_key="crawled_not_in_sitemap") + + +def list_sitemap_urls_not_in_crawl(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + _, cov = _indexation_cov(conn, ctx, args) + if cov.get("error"): + return cov + assert isinstance(cov, dict) + lists = cov.get("lists") or {} + urls = lists.get("sitemap_only") or [] + totals = cov.get("lists_total") or {} + result = _cap_indexation_urls(urls, args, gap_type="sitemap_only", totals=totals, total_key="sitemap_only") + result["source"] = "indexation_coverage.lists.sitemap_only" + return result + + +def list_crawl_urls_not_in_sitemap(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + _, cov = _indexation_cov(conn, ctx, args) + if cov.get("error"): + return cov + assert isinstance(cov, dict) + lists = cov.get("lists") or {} + urls = lists.get("crawled_not_in_sitemap") or [] + totals = cov.get("lists_total") or {} + result = _cap_indexation_urls(urls, args, gap_type="crawled_not_in_sitemap", totals=totals, total_key="crawled_not_in_sitemap") + result["source"] = "indexation_coverage.lists.crawled_not_in_sitemap" + return result + + +def _require_log(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any]]: + scoped = ctx.with_args(args) + if scoped.property_id is None: + return None, {"error": "property_id is required", "paths": [], "total": 0, "truncated": False} + row = _load_log_analysis(conn, int(scoped.property_id)) + if not row: + return None, {"error": "no log uploads found", "missing": True, "paths": [], "total": 0, "truncated": False} + return row, {} + + +def list_log_paths_by_hits(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + row, err = _require_log(conn, ctx, args) + if err: + return err + assert row is not None + analysis = row.get("analysis") or {} + paths = analysis.get("top_paths") or [] + if not isinstance(paths, list): + paths = [] + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(paths, limit, max_cap=100) + return { + "paths": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "upload_id": row.get("upload_id"), + "filename": row.get("filename"), + } + + +def list_log_5xx_paths(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + row, err = _require_log(conn, ctx, args) + if err: + return err + assert row is not None + analysis = row.get("analysis") or {} + paths = analysis.get("paths_5xx") or [] + if not isinstance(paths, list): + paths = [] + if not paths: + status_counts = analysis.get("status_counts") or {} + paths = [ + {"path": f"status_{code}", "hits": hits, "aggregate": True} + for code, hits in status_counts.items() + if str(code).startswith("5") + ] + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(paths, limit, max_cap=100) + return { + "paths": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "upload_id": row.get("upload_id"), + } + + +def list_log_googlebot_low_crawl(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """High-traffic log paths with low or zero Googlebot hits (under-crawled).""" + scoped = ctx.with_args(args) + row, err = _require_log(conn, ctx, args) + if err: + return err + assert row is not None + analysis = row.get("analysis") or {} + top_paths = analysis.get("top_paths") or [] + bot_paths = { + str(r.get("path") or ""): int(r.get("hits") or 0) + for r in (analysis.get("googlebot_paths") or []) + if isinstance(r, dict) and r.get("path") + } + try: + min_hits = int(args.get("min_hits") or 20) + max_bot_hits = int(args.get("max_googlebot_hits") or 0) + except (TypeError, ValueError): + min_hits, max_bot_hits = 20, 0 + payload = scoped.load_payload(conn) + crawl_paths: set[str] = set() + if payload: + for link in payload.get("links") or []: + if isinstance(link, dict) and link.get("url"): + crawl_paths.add(_norm_path(str(link["url"]))) + items: list[dict[str, Any]] = [] + for row_data in top_paths if isinstance(top_paths, list) else []: + if not isinstance(row_data, dict): + continue + path = str(row_data.get("path") or "") + hits = int(row_data.get("hits") or 0) + bot_hits = bot_paths.get(path, 0) + if hits < min_hits: + continue + if bot_hits > max_bot_hits: + continue + if path in crawl_paths and bot_hits > 0: + continue + items.append({ + "path": path, + "total_hits": hits, + "googlebot_hits": bot_hits, + "in_crawl": path in crawl_paths, + }) + items.sort(key=lambda x: (-x["total_hits"], x["googlebot_hits"])) + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(items, limit, max_cap=100) + return {"paths": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_log_orphan_high_traffic(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Access-log paths with high hits that map to crawl orphan URLs.""" + scoped = ctx.with_args(args) + row, err = _require_log(conn, ctx, args) + if err: + return err + assert row is not None + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "paths": [], "total": 0, "truncated": False} + orphan_path_set: set[str] = set() + for url in payload.get("orphan_urls") or []: + if not url: + continue + path = _norm_path(str(url)) + orphan_path_set.add(path) + orphan_path_set.add(path.rstrip("/") or "/") + if not orphan_path_set: + return {"paths": [], "total": 0, "truncated": False, "note": "no orphan URLs in report"} + analysis = row.get("analysis") or {} + top_paths = analysis.get("top_paths") or [] + try: + min_hits = int(args.get("min_hits") or 10) + except (TypeError, ValueError): + min_hits = 10 + items: list[dict[str, Any]] = [] + for row_data in top_paths if isinstance(top_paths, list) else []: + if not isinstance(row_data, dict): + continue + path = str(row_data.get("path") or "") + hits = int(row_data.get("hits") or 0) + if hits < min_hits: + continue + if path not in orphan_path_set and path.rstrip("/") not in orphan_path_set: + continue + items.append({"path": path, "hits": hits}) + items.sort(key=lambda x: -x["hits"]) + limit = parse_limit(args.get("limit"), 30, 100) + sliced = cap_list(items, limit, max_cap=100) + return {"paths": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_redirect_chains_by_length(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False} + try: + min_length = int(args.get("min_length") or args.get("chain_length") or _REDIRECT_CHAIN_MIN) + except (TypeError, ValueError): + min_length = _REDIRECT_CHAIN_MIN + pages: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + try: + chain_len = int(rec.get("redirect_chain_length") or 0) + except (TypeError, ValueError): + chain_len = 0 + if chain_len < min_length: + continue + pages.append({ + "url": str(rec.get("url") or ""), + "status": str(rec.get("status") or ""), + "redirect_chain_length": chain_len, + "final_url": str(rec.get("final_url") or ""), + }) + pages.sort(key=lambda p: -int(p.get("redirect_chain_length") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return { + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "min_length": min_length, + } + + +def list_hreflang_reciprocal_gaps(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + """Pages whose hreflang alternates do not link back reciprocally.""" + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "missing": True} + href_map: dict[str, set[str]] = {} + url_by_norm: dict[str, str] = {} + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + url = str(rec.get("url") or "").strip() + if not url: + continue + norm = normalize_url(url) + url_by_norm[norm] = url + pa = _parse_page_analysis(rec) + alts = pa.get("hreflang_alternates") or [] + targets: set[str] = set() + for alt in alts if isinstance(alts, list) else []: + if not isinstance(alt, dict): + continue + href = str(alt.get("href") or "").strip() + if href: + targets.add(normalize_url(href)) + if targets: + href_map[norm] = targets + gaps: list[dict[str, Any]] = [] + for src_norm, targets in href_map.items(): + src_url = url_by_norm.get(src_norm, src_norm) + missing_returns: list[str] = [] + for tgt_norm in targets: + if tgt_norm == src_norm: + continue + tgt_targets = href_map.get(tgt_norm, set()) + if src_norm not in tgt_targets: + missing_returns.append(url_by_norm.get(tgt_norm, tgt_norm)) + if missing_returns: + gaps.append({ + "url": src_url, + "missing_reciprocal_from": missing_returns[:10], + "gap_count": len(missing_returns), + }) + gaps.sort(key=lambda g: -int(g.get("gap_count") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(gaps, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} diff --git a/src/website_profiling/tools/audit_tools/insight_helpers.py b/src/website_profiling/tools/audit_tools/insight_helpers.py index 45431067..e38d09ff 100644 --- a/src/website_profiling/tools/audit_tools/insight_helpers.py +++ b/src/website_profiling/tools/audit_tools/insight_helpers.py @@ -163,7 +163,7 @@ def composite_page_score( issue_flags: list[dict[str, Any]], lighthouse: dict[str, Any] | None, ) -> dict[str, Any]: - score = 70.0 + score = 75.0 flags_out: list[str] = [] site_pos = _num((gsc_site or {}).get("position"), 10) diff --git a/src/website_profiling/tools/audit_tools/issue_lists.py b/src/website_profiling/tools/audit_tools/issue_lists.py new file mode 100644 index 00000000..edd870f9 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/issue_lists.py @@ -0,0 +1,526 @@ +"""Issue and gap list tools from report payload buckets and crawl columns.""" +from __future__ import annotations + +from collections import Counter +from typing import Any, Callable + +import pandas as pd +from psycopg import Connection + +from ...reporting.categories._helpers import ( + RESPONSE_TIME_SLOW_MS, + TITLE_LEN_MAX, + TITLE_LEN_MIN, + _hreflang_issues, + _orphan_hub_suggestions, +) +from ...reporting.categories.accessibility import contrast_issues_from_sources +from ._slice import _parse_page_analysis, _row_schema_types_list, cap_list, parse_limit +from .context import AuditToolContext + +_READING_LEVEL_HIGH = 12.0 +_VERY_THIN_WORDS = 100 + + +def _payload_url_bucket( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + bucket: str, + *, + item_key: str = "pages", +) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", item_key: [], "total": 0, "truncated": False} + content_urls = payload.get("content_urls") or {} + if not isinstance(content_urls, dict): + return {"error": "content_urls not in report", "missing": True, item_key: [], "total": 0, "truncated": False} + items = content_urls.get(bucket) or [] + if not isinstance(items, list): + items = [] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(items, limit, max_cap=50) + return { + "bucket": bucket, + item_key: sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + } + + +def _payload_list_key( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + key: str, + *, + item_key: str = "pages", + nested: str = "", +) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", item_key: [], "total": 0, "truncated": False} + raw = payload.get(key) + if nested and isinstance(raw, dict): + items = raw.get(nested) or [] + elif isinstance(raw, list): + items = raw + else: + items = [] + if not items: + return {"missing": True, item_key: [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(items if isinstance(items, list) else [], limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _filter_crawl_df( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + *, + predicate: Callable[[dict[str, Any]], bool], + projection: Callable[[dict[str, Any]], dict[str, Any]], + only_2xx: bool = True, + item_key: str = "pages", +) -> dict[str, Any]: + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {item_key: [], "total": 0, "truncated": False} + work = df + if only_2xx and "status" in df.columns: + work = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] + pages: list[dict[str, Any]] = [] + for _, row in work.iterrows(): + rec = row.to_dict() + if predicate(rec): + pages.append(projection(rec)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _issues_by_type( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + issue_type: str, + *, + item_key: str = "issues", +) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", item_key: [], "total": 0, "truncated": False} + issues_root = payload.get("issues") or {} + seo = issues_root.get("seo") if isinstance(issues_root, dict) else [] + if not isinstance(seo, list): + seo = [] + needle = issue_type.strip().lower() + filtered = [ + x for x in seo + if isinstance(x, dict) and str(x.get("type") or "").lower() == needle + ] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(filtered, limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _bucket_or_crawl( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + bucket: str, + *, + predicate: Callable[[dict[str, Any]], bool], + projection: Callable[[dict[str, Any]], dict[str, Any]], +) -> dict[str, Any]: + result = _payload_url_bucket(conn, ctx, args, bucket) + if result.get("total", 0) > 0 or result.get("missing"): + return result + return _filter_crawl_df(conn, ctx, args, predicate=predicate, projection=projection) + + +def list_pages_title_too_short(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _short(r: dict[str, Any]) -> bool: + try: + tl = int(r.get("title_length") or 0) + except (TypeError, ValueError): + tl = 0 + return tl > 0 and tl < TITLE_LEN_MIN + + return _bucket_or_crawl( + conn, ctx, args, "title_short", + predicate=_short, + projection=lambda r: { + "url": str(r.get("url") or ""), + "title": str(r.get("title") or ""), + "title_length": int(r.get("title_length") or 0), + }, + ) + + +def list_pages_title_too_long(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _long(r: dict[str, Any]) -> bool: + try: + tl = int(r.get("title_length") or 0) + except (TypeError, ValueError): + tl = 0 + return tl > TITLE_LEN_MAX + + return _bucket_or_crawl( + conn, ctx, args, "title_long", + predicate=_long, + projection=lambda r: { + "url": str(r.get("url") or ""), + "title": str(r.get("title") or ""), + "title_length": int(r.get("title_length") or 0), + }, + ) + + +def list_pages_slow_response(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + try: + threshold = int(args.get("threshold_ms") or RESPONSE_TIME_SLOW_MS) + except (TypeError, ValueError): + threshold = RESPONSE_TIME_SLOW_MS + + def _slow(r: dict[str, Any]) -> bool: + try: + ms = float(r.get("response_time_ms") or 0) + except (TypeError, ValueError): + ms = 0 + return ms >= threshold + + return _bucket_or_crawl( + conn, ctx, args, "slow_response", + predicate=_slow, + projection=lambda r: { + "url": str(r.get("url") or ""), + "response_time_ms": float(r.get("response_time_ms") or 0), + "status": str(r.get("status") or ""), + }, + ) + + +def list_pages_missing_html_lang(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _missing(r: dict[str, Any]) -> bool: + pa = _parse_page_analysis(r) + return not str(pa.get("html_lang") or "").strip() + + return _bucket_or_crawl( + conn, ctx, args, "missing_html_lang", + predicate=_missing, + projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")}, + ) + + +def list_pages_invalid_viewport(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _truthy(val: Any) -> bool: + return str(val or "").lower() in ("true", "1", "yes") + + def _invalid(r: dict[str, Any]) -> bool: + if not _truthy(r.get("viewport_present")): + return False + content = str(r.get("viewport_content") or "").strip() + return not content or "width" not in content.lower() and "device-width" not in content.lower() + + bucket = _payload_url_bucket(conn, ctx, args, "invalid_viewport") + if bucket.get("total", 0) > 0 or bucket.get("missing"): + return bucket + return _filter_crawl_df(conn, ctx, args, predicate=_invalid, projection=lambda r: { + "url": str(r.get("url") or ""), + "viewport_content": str(r.get("viewport_content") or ""), + }) + + +def list_pages_color_contrast_failures(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + df = scoped.load_crawl_df(conn) + lh = payload.get("lighthouse_by_url") if isinstance(payload.get("lighthouse_by_url"), dict) else {} + issues = contrast_issues_from_sources(df if df is not None else pd.DataFrame(), lh) + pages = [ + {"url": str(i.get("url") or ""), "message": str(i.get("message") or "")} + for i in issues if isinstance(i, dict) and i.get("url") + ] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_pages_high_reading_level(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + try: + min_grade = float(args.get("min_reading_level") or _READING_LEVEL_HIGH) + except (TypeError, ValueError): + min_grade = _READING_LEVEL_HIGH + + def _high(r: dict[str, Any]) -> bool: + try: + lvl = float(r.get("reading_level") or 0) + except (TypeError, ValueError): + lvl = 0 + return lvl >= min_grade + + return _bucket_or_crawl( + conn, ctx, args, "high_reading_level", + predicate=_high, + projection=lambda r: { + "url": str(r.get("url") or ""), + "reading_level": float(r.get("reading_level") or 0), + "title": str(r.get("title") or ""), + }, + ) + + +def list_pages_very_thin_content(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + try: + max_words = int(args.get("max_word_count") or _VERY_THIN_WORDS) + except (TypeError, ValueError): + max_words = _VERY_THIN_WORDS + + def _thin(r: dict[str, Any]) -> bool: + try: + wc = int(r.get("word_count") or 0) + except (TypeError, ValueError): + wc = 0 + return 0 < wc < max_words + + bucket = _payload_url_bucket(conn, ctx, args, "very_thin_content") + if bucket.get("total", 0) > 0 or bucket.get("missing"): + return bucket + return _filter_crawl_df( + conn, ctx, args, + predicate=_thin, + projection=lambda r: { + "url": str(r.get("url") or ""), + "word_count": int(r.get("word_count") or 0), + }, + ) + + +def list_hreflang_issue_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + payload_result = _payload_list_key(conn, ctx, args, "hreflang_issue_urls") + if payload_result.get("total", 0) > 0: + return payload_result + scoped = ctx.with_args(args) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "missing": True} + success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + issues = _hreflang_issues(success) + pages = [{"url": str(i.get("url") or ""), "message": str(i.get("message") or "")} for i in issues] + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_pages_missing_og_tags(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + social = payload.get("social_coverage") if isinstance(payload.get("social_coverage"), dict) else {} + urls = social.get("missing_og") or [] + if isinstance(urls, list) and urls: + limit = parse_limit(args.get("limit"), 30, 50) + pages = [{"url": str(u)} for u in urls if u] + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + return _filter_crawl_df( + conn, ctx, args, + predicate=lambda r: not str(r.get("og_title") or "").strip(), + projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")}, + ) + + +def list_pages_missing_twitter_cards(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + social = payload.get("social_coverage") if isinstance(payload.get("social_coverage"), dict) else {} + urls = social.get("missing_twitter") or [] + if isinstance(urls, list) and urls: + limit = parse_limit(args.get("limit"), 30, 50) + pages = [{"url": str(u)} for u in urls if u] + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + return _filter_crawl_df( + conn, ctx, args, + predicate=lambda r: not str(r.get("twitter_card") or "").strip(), + projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")}, + ) + + +def list_pages_invalid_json_ld(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + def _invalid(r: dict[str, Any]) -> bool: + has_schema = str(r.get("has_schema") or "").lower() in ("true", "1", "yes") + types = _row_schema_types_list(r) + return has_schema and not types + + return _filter_crawl_df( + conn, ctx, args, + predicate=_invalid, + projection=lambda r: {"url": str(r.get("url") or ""), "title": str(r.get("title") or "")}, + ) + + +def list_pages_mixed_language(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + df = scoped.load_crawl_df(conn) + if df is None or df.empty: + return {"pages": [], "total": 0, "truncated": False, "missing": True} + lang_summary = payload.get("language_summary") if isinstance(payload.get("language_summary"), dict) else {} + counts = lang_summary.get("counts") if isinstance(lang_summary.get("counts"), dict) else {} + if counts: + dominant = max(counts.items(), key=lambda x: x[1])[0] + else: + dominant = "" + pages: list[dict[str, Any]] = [] + for _, row in df.iterrows(): + rec = row.to_dict() + if not str(rec.get("status") or "").startswith("2"): + continue + lang = str(rec.get("detected_language") or rec.get("language") or "").strip().lower() + if not lang or not dominant: + continue + if lang != str(dominant).lower(): + pages.append({ + "url": str(rec.get("url") or ""), + "language": lang, + "dominant_language": dominant, + }) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_orphan_hub_suggestions(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "suggestions": [], "total": 0, "truncated": False} + orphans = payload.get("orphan_urls") or [] + if not isinstance(orphans, list): + orphans = [] + edges_raw = payload.get("graph_edges") or [] + edges: list[tuple[str, str]] = [] + nodes = payload.get("graph_nodes") or [] + node_urls: dict[int, str] = {} + if isinstance(nodes, list): + for i, n in enumerate(nodes): + if isinstance(n, dict): + node_urls[i] = str(n.get("url") or n.get("id") or "") + elif isinstance(n, str): + node_urls[i] = n + for e in edges_raw: + if isinstance(e, (list, tuple)) and len(e) >= 2: + src = node_urls.get(int(e[0]), str(e[0])) + tgt = node_urls.get(int(e[1]), str(e[1])) + if src and tgt: + edges.append((src, tgt)) + elif isinstance(e, dict): + src = str(e.get("source") or e.get("from") or "") + tgt = str(e.get("target") or e.get("to") or "") + if src and tgt: + edges.append((src, tgt)) + issues = _orphan_hub_suggestions(edges, [str(u) for u in orphans if u]) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(issues, limit, max_cap=50) + return {"suggestions": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _lighthouse_failure_bucket( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + metric: str, +) -> dict[str, Any]: + payload_result = _payload_list_key( + conn, ctx, args, "lighthouse_failure_urls", nested=metric, item_key="pages", + ) + if payload_result.get("total", 0) > 0: + return payload_result + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + lh_by_url = payload.get("lighthouse_by_url") if isinstance(payload.get("lighthouse_by_url"), dict) else {} + pages: list[dict[str, Any]] = [] + metric_key = metric.lower() + for url, summary in lh_by_url.items(): + if not isinstance(summary, dict): + continue + audits = summary.get("audits") if isinstance(summary.get("audits"), dict) else {} + score_val = summary.get(metric_key) or summary.get(metric.upper()) + failed = False + if metric_key in ("lcp", "inp", "cls"): + try: + failed = float(score_val or 0) > 0 and metric_key in str(summary.get("cwv_failures") or "").lower() + except (TypeError, ValueError): + failed = False + if not failed: + for fail in summary.get("top_failures") or []: + if isinstance(fail, dict) and metric_key in str(fail.get("id") or "").lower(): + failed = True + break + elif metric_key == "seo": + try: + failed = float(summary.get("seo") or 100) < 70 + except (TypeError, ValueError): + failed = False + if failed or (metric_key == "seo" and isinstance(score_val, (int, float)) and float(score_val) < 70): + pages.append({"url": str(url), "lighthouse": {metric: score_val}}) + elif metric_key in audits and isinstance(audits[metric_key], dict): + audit = audits[metric_key] + if audit.get("score") is not None and float(audit.get("score") or 1) < 0.9: + pages.append({"url": str(url), "audit": audit.get("title") or metric}) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_lighthouse_failure_lcp(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _lighthouse_failure_bucket(conn, ctx, args, "lcp") + + +def list_lighthouse_failure_inp(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _lighthouse_failure_bucket(conn, ctx, args, "inp") + + +def list_lighthouse_failure_cls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _lighthouse_failure_bucket(conn, ctx, args, "cls") + + +def list_lighthouse_failure_seo(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + try: + threshold = int(args.get("seo_threshold") or 70) + except (TypeError, ValueError): + threshold = 70 + lh_by_url = payload.get("lighthouse_by_url") if isinstance(payload.get("lighthouse_by_url"), dict) else {} + pages: list[dict[str, Any]] = [] + for url, summary in lh_by_url.items(): + if not isinstance(summary, dict): + continue + try: + seo = float(summary.get("seo") or 100) + except (TypeError, ValueError): + seo = 100 + if seo < threshold: + pages.append({"url": str(url), "seo_score": seo}) + pages.sort(key=lambda p: float(p.get("seo_score") or 0)) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(pages, limit, max_cap=50) + return {"pages": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} diff --git a/src/website_profiling/tools/audit_tools/keyword_lists.py b/src/website_profiling/tools/audit_tools/keyword_lists.py new file mode 100644 index 00000000..72ca6b4f --- /dev/null +++ b/src/website_profiling/tools/audit_tools/keyword_lists.py @@ -0,0 +1,558 @@ +"""Keyword list and delta audit tools.""" +from __future__ import annotations + +from collections import defaultdict +from typing import Any, Callable + +from psycopg import Connection + +from ...integrations.google.keyword_enrich import opportunity_clicks +from ...integrations.google.keyword_store import read_keyword_snapshots_for_property +from ._slice import cap_list, parse_limit +from .context import AuditToolContext +from .insight_helpers import _num + + +def _require_property(ctx: AuditToolContext) -> dict[str, Any] | None: + if ctx.property_id is None: + return {"error": "property_id is required for keyword data", "missing": True} + return None + + +def _load_keywords(scoped: AuditToolContext, conn: Connection) -> dict[str, Any] | None: + return scoped.load_keywords(conn) + + +def _keyword_rows(data: dict[str, Any] | None) -> list[dict[str, Any]]: + if not data: + return [] + rows = data.get("rows") or [] + return [r for r in rows if isinstance(r, dict)] + + +def _index_keywords(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for row in rows: + key = str(row.get("keyword") or row.get("normalized") or "").strip().lower() + if key: + out[key] = row + return out + + +def _position(row: dict[str, Any]) -> float | None: + raw = row.get("gsc_position") + if raw is None: + return None + try: + pos = float(raw) + except (TypeError, ValueError): + return None + return pos if pos > 0 else None + + +def _load_keyword_pair( + scoped: AuditToolContext, + conn: Connection, +) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + current = _load_keywords(scoped, conn) + snapshots = read_keyword_snapshots_for_property(conn, scoped.property_id, limit=2) + prior: dict[str, Any] | None = None + if len(snapshots) >= 2: + prior = snapshots[1] + elif len(snapshots) == 1 and current is None: + current = snapshots[0] + return current, prior + + +def _rank_delta_rows( + current: dict[str, Any], + prior: dict[str, Any], + *, + improved: bool, +) -> list[dict[str, Any]]: + curr = _index_keywords(_keyword_rows(current)) + prev = _index_keywords(_keyword_rows(prior)) + deltas: list[dict[str, Any]] = [] + for key, row in curr.items(): + old = prev.get(key) + if not old: + continue + cur_pos = _position(row) + old_pos = _position(old) + if cur_pos is None or old_pos is None: + continue + pos_delta = cur_pos - old_pos + if improved: + if pos_delta >= 0: + continue + elif pos_delta <= 0: + continue + entry = { + "keyword": row.get("keyword") or key, + "gsc_position": cur_pos, + "prior_position": old_pos, + "position_delta": round(pos_delta, 2), + "gsc_clicks": row.get("gsc_clicks"), + "gsc_impressions": row.get("gsc_impressions"), + "gsc_url": row.get("gsc_url"), + } + deltas.append(entry) + deltas.sort(key=lambda r: r.get("position_delta", 0), reverse=not improved) + return deltas + + +def _top_ten_transitions( + current: dict[str, Any], + prior: dict[str, Any], + *, + entered: bool, +) -> list[dict[str, Any]]: + curr = _index_keywords(_keyword_rows(current)) + prev = _index_keywords(_keyword_rows(prior)) + rows: list[dict[str, Any]] = [] + if entered: + for key, row in curr.items(): + cur_pos = _position(row) + if cur_pos is None or cur_pos > 10: + continue + old_pos = _position(prev.get(key, {})) + if old_pos is not None and old_pos <= 10: + continue + rows.append({ + "keyword": row.get("keyword") or key, + "gsc_position": cur_pos, + "prior_position": old_pos, + "gsc_clicks": row.get("gsc_clicks"), + "gsc_impressions": row.get("gsc_impressions"), + }) + else: + for key, old in prev.items(): + old_pos = _position(old) + if old_pos is None or old_pos > 10: + continue + row = curr.get(key, {}) + cur_pos = _position(row) + if cur_pos is not None and cur_pos <= 10: + continue + rows.append({ + "keyword": old.get("keyword") or key, + "prior_position": old_pos, + "gsc_position": cur_pos, + "gsc_clicks": row.get("gsc_clicks") if row else old.get("gsc_clicks"), + "gsc_impressions": row.get("gsc_impressions") if row else old.get("gsc_impressions"), + }) + rows.sort(key=lambda r: -_num(r.get("gsc_impressions"))) + return rows + + +def _keyword_bucket( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + *, + key: str, + item_key: str, + empty_error: str | None = None, +) -> dict[str, Any]: + scoped = ctx.with_args(args) + err = _require_property(scoped) + if err: + return {**err, item_key: [], "total": 0, "truncated": False} + data = _load_keywords(scoped, conn) + if not data: + return {"error": empty_error or "no keyword data found", "missing": True, item_key: [], "total": 0, "truncated": False} + items = data.get(key) or [] + if key == "semantic_keyword_clusters": + payload = scoped.load_payload(conn) + items = payload.get("semantic_keyword_clusters") or items + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(items if isinstance(items, list) else [], limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _filter_keywords( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + predicate: Callable[[dict[str, Any]], bool], + *, + sort_key: Callable[[dict[str, Any]], Any] | None = None, + reverse: bool = True, +) -> dict[str, Any]: + scoped = ctx.with_args(args) + err = _require_property(scoped) + if err: + return {**err, "keywords": [], "total": 0, "truncated": False} + data = _load_keywords(scoped, conn) + if not data: + return {"error": "no keyword data found", "missing": True, "keywords": [], "total": 0, "truncated": False} + matches = [r for r in _keyword_rows(data) if predicate(r)] + if sort_key: + matches.sort(key=sort_key, reverse=reverse) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(matches, limit, max_cap=50) + return {"keywords": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _pair_delta_tool( + conn: Connection, + ctx: AuditToolContext, + args: dict[str, Any], + *, + builder: Callable[[dict[str, Any], dict[str, Any]], list[dict[str, Any]]], + item_key: str, +) -> dict[str, Any]: + scoped = ctx.with_args(args) + err = _require_property(scoped) + if err: + return {**err, item_key: [], "total": 0, "truncated": False} + current, prior = _load_keyword_pair(scoped, conn) + if not current: + return {"error": "no keyword data found", "missing": True, item_key: [], "total": 0, "truncated": False} + if not prior: + return {"error": "no prior keyword snapshot for comparison", "missing": True, item_key: [], "total": 0, "truncated": False} + rows = builder(current, prior) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(rows, limit, max_cap=50) + return {item_key: sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def _serp_features(row: dict[str, Any]) -> list[str]: + raw = row.get("serp_features") + if isinstance(raw, list): + return [str(f).lower() for f in raw if f] + if isinstance(raw, str) and raw.strip(): + return [raw.strip().lower()] + return [] + + +def _has_serp_feature(row: dict[str, Any], *needles: str) -> bool: + features = _serp_features(row) + return any(any(n in f for n in needles) for f in features) + + +def list_keyword_rank_improvements(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _pair_delta_tool( + conn, ctx, args, + builder=lambda cur, prev: _rank_delta_rows(cur, prev, improved=True), + item_key="keywords", + ) + + +def list_keyword_rank_declines(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _pair_delta_tool( + conn, ctx, args, + builder=lambda cur, prev: _rank_delta_rows(cur, prev, improved=False), + item_key="keywords", + ) + + +def list_keywords_new_to_top_10(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _pair_delta_tool( + conn, ctx, args, + builder=lambda cur, prev: _top_ten_transitions(cur, prev, entered=True), + item_key="keywords", + ) + + +def list_keywords_fell_out_of_top_10(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _pair_delta_tool( + conn, ctx, args, + builder=lambda cur, prev: _top_ten_transitions(cur, prev, entered=False), + item_key="keywords", + ) + + +def list_cannibalisation_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _keyword_bucket(conn, ctx, args, key="cannibalisation", item_key="queries") + + +def list_cannibalisation_urls(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + err = _require_property(scoped) + if err: + return {**err, "urls": [], "total": 0, "truncated": False} + data = _load_keywords(scoped, conn) + if not data: + return {"error": "no keyword data found", "missing": True, "urls": [], "total": 0, "truncated": False} + by_url: dict[str, dict[str, Any]] = {} + for issue in data.get("cannibalisation") or []: + if not isinstance(issue, dict): + continue + query = str(issue.get("query") or "") + for page in issue.get("pages") or []: + if not isinstance(page, dict): + continue + url = str(page.get("url") or "").strip() + if not url: + continue + bucket = by_url.setdefault(url, { + "url": url, + "queries": [], + "query_count": 0, + "total_clicks": 0, + "total_impressions": 0, + }) + bucket["queries"].append({ + "query": query, + "position": page.get("position"), + "clicks": page.get("clicks"), + "impressions": page.get("impressions"), + }) + bucket["query_count"] += 1 + bucket["total_clicks"] += int(_num(page.get("clicks"))) + bucket["total_impressions"] += int(_num(page.get("impressions"))) + urls = sorted(by_url.values(), key=lambda r: (-r["query_count"], -r["total_impressions"])) + limit = parse_limit(args.get("limit"), 30, 50) + sliced = cap_list(urls, limit, max_cap=50) + return {"urls": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_misaligned_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _keyword_bucket( + conn, ctx, args, + key="query_page_misalignment", + item_key="misalignments", + ) + + +def list_keywords_by_recommended_action(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + action = str(args.get("recommended_action") or args.get("action") or "").strip().lower() + if not action: + return {"error": "recommended_action is required", "keywords": [], "total": 0, "truncated": False} + return _filter_keywords( + conn, ctx, args, + lambda r: action in str(r.get("recommended_action") or "").lower(), + sort_key=lambda r: _num(r.get("gsc_impressions")), + ) + + +def list_keywords_by_serp_feature(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + feature = str(args.get("serp_feature") or args.get("feature") or "").strip().lower() + if not feature: + return {"error": "serp_feature is required", "keywords": [], "total": 0, "truncated": False} + return _filter_keywords( + conn, ctx, args, + lambda r: _has_serp_feature(r, feature), + sort_key=lambda r: _num(r.get("gsc_impressions")), + ) + + +def _semantic_clusters(scoped: AuditToolContext, conn: Connection) -> list[dict[str, Any]]: + payload = scoped.load_payload(conn) + clusters = payload.get("semantic_keyword_clusters") if isinstance(payload, dict) else [] + if isinstance(clusters, list) and clusters: + return [c for c in clusters if isinstance(c, dict)] + data = _load_keywords(scoped, conn) + if not data: + return [] + fallback = data.get("semantic_keyword_clusters") or [] + return [c for c in fallback if isinstance(c, dict)] if isinstance(fallback, list) else [] + + +def list_semantic_cluster_queries(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + clusters = _semantic_clusters(scoped, conn) + if not clusters: + return {"missing": True, "clusters": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 20, 50) + sliced = cap_list(clusters, limit, max_cap=50) + return {"clusters": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_semantic_cluster_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + err = _require_property(scoped) + if err: + return {**err, "clusters": [], "total": 0, "truncated": False} + clusters = _semantic_clusters(scoped, conn) + if not clusters: + return {"missing": True, "clusters": [], "total": 0, "truncated": False} + kw_to_url: dict[str, str] = {} + for row in _keyword_rows(_load_keywords(scoped, conn)): + kw = str(row.get("keyword") or "").strip().lower() + url = str(row.get("gsc_url") or "").strip() + if kw and url: + kw_to_url[kw] = url + enriched: list[dict[str, Any]] = [] + for cluster in clusters: + keywords = [str(k).strip().lower() for k in (cluster.get("keywords") or []) if k] + pages: dict[str, list[str]] = defaultdict(list) + for kw in keywords: + url = kw_to_url.get(kw) + if url: + pages[url].append(kw) + enriched.append({ + "top_keyword": cluster.get("top_keyword") or cluster.get("representative"), + "cluster_score": cluster.get("cluster_score"), + "keywords": cluster.get("keywords") or [], + "pages": [ + {"url": url, "keywords": kws, "keyword_count": len(kws)} + for url, kws in sorted(pages.items(), key=lambda x: -len(x[1])) + ], + "page_count": len(pages), + }) + limit = parse_limit(args.get("limit"), 20, 50) + sliced = cap_list(enriched, limit, max_cap=50) + return {"clusters": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def get_keyword_opportunity_score(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + err = _require_property(scoped) + if err: + return err + keyword = str(args.get("keyword") or "").strip() + if not keyword: + return {"error": "keyword is required"} + data = _load_keywords(scoped, conn) + if not data: + return {"error": "no keyword data found", "missing": True} + needle = keyword.lower() + row = next( + (r for r in _keyword_rows(data) if str(r.get("keyword") or "").lower() == needle), + None, + ) + if not row: + return {"error": "keyword not found", "keyword": keyword, "missing": True} + pos = _position(row) or 0.0 + impressions = int(_num(row.get("gsc_impressions"))) + opp_clicks = row.get("opportunity_clicks") + if opp_clicks is None and pos > 0: + opp_clicks = opportunity_clicks(impressions, pos, target_pos=3) + score = float(row.get("score") or 0) + traffic_potential = int(_num(row.get("traffic_potential"))) + composite = round( + min(100.0, (float(opp_clicks or 0) * 2) + (traffic_potential / 50.0) + score), + 2, + ) + return { + "keyword": row.get("keyword") or keyword, + "opportunity_score": composite, + "opportunity_clicks": opp_clicks, + "traffic_potential": traffic_potential, + "gsc_position": pos or None, + "gsc_impressions": impressions, + "gsc_clicks": row.get("gsc_clicks"), + "recommended_action": row.get("recommended_action"), + "fetched_at": data.get("fetched_at"), + } + + +def list_keywords_near_page_one(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + try: + min_pos = float(args.get("min_position") or 4) + max_pos = float(args.get("max_position") or 20) + min_impressions = int(args.get("min_impressions") or 50) + except (TypeError, ValueError): + return {"error": "min_position, max_position, and min_impressions must be numbers"} + + def _near(row: dict[str, Any]) -> bool: + pos = _position(row) + if pos is None: + return False + return min_pos <= pos <= max_pos and _num(row.get("gsc_impressions")) >= min_impressions + + return _filter_keywords( + conn, ctx, args, _near, + sort_key=lambda r: (_num(r.get("gsc_impressions")), -(_position(r) or 99)), + ) + + +def list_keywords_high_impression_zero_click(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + min_impressions = parse_limit(args.get("min_impressions"), 100, 1_000_000) + + def _zero_click(row: dict[str, Any]) -> bool: + return int(_num(row.get("gsc_clicks"))) == 0 and int(_num(row.get("gsc_impressions"))) >= min_impressions + + return _filter_keywords( + conn, ctx, args, _zero_click, + sort_key=lambda r: _num(r.get("gsc_impressions")), + ) + + +def list_keywords_by_competition_band(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + try: + min_comp = float(args.get("min_competition") if args.get("min_competition") is not None else 0) + max_comp = float(args.get("max_competition") if args.get("max_competition") is not None else 100) + except (TypeError, ValueError): + return {"error": "min_competition and max_competition must be numbers"} + + def _in_band(row: dict[str, Any]) -> bool: + raw = row.get("serp_estimated_competition") + if raw is None: + return False + try: + val = float(raw) + except (TypeError, ValueError): + return False + return min_comp <= val <= max_comp + + return _filter_keywords( + conn, ctx, args, _in_band, + sort_key=lambda r: float(r.get("serp_estimated_competition") or 0), + reverse=False, + ) + + +def get_keyword_serp_snapshot(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + err = _require_property(scoped) + if err: + return err + keyword = str(args.get("keyword") or "").strip() + if not keyword: + return {"error": "keyword is required"} + data = _load_keywords(scoped, conn) + if not data: + return {"error": "no keyword data found", "missing": True} + needle = keyword.lower() + row = next( + (r for r in _keyword_rows(data) if str(r.get("keyword") or "").lower() == needle), + None, + ) + if not row: + return {"error": "keyword not found", "keyword": keyword, "missing": True} + return { + "keyword": row.get("keyword") or keyword, + "serp_features": row.get("serp_features"), + "serp_estimated_competition": row.get("serp_estimated_competition"), + "serp_organic_count": row.get("serp_organic_count"), + "serp_provenance": row.get("serp_provenance") or "Estimated", + "gsc_position": row.get("gsc_position"), + "gsc_impressions": row.get("gsc_impressions"), + "fetched_at": data.get("fetched_at"), + } + + +def list_keywords_with_ai_overview(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _filter_keywords( + conn, ctx, args, + lambda r: _has_serp_feature(r, "ai_overview", "answer_box", "featured_snippet", "knowledge_graph"), + sort_key=lambda r: _num(r.get("gsc_impressions")), + ) + + +def list_keywords_local_pack(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _filter_keywords( + conn, ctx, args, + lambda r: _has_serp_feature(r, "local_pack", "local", "map"), + sort_key=lambda r: _num(r.get("gsc_impressions")), + ) + + +def list_keywords_question_intent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + return _filter_keywords( + conn, ctx, args, + lambda r: bool(r.get("is_question")), + sort_key=lambda r: _num(r.get("gsc_impressions")), + ) + + +def list_keywords_commercial_intent(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + commercial = {"commercial", "transactional"} + return _filter_keywords( + conn, ctx, args, + lambda r: str(r.get("intent") or "").lower() in commercial, + sort_key=lambda r: _num(r.get("gsc_impressions")), + ) diff --git a/src/website_profiling/tools/audit_tools/link_lists.py b/src/website_profiling/tools/audit_tools/link_lists.py new file mode 100644 index 00000000..0d207896 --- /dev/null +++ b/src/website_profiling/tools/audit_tools/link_lists.py @@ -0,0 +1,208 @@ +"""Link graph list tools from payload link_edges, graph_edges, and PageRank.""" +from __future__ import annotations + +from typing import Any +from urllib.parse import urlparse + +from psycopg import Connection + +from ._slice import cap_list, parse_limit +from .context import AuditToolContext + + +def _norm_url(url: str) -> str: + return str(url or "").strip().rstrip("/").lower() + + +def _load_link_edges(payload: dict[str, Any]) -> list[dict[str, Any]]: + edges = payload.get("link_edges") or [] + if isinstance(edges, list) and edges: + return [e for e in edges if isinstance(e, dict)] + graph = payload.get("graph_edges") or [] + if not isinstance(graph, list): + return [] + converted: list[dict[str, Any]] = [] + for edge in graph: + if isinstance(edge, dict): + converted.append({ + "from_url": edge.get("from") or edge.get("source") or edge.get("from_url"), + "to_url": edge.get("to") or edge.get("target") or edge.get("to_url"), + "link_type": edge.get("link_type") or "internal", + "is_nofollow": bool(edge.get("is_nofollow")), + "rel": edge.get("rel"), + "anchor_text": edge.get("anchor_text") or edge.get("label"), + }) + elif isinstance(edge, (list, tuple)) and len(edge) >= 2: + converted.append({ + "from_url": edge[0], + "to_url": edge[1], + "link_type": "internal", + "is_nofollow": False, + }) + return converted + + +def _pagerank_rows(payload: dict[str, Any]) -> list[dict[str, Any]]: + candidates = payload.get("top_pages") or payload.get("links") or [] + if not isinstance(candidates, list): + return [] + ranked: list[dict[str, Any]] = [] + for rec in candidates: + if not isinstance(rec, dict): + continue + pr = rec.get("pagerank") + if pr is None: + continue + try: + score = float(pr) + except (TypeError, ValueError): + continue + ranked.append({ + "url": rec.get("url"), + "pagerank": round(score, 5), + "inlinks": rec.get("inlinks"), + "outlinks": rec.get("outlinks"), + }) + ranked.sort(key=lambda x: float(x.get("pagerank") or 0)) + return ranked + + +def list_outbound_links(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "links": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 100) + edges = _load_link_edges(payload) + items = [ + { + "from_url": e.get("from_url"), + "to_url": e.get("to_url"), + "anchor_text": e.get("anchor_text"), + "rel": e.get("rel"), + "is_nofollow": bool(e.get("is_nofollow")), + } + for e in edges + if str(e.get("link_type") or "") == "external" + ] + if not items: + start = str(payload.get("start_url") or payload.get("origin") or "").strip() + origin_host = urlparse(start).netloc.lower().lstrip("www.") if start else "" + for e in edges: + to_url = str(e.get("to_url") or "") + host = urlparse(to_url).netloc.lower().lstrip("www.") + if origin_host and host and host != origin_host: + items.append({ + "from_url": e.get("from_url"), + "to_url": to_url, + "anchor_text": e.get("anchor_text"), + "rel": e.get("rel"), + "is_nofollow": bool(e.get("is_nofollow")), + }) + sliced = cap_list(items, limit, max_cap=100) + return {"links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_internal_links_from_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + source = str(args.get("url") or args.get("from_url") or "").strip() + if not source: + return {"error": "url is required", "links": [], "total": 0, "truncated": False} + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "links": [], "total": 0, "truncated": False} + needle = _norm_url(source) + limit = parse_limit(args.get("limit"), 30, 100) + edges = _load_link_edges(payload) + items = [ + { + "from_url": e.get("from_url"), + "to_url": e.get("to_url"), + "anchor_text": e.get("anchor_text"), + "rel": e.get("rel"), + "is_nofollow": bool(e.get("is_nofollow")), + } + for e in edges + if _norm_url(str(e.get("from_url") or "")) == needle + and str(e.get("link_type") or "internal") == "internal" + ] + sliced = cap_list(items, limit, max_cap=100) + return {"url": source, "links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_internal_links_to_url(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + target = str(args.get("url") or args.get("to_url") or args.get("target_url") or "").strip() + if not target: + return {"error": "url is required", "links": [], "total": 0, "truncated": False} + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "links": [], "total": 0, "truncated": False} + needle = _norm_url(target) + limit = parse_limit(args.get("limit"), 30, 100) + edges = _load_link_edges(payload) + items = [ + { + "from_url": e.get("from_url"), + "to_url": e.get("to_url"), + "anchor_text": e.get("anchor_text"), + "rel": e.get("rel"), + "is_nofollow": bool(e.get("is_nofollow")), + } + for e in edges + if _norm_url(str(e.get("to_url") or "")) == needle + and str(e.get("link_type") or "internal") == "internal" + ] + sliced = cap_list(items, limit, max_cap=100) + return {"url": target, "links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"]} + + +def list_links_by_rel_nofollow(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "links": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 100) + rel_filter = str(args.get("rel") or "nofollow").strip().lower() + edges = _load_link_edges(payload) + items = [] + for e in edges: + rel = str(e.get("rel") or "").lower() + is_nf = bool(e.get("is_nofollow")) + if rel_filter == "nofollow" and not is_nf and "nofollow" not in rel: + continue + if rel_filter not in ("nofollow", "") and rel_filter not in rel: + continue + items.append({ + "from_url": e.get("from_url"), + "to_url": e.get("to_url"), + "link_type": e.get("link_type"), + "rel": e.get("rel"), + "is_nofollow": is_nf, + "anchor_text": e.get("anchor_text"), + }) + sliced = cap_list(items, limit, max_cap=100) + return {"links": sliced["items"], "total": sliced["total"], "truncated": sliced["truncated"], "rel_filter": rel_filter} + + +def list_pagerank_low_pages(conn: Connection, ctx: AuditToolContext, args: dict[str, Any]) -> dict[str, Any]: + scoped = ctx.with_args(args) + payload = scoped.load_payload(conn) + if not payload: + return {"error": "no report found", "pages": [], "total": 0, "truncated": False} + limit = parse_limit(args.get("limit"), 30, 50) + try: + max_pr = float(args.get("max_pagerank", 0.01)) + except (TypeError, ValueError): + max_pr = 0.01 + ranked = _pagerank_rows(payload) + if not ranked: + return {"missing": True, "pages": [], "total": 0, "truncated": False, "note": "pagerank not in report"} + low = [r for r in ranked if float(r.get("pagerank") or 0) <= max_pr] + sliced = cap_list(low, limit, max_cap=50) + return { + "pages": sliced["items"], + "total": sliced["total"], + "truncated": sliced["truncated"], + "max_pagerank": max_pr, + } diff --git a/src/website_profiling/tools/audit_tools/registry.py b/src/website_profiling/tools/audit_tools/registry.py index 4eb75f7e..c1b472fa 100644 --- a/src/website_profiling/tools/audit_tools/registry.py +++ b/src/website_profiling/tools/audit_tools/registry.py @@ -6,6 +6,123 @@ from psycopg import Connection from ...db.storage import db_session + +from .backlink_lists import ( + get_anchor_text_distribution, + list_backlinks_by_anchor_text, + list_backlinks_from_domain, + list_backlinks_to_url, + list_referring_domains, +) +from .compare_list_tools import ( + list_compare_lighthouse_regressions, + list_compare_new_issues, + list_compare_new_urls, + list_compare_removed_urls, + list_compare_resolved_issues, + list_compare_traffic_losers, +) +from .content_lists import ( + get_text_content_analysis, + list_amp_validation_issues, + list_duplicate_content_pairs, + list_html_validation_issues, + list_pages_by_word_count_band, + list_pages_containing_keyword, + list_pages_missing_article_schema, + list_pagination_issues, + list_schema_errors_by_type, + list_spell_check_issues, +) +from .geo_list_tools import ( + list_pages_ai_citation_signals, + list_pages_missing_howto_schema, + list_pages_missing_llms_txt_reference, + list_robots_blocked_ai_crawlers, +) +from .google_lists import ( + compare_gsc_periods, + get_ga4_path_trend, + get_gsc_page_trend, + get_gsc_query_trend, + get_gsc_site_benchmarks, + list_ga4_landing_pages, + list_ga4_pages_by_bounce_rate, + list_ga4_pages_by_engagement_rate, + list_gsc_branded_queries, + list_gsc_ctr_underperformers, + list_gsc_decaying_pages, + list_gsc_decaying_queries, + list_gsc_ga4_mismatch_pages, + list_gsc_new_queries, + list_gsc_non_branded_queries, + list_gsc_pages_by_clicks, + list_gsc_pages_by_impressions, + list_gsc_pages_by_position_band, + list_gsc_queries_by_clicks, + list_gsc_queries_by_impressions, +) +from .indexation_lists import ( + list_crawl_urls_not_in_sitemap, + list_hreflang_reciprocal_gaps, + list_indexation_indexed_not_submitted, + list_indexation_submitted_not_indexed, + list_log_5xx_paths, + list_log_googlebot_low_crawl, + list_log_orphan_high_traffic, + list_log_paths_by_hits, + list_redirect_chains_by_length, + list_sitemap_urls_not_in_crawl, +) +from .issue_lists import ( + list_hreflang_issue_pages, + list_lighthouse_failure_cls, + list_lighthouse_failure_inp, + list_lighthouse_failure_lcp, + list_lighthouse_failure_seo, + list_orphan_hub_suggestions, + list_pages_color_contrast_failures, + list_pages_high_reading_level, + list_pages_invalid_json_ld, + list_pages_invalid_viewport, + list_pages_missing_html_lang, + list_pages_missing_og_tags, + list_pages_missing_twitter_cards, + list_pages_mixed_language, + list_pages_slow_response, + list_pages_title_too_long, + list_pages_title_too_short, + list_pages_very_thin_content, +) +from .keyword_lists import ( + get_keyword_opportunity_score, + get_keyword_serp_snapshot, + list_cannibalisation_queries, + list_cannibalisation_urls, + list_keyword_rank_declines, + list_keyword_rank_improvements, + list_keywords_by_competition_band, + list_keywords_by_recommended_action, + list_keywords_by_serp_feature, + list_keywords_commercial_intent, + list_keywords_fell_out_of_top_10, + list_keywords_high_impression_zero_click, + list_keywords_local_pack, + list_keywords_near_page_one, + list_keywords_new_to_top_10, + list_keywords_question_intent, + list_keywords_with_ai_overview, + list_misaligned_queries, + list_semantic_cluster_pages, + list_semantic_cluster_queries, +) +from .link_lists import ( + list_internal_links_from_url, + list_internal_links_to_url, + list_links_by_rel_nofollow, + list_outbound_links, + list_pagerank_low_pages, +) from .backlinks import ( get_backlinks_velocity, get_bing_backlinks_summary, @@ -117,6 +234,8 @@ list_broken_links, list_pages_by_fetch_method, list_pages_with_console_errors, + list_pages_console_errors_by_type, + list_pages_js_rendering_delta, list_redirects, list_status_4xx_pages, list_status_5xx_pages, @@ -538,6 +657,106 @@ "get_gsc_page_queries": get_gsc_page_queries, "get_brand_keyword_split": get_brand_keyword_split, "list_keywords_by_intent": list_keywords_by_intent, + "list_pages_title_too_short": list_pages_title_too_short, + "list_pages_title_too_long": list_pages_title_too_long, + "list_pages_slow_response": list_pages_slow_response, + "list_pages_missing_html_lang": list_pages_missing_html_lang, + "list_pages_invalid_viewport": list_pages_invalid_viewport, + "list_pages_color_contrast_failures": list_pages_color_contrast_failures, + "list_pages_high_reading_level": list_pages_high_reading_level, + "list_pages_very_thin_content": list_pages_very_thin_content, + "list_hreflang_issue_pages": list_hreflang_issue_pages, + "list_pages_missing_og_tags": list_pages_missing_og_tags, + "list_pages_missing_twitter_cards": list_pages_missing_twitter_cards, + "list_pages_invalid_json_ld": list_pages_invalid_json_ld, + "list_pages_mixed_language": list_pages_mixed_language, + "list_orphan_hub_suggestions": list_orphan_hub_suggestions, + "list_lighthouse_failure_lcp": list_lighthouse_failure_lcp, + "list_lighthouse_failure_inp": list_lighthouse_failure_inp, + "list_lighthouse_failure_cls": list_lighthouse_failure_cls, + "list_lighthouse_failure_seo": list_lighthouse_failure_seo, + "list_gsc_pages_by_impressions": list_gsc_pages_by_impressions, + "list_gsc_pages_by_clicks": list_gsc_pages_by_clicks, + "list_gsc_queries_by_impressions": list_gsc_queries_by_impressions, + "list_gsc_queries_by_clicks": list_gsc_queries_by_clicks, + "list_gsc_ctr_underperformers": list_gsc_ctr_underperformers, + "list_gsc_decaying_pages": list_gsc_decaying_pages, + "list_gsc_decaying_queries": list_gsc_decaying_queries, + "list_gsc_new_queries": list_gsc_new_queries, + "list_ga4_landing_pages": list_ga4_landing_pages, + "list_ga4_pages_by_bounce_rate": list_ga4_pages_by_bounce_rate, + "list_ga4_pages_by_engagement_rate": list_ga4_pages_by_engagement_rate, + "get_gsc_query_trend": get_gsc_query_trend, + "get_gsc_page_trend": get_gsc_page_trend, + "get_ga4_path_trend": get_ga4_path_trend, + "list_gsc_ga4_mismatch_pages": list_gsc_ga4_mismatch_pages, + "list_gsc_pages_by_position_band": list_gsc_pages_by_position_band, + "get_gsc_site_benchmarks": get_gsc_site_benchmarks, + "list_gsc_branded_queries": list_gsc_branded_queries, + "list_gsc_non_branded_queries": list_gsc_non_branded_queries, + "compare_gsc_periods": compare_gsc_periods, + "list_keyword_rank_improvements": list_keyword_rank_improvements, + "list_keyword_rank_declines": list_keyword_rank_declines, + "list_keywords_new_to_top_10": list_keywords_new_to_top_10, + "list_keywords_fell_out_of_top_10": list_keywords_fell_out_of_top_10, + "list_cannibalisation_queries": list_cannibalisation_queries, + "list_cannibalisation_urls": list_cannibalisation_urls, + "list_misaligned_queries": list_misaligned_queries, + "list_keywords_by_recommended_action": list_keywords_by_recommended_action, + "list_keywords_by_serp_feature": list_keywords_by_serp_feature, + "list_semantic_cluster_queries": list_semantic_cluster_queries, + "list_semantic_cluster_pages": list_semantic_cluster_pages, + "get_keyword_opportunity_score": get_keyword_opportunity_score, + "list_keywords_near_page_one": list_keywords_near_page_one, + "list_keywords_high_impression_zero_click": list_keywords_high_impression_zero_click, + "list_keywords_by_competition_band": list_keywords_by_competition_band, + "get_keyword_serp_snapshot": get_keyword_serp_snapshot, + "list_keywords_with_ai_overview": list_keywords_with_ai_overview, + "list_keywords_local_pack": list_keywords_local_pack, + "list_keywords_question_intent": list_keywords_question_intent, + "list_keywords_commercial_intent": list_keywords_commercial_intent, + "list_referring_domains": list_referring_domains, + "list_backlinks_by_anchor_text": list_backlinks_by_anchor_text, + "list_backlinks_to_url": list_backlinks_to_url, + "list_backlinks_from_domain": list_backlinks_from_domain, + "get_anchor_text_distribution": get_anchor_text_distribution, + "get_text_content_analysis": get_text_content_analysis, + "list_pages_containing_keyword": list_pages_containing_keyword, + "list_pages_by_word_count_band": list_pages_by_word_count_band, + "list_duplicate_content_pairs": list_duplicate_content_pairs, + "list_spell_check_issues": list_spell_check_issues, + "list_html_validation_issues": list_html_validation_issues, + "list_amp_validation_issues": list_amp_validation_issues, + "list_pagination_issues": list_pagination_issues, + "list_schema_errors_by_type": list_schema_errors_by_type, + "list_pages_missing_article_schema": list_pages_missing_article_schema, + "list_outbound_links": list_outbound_links, + "list_internal_links_from_url": list_internal_links_from_url, + "list_internal_links_to_url": list_internal_links_to_url, + "list_links_by_rel_nofollow": list_links_by_rel_nofollow, + "list_pagerank_low_pages": list_pagerank_low_pages, + "list_indexation_submitted_not_indexed": list_indexation_submitted_not_indexed, + "list_indexation_indexed_not_submitted": list_indexation_indexed_not_submitted, + "list_sitemap_urls_not_in_crawl": list_sitemap_urls_not_in_crawl, + "list_crawl_urls_not_in_sitemap": list_crawl_urls_not_in_sitemap, + "list_log_paths_by_hits": list_log_paths_by_hits, + "list_log_5xx_paths": list_log_5xx_paths, + "list_log_googlebot_low_crawl": list_log_googlebot_low_crawl, + "list_log_orphan_high_traffic": list_log_orphan_high_traffic, + "list_redirect_chains_by_length": list_redirect_chains_by_length, + "list_hreflang_reciprocal_gaps": list_hreflang_reciprocal_gaps, + "list_compare_new_issues": list_compare_new_issues, + "list_compare_resolved_issues": list_compare_resolved_issues, + "list_compare_new_urls": list_compare_new_urls, + "list_compare_removed_urls": list_compare_removed_urls, + "list_compare_lighthouse_regressions": list_compare_lighthouse_regressions, + "list_compare_traffic_losers": list_compare_traffic_losers, + "list_pages_missing_howto_schema": list_pages_missing_howto_schema, + "list_pages_ai_citation_signals": list_pages_ai_citation_signals, + "list_pages_missing_llms_txt_reference": list_pages_missing_llms_txt_reference, + "list_robots_blocked_ai_crawlers": list_robots_blocked_ai_crawlers, + "list_pages_console_errors_by_type": list_pages_console_errors_by_type, + "list_pages_js_rendering_delta": list_pages_js_rendering_delta, } diff --git a/src/website_profiling/tools/audit_tools/tool_catalog.py b/src/website_profiling/tools/audit_tools/tool_catalog.py index a85e89ef..facbcad9 100644 --- a/src/website_profiling/tools/audit_tools/tool_catalog.py +++ b/src/website_profiling/tools/audit_tools/tool_catalog.py @@ -401,4 +401,106 @@ def _tool(name: str, description: str, properties: dict[str, Any], required: lis # Keyword dimensions _tool("get_brand_keyword_split", "Branded vs non-branded keyword counts and samples.", {"property_id": _PID}, ["property_id"]), _tool("list_keywords_by_intent", "Keywords filtered by intent (informational, commercial, etc.).", {"property_id": _PID, "intent": {"type": "string"}, "limit": _LIMIT}, ["property_id", "intent"]), + + # --- 100 new audit tools (240→340) --- + _tool("list_pages_title_too_short", "Pages with title shorter than SEO minimum.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_title_too_long", "Pages with title longer than SEO maximum.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_slow_response", "Pages with server response time above threshold.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_missing_html_lang", "Pages missing html lang attribute.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_invalid_viewport", "Pages missing or invalid viewport meta.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_color_contrast_failures", "Pages failing color contrast checks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_high_reading_level", "Pages with high reading grade level.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_very_thin_content", "Pages with very low word count.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_hreflang_issue_pages", "Pages with hreflang cluster issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_missing_og_tags", "Pages missing Open Graph tags.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_missing_twitter_cards", "Pages missing Twitter card tags.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_invalid_json_ld", "Pages with invalid JSON-LD schema.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_mixed_language", "Pages with mixed-language content signals.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_orphan_hub_suggestions", "Suggested hub pages to link orphan URLs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_lighthouse_failure_lcp", "URLs failing Lighthouse LCP audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_lighthouse_failure_inp", "URLs failing Lighthouse INP audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_lighthouse_failure_cls", "URLs failing Lighthouse CLS audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_lighthouse_failure_seo", "URLs failing Lighthouse SEO audit.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_pages_by_impressions", "GSC pages sorted by impressions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_pages_by_clicks", "GSC pages sorted by clicks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_queries_by_impressions", "GSC queries sorted by impressions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_queries_by_clicks", "GSC queries sorted by clicks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_ctr_underperformers", "Queries/pages with low CTR for position band.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_decaying_pages", "GSC pages with click decline vs prior period.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_decaying_queries", "GSC queries with click decline vs prior period.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_new_queries", "New GSC queries vs prior period.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_ga4_landing_pages", "GA4 landing pages by sessions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_ga4_pages_by_bounce_rate", "GA4 pages with high bounce rate.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_ga4_pages_by_engagement_rate", "GA4 pages sorted by engagement rate.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("get_gsc_query_trend", "Daily GSC clicks/impressions for one query.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "query": {'type': 'string'}}, ['query']), + _tool("get_gsc_page_trend", "Daily GSC clicks/impressions for one URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Page URL'}}, ['url']), + _tool("get_ga4_path_trend", "GA4 sessions trend for one path.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "path": {'type': 'string'}}, ['path']), + _tool("list_gsc_ga4_mismatch_pages", "High GSC clicks but low GA4 sessions.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_pages_by_position_band", "GSC pages in position range.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_position": {'type': 'number'}, "max_position": {'type': 'number'}}), + _tool("get_gsc_site_benchmarks", "Sitewide median CTR and position benchmarks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_branded_queries", "Branded GSC queries.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_gsc_non_branded_queries", "Non-branded GSC queries.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("compare_gsc_periods", "GSC summary delta vs prior Google snapshot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_keyword_rank_improvements", "Keywords with improved position vs prior snapshot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keyword_rank_declines", "Keywords with declined position vs prior snapshot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_new_to_top_10", "Keywords newly in top 10.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_fell_out_of_top_10", "Keywords that fell out of top 10.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_cannibalisation_queries", "Queries with multiple ranking URLs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_cannibalisation_urls", "URLs involved in keyword cannibalisation.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_misaligned_queries", "Queries landing on misaligned URLs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_by_recommended_action", "Keywords filtered by recommended_action.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "recommended_action": {'type': 'string'}}, ['property_id']), + _tool("list_keywords_by_serp_feature", "Keywords with given SERP feature.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "serp_feature": {'type': 'string'}}, ['property_id']), + _tool("list_semantic_cluster_queries", "Queries in a semantic keyword cluster.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "cluster_id": {'type': 'string'}}), + _tool("list_semantic_cluster_pages", "Pages in a semantic keyword cluster.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "cluster_id": {'type': 'string'}}), + _tool("get_keyword_opportunity_score", "Composite opportunity score for one keyword.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "keyword": {'type': 'string'}}, ['keyword', 'property_id']), + _tool("list_keywords_near_page_one", "Keywords in striking-distance position band.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_high_impression_zero_click", "High-impression keywords with zero clicks.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_by_competition_band", "Keywords by SERP competition band.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_competition": {'type': 'number'}, "max_competition": {'type': 'number'}}, ['property_id']), + _tool("get_keyword_serp_snapshot", "SERP overlay fields for one keyword.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "keyword": {'type': 'string'}}, ['keyword', 'property_id']), + _tool("list_keywords_with_ai_overview", "Keywords triggering AI overview SERP features.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_local_pack", "Keywords with local pack SERP feature.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_question_intent", "Question-intent keywords.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_keywords_commercial_intent", "Commercial-intent keywords.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_referring_domains", "Top referring domains from GSC Links.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("list_backlinks_by_anchor_text", "Backlinks filtered by anchor text.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "anchor_text": {'type': 'string'}}, ['property_id']), + _tool("list_backlinks_to_url", "Inbound links to target URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Target URL'}}, ['url', 'property_id']), + _tool("list_backlinks_from_domain", "Backlinks from referring domain.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "domain": {'type': 'string'}}, ['domain', 'property_id']), + _tool("get_anchor_text_distribution", "Sitewide anchor text frequency from GSC Links.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}, ['property_id']), + _tool("get_text_content_analysis", "Sitewide text content analysis summary.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_containing_keyword", "Pages containing keyword in body.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "keyword": {'type': 'string'}}, ['keyword']), + _tool("list_pages_by_word_count_band", "Pages filtered by word count range.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_words": {'type': 'integer'}, "max_words": {'type': 'integer'}}), + _tool("list_duplicate_content_pairs", "Near-duplicate content pairs.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_spell_check_issues", "Spell-check optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_html_validation_issues", "HTML validation optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_amp_validation_issues", "AMP validation optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pagination_issues", "Pagination optional audit issues.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_schema_errors_by_type", "Schema validation errors by type.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "schema_type": {'type': 'string'}}), + _tool("list_pages_missing_article_schema", "Article/blog URLs missing Article schema.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_outbound_links", "Outbound links from crawl graph.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_internal_links_from_url", "Internal outlinks from source URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Source URL'}}, ['url']), + _tool("list_internal_links_to_url", "Internal inlinks to target URL.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "url": {'type': 'string', 'description': 'Target URL'}}, ['url']), + _tool("list_links_by_rel_nofollow", "Links filtered by rel attribute.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "rel": {'type': 'string'}}), + _tool("list_pagerank_low_pages", "Pages with lowest internal PageRank.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "max_pagerank": {'type': 'number'}}), + _tool("list_indexation_submitted_not_indexed", "URLs submitted but not indexed.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_indexation_indexed_not_submitted", "Indexed URLs not in sitemap.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_sitemap_urls_not_in_crawl", "Sitemap URLs missing from crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_crawl_urls_not_in_sitemap", "Crawled URLs missing from sitemap.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_log_paths_by_hits", "Access log paths ranked by hits.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_log_5xx_paths", "Access log paths with 5xx responses.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_log_googlebot_low_crawl", "High-value paths under-crawled by Googlebot.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_log_orphan_high_traffic", "Orphan URLs with high log traffic.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_redirect_chains_by_length", "Redirect chains above min_length.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "min_length": {'type': 'integer'}}), + _tool("list_hreflang_reciprocal_gaps", "Hreflang alternates missing reciprocal return tags.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_compare_new_issues", "Issues new since baseline report.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']), + _tool("list_compare_resolved_issues", "Issues resolved since baseline report.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']), + _tool("list_compare_new_urls", "URLs new since baseline crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']), + _tool("list_compare_removed_urls", "URLs removed since baseline crawl.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']), + _tool("list_compare_lighthouse_regressions", "Lighthouse score regressions vs baseline.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']), + _tool("list_compare_traffic_losers", "GSC click losers vs baseline report.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "baseline_report_id": {'type': 'integer', 'description': 'Baseline report ID'}}, ['baseline_report_id']), + _tool("list_pages_missing_howto_schema", "How-to URLs missing HowTo schema.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_ai_citation_signals", "Pages with AEO/AI citation readiness signals.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_missing_llms_txt_reference", "Pages not referenced in llms.txt.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_robots_blocked_ai_crawlers", "Pages blocking AI crawler user-agents.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), + _tool("list_pages_console_errors_by_type", "Console errors filtered by error_type.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT, "error_type": {'type': 'string'}}), + _tool("list_pages_js_rendering_delta", "URLs with static vs rendered content delta.", {"property_id": _PID, "report_id": _RID, "limit": _LIMIT}), ] diff --git a/src/website_profiling/tools/audit_tools/tool_domains.py b/src/website_profiling/tools/audit_tools/tool_domains.py index aecb51b4..6f69d672 100644 --- a/src/website_profiling/tools/audit_tools/tool_domains.py +++ b/src/website_profiling/tools/audit_tools/tool_domains.py @@ -102,6 +102,35 @@ "list_keywords_ctr_opportunity": "ctr", "analyze_serp_snippet_for_url": "ctr", "compare_reports": "drift", + "compare_gsc_periods": "google", + "list_pages_title_too_short": "onpage", + "list_pages_title_too_long": "onpage", + "list_pages_slow_response": "performance", + "list_pages_color_contrast_failures": "accessibility", + "list_pages_high_reading_level": "content", + "list_pages_very_thin_content": "content", + "list_hreflang_issue_pages": "indexation", + "list_pages_mixed_language": "content", + "list_misaligned_queries": "keywords", + "list_referring_domains": "backlinks", + "get_anchor_text_distribution": "backlinks", + "list_backlinks_by_anchor_text": "backlinks", + "list_backlinks_to_url": "backlinks", + "list_backlinks_from_domain": "backlinks", + "get_keyword_opportunity_score": "keywords", + "list_sitemap_urls_not_in_crawl": "indexation", + "list_crawl_urls_not_in_sitemap": "indexation", + "list_log_googlebot_low_crawl": "ops", + "list_redirect_chains_by_length": "crawl", + "list_compare_new_issues": "drift", + "list_compare_resolved_issues": "drift", + "list_compare_lighthouse_regressions": "drift", + "list_pages_ai_citation_signals": "geo", + "list_pages_missing_llms_txt_reference": "geo", + "list_robots_blocked_ai_crawlers": "geo", + "list_pages_missing_howto_schema": "geo", + "list_pages_missing_article_schema": "geo", + "list_gsc_ctr_underperformers": "google", } _ONPAGE_PREFIXES = ( @@ -154,6 +183,8 @@ def classify_tool_domain(name: str) -> str: return _DOMAIN_OVERRIDES[name] if name.startswith("compare_"): return "drift" + if name.startswith("list_compare_"): + return "drift" if name in TIER_0_TOOLS: return _DOMAIN_OVERRIDES.get(name, "core") diff --git a/src/website_profiling/tools/audit_tools/tool_selector.py b/src/website_profiling/tools/audit_tools/tool_selector.py index 69b87c0b..949ff7e4 100644 --- a/src/website_profiling/tools/audit_tools/tool_selector.py +++ b/src/website_profiling/tools/audit_tools/tool_selector.py @@ -27,7 +27,7 @@ "security": ("security", "tls", "hsts", "ssl"), "indexation": ("indexation", "sitemap", "hreflang", "indexed"), "content": ("duplicate content", "thin content", "word count", "readability"), - "ops": ("access log", "log analysis", "log upload", "crawl run", "integration status"), + "ops": ("access log", "log analysis", "log upload", "crawl run", "integration status", "5xx", "googlebot"), "portfolio": ("overview", "health score", "category scores", "executive", "portfolio", "audit summary"), "ctr": ("ctr", "snippet", "title meta ctr"), } @@ -38,9 +38,13 @@ "export": ("export_audit_report", "export_list_as_csv"), "issues": ("get_critical_issues", "get_issue_priority_breakdown", "list_issues"), "portfolio": ("get_category_scores", "list_audit_categories"), - "performance": ("get_lighthouse_summary",), - "drift": ("compare_reports", "compare_issue_deltas"), - "google": ("get_gsc_top_queries", "get_ga4_page_metrics"), + "performance": ("get_lighthouse_summary", "list_pages_slow_response", "list_lighthouse_failure_lcp"), + "drift": ("compare_reports", "compare_issue_deltas", "list_compare_traffic_losers"), + "google": ("get_gsc_top_queries", "get_ga4_page_metrics", "list_gsc_decaying_queries", "list_gsc_decaying_pages"), + "keywords": ("get_striking_distance_keywords", "get_keyword_cannibalisation", "list_keyword_rank_declines"), + "indexation": ("list_hreflang_issue_pages", "list_indexation_gaps"), + "backlinks": ("list_referring_domains", "list_backlinks_by_anchor_text"), + "ops": ("list_log_paths_by_hits", "list_log_5xx_paths"), } diff --git a/tests/test_audit_tools_batch100_coverage.py b/tests/test_audit_tools_batch100_coverage.py new file mode 100644 index 00000000..27572834 --- /dev/null +++ b/tests/test_audit_tools_batch100_coverage.py @@ -0,0 +1,1686 @@ +"""100% line-coverage tests for batch-100 audit list tool modules.""" +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest +import requests + +from website_profiling.tools.audit_tools import crawl as crawl_mod +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx +from website_profiling.tools.audit_tools import issue_lists as issue_mod +from website_profiling.tools.audit_tools import google_lists as google_mod +from website_profiling.tools.audit_tools import keyword_lists as kw_mod +from website_profiling.tools.audit_tools import backlink_lists as bl_mod +from website_profiling.tools.audit_tools import content_lists as content_mod +from website_profiling.tools.audit_tools import link_lists as link_mod +from website_profiling.tools.audit_tools import indexation_lists as idx_mod +from website_profiling.tools.audit_tools import compare_list_tools as cmp_mod +from website_profiling.tools.audit_tools import geo_list_tools as geo_list_mod + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +@pytest.fixture +def ctx() -> Ctx: + return Ctx(property_id=1, report_id=1) + + +def _payload() -> dict: + return { + "start_url": "https://ex.com/", + "origin": "https://ex.com/", + "orphan_urls": ["https://ex.com/orphan", "https://ex.com/hub-target"], + "top_pages": [ + {"url": "https://ex.com/", "inlinks": 5, "pagerank": 0.05, "outlinks": 2}, + {"url": "https://ex.com/blog/post", "inlinks": 1, "pagerank": 0.001, "outlinks": 1}, + ], + "links": [{"url": "https://ex.com/orphan", "inlinks": 0}], + "graph_nodes": [ + {"url": "https://ex.com/hub", "id": "hub"}, + "https://ex.com/orphan", + ], + "graph_edges": [ + [0, 1], + {"source": "https://ex.com/hub", "target": "https://ex.com/orphan"}, + ], + "link_edges": [ + { + "from_url": "https://ex.com/", + "to_url": "https://external.com/page", + "link_type": "external", + "anchor_text": "ext", + "rel": "nofollow", + "is_nofollow": True, + }, + { + "from_url": "https://ex.com/", + "to_url": "https://ex.com/about", + "link_type": "internal", + "anchor_text": "about", + "rel": "", + "is_nofollow": False, + }, + { + "from_url": "https://ex.com/about", + "to_url": "https://ex.com/", + "link_type": "internal", + "anchor_text": "home", + "rel": "ugc nofollow", + "is_nofollow": True, + }, + ], + "content_urls": { + "title_short": [{"url": "https://ex.com/short", "title": "Hi", "title_length": 2}], + "title_long": [{"url": "https://ex.com/long", "title": "x" * 80, "title_length": 80}], + "slow_response": [{"url": "https://ex.com/slow", "response_time_ms": 5000, "status": "200"}], + "missing_html_lang": [{"url": "https://ex.com/nolang"}], + "invalid_viewport": [{"url": "https://ex.com/badvp"}], + "high_reading_level": [{"url": "https://ex.com/hard", "reading_level": 14}], + "very_thin_content": [{"url": "https://ex.com/thin", "word_count": 50}], + }, + "hreflang_issue_urls": [{"url": "https://ex.com/href", "message": "missing return"}], + "lighthouse_failure_urls": { + "lcp": ["https://ex.com/lcp-fail"], + "inp": ["https://ex.com/inp-fail"], + "cls": ["https://ex.com/cls-fail"], + "seo": ["https://ex.com/seo-fail"], + }, + "optional_audit_urls": { + "spell": [{"url": "https://ex.com/spell", "message": "typo"}], + "html": [{"url": "https://ex.com/html", "message": "invalid html"}], + "amp": [{"url": "https://ex.com/amp", "message": "amp error"}], + "pagination": [{"url": "https://ex.com/pag", "message": "pagination rel=next"}], + }, + "indexation_coverage": { + "lists": { + "sitemap_only": ["https://ex.com/sitemap-only", "https://ex.com/s2"], + "crawled_not_in_sitemap": ["https://ex.com/crawl-only"], + }, + "lists_total": {"sitemap_only": 2, "crawled_not_in_sitemap": 1}, + }, + "social_coverage": { + "missing_og": ["https://ex.com/no-og"], + "missing_twitter": ["https://ex.com/no-tw"], + }, + "language_summary": {"counts": {"en": 8, "fr": 2}}, + "text_content_analysis": { + "keyword_index": [ + { + "word": "widgets", + "top_pages": [ + {"url": "https://ex.com/", "count": 3}, + ["https://ex.com/about", 2], + "skip", + ], + } + ], + }, + "content_analytics": {"keyword_index": [{"word": "fallback", "top_pages": []}]}, + "content_duplicates": [ + { + "id": "c1", + "representative_url": "https://ex.com/a", + "member_urls": ["https://ex.com/a", "https://ex.com/b"], + "similarity": 0.95, + }, + "skip", + ], + "rich_results_validation": [ + {"url": "https://ex.com/", "status": "pass", "type": "Organization"}, + {"url": "https://ex.com/bad", "status": "fail", "schema_type": "FAQPage", "message": "invalid"}, + ], + "semantic_keyword_clusters": [ + {"top_keyword": "widgets", "keywords": ["widgets", "widget repair"], "cluster_score": 0.8}, + ], + "issues": { + "seo": [ + {"type": "missing_title", "url": "https://ex.com/x", "message": "Missing title"}, + {"type": "OTHER", "url": "https://ex.com/y"}, + "skip", + ], + }, + "categories": [ + { + "issues": [ + {"message": "Spell check found typo on page"}, + {"message": "HTML markup validation failed"}, + {"message": "AMP validation issue detected"}, + {"message": "pagination rel=prev missing on series"}, + {"message": "custom audit needle found"}, + ], + }, + "skip", + ], + "lighthouse_by_url": { + "https://ex.com/lcp-live": { + "lcp": 4.5, + "cwv_failures": "lcp", + "top_failures": [{"id": "largest-contentful-paint"}], + "audits": {"lcp": {"score": 0.5, "title": "LCP slow"}}, + }, + "https://ex.com/inp-live": { + "inp": 500, + "top_failures": [{"id": "interaction-to-next-paint"}], + }, + "https://ex.com/cls-live": {"cls": 0.3, "cwv_failures": "cls"}, + "https://ex.com/seo-live": {"seo": 55, "audits": {"seo": {"score": 0.6, "title": "SEO"}}}, + "https://ex.com/contrast": { + "audits": { + "color-contrast": {"score": 0.5, "title": "Contrast"}, + }, + }, + }, + "google": { + "fetched_at": "2026-06-01", + "gsc": { + "daily": [ + {"query": "widgets", "date": "2026-06-01", "clicks": 1}, + {"page": "https://ex.com/", "date": "2026-06-01", "clicks": 2}, + ], + }, + "gsc_full": { + "summary": {"clicks": 100, "impressions": 1000, "ctr": 0.1, "position": 5}, + "pages": [ + {"page": "https://ex.com/", "clicks": 50, "impressions": 500, "ctr": "0.5%", "position": 5}, + {"page": "https://ex.com/low-ctr", "clicks": 1, "impressions": 800, "ctr": "0.1%", "position": 4}, + {"page": "https://ex.com/high-ctr", "clicks": 100, "impressions": 500, "ctr": "20%", "position": 3}, + {"page": "https://ex.com/band", "clicks": 5, "impressions": 100, "ctr": "5%", "position": 12}, + ], + "queries": [ + {"query": "widgets", "clicks": 10, "impressions": 200, "position": 6}, + {"query": "new query", "clicks": 2, "impressions": 50, "position": 8}, + ], + "daily": [ + {"query": "widgets", "date": "2026-06-01", "clicks": 1}, + {"page": "https://ex.com/", "date": "2026-06-01", "clicks": 2}, + ], + }, + "ga4_full": { + "summary": {"sessions": 200, "users": 150}, + "top_pages": [ + {"path": "/", "sessions": 100, "bounceRate": 0.8, "engagementRate": 0.2}, + {"path": "/mismatch", "sessions": 50, "bounceRate": 0.9, "engagementRate": 0.1}, + ], + "by_path": { + "/alt": {"sessions": 30, "bounceRate": 0.7, "engagementRate": 0.3}, + }, + "daily": [{"path": "/", "date": "2026-06-01", "sessions": 10}], + }, + }, + } + + +def _prior_google() -> dict: + return { + "fetched_at": "2026-05-01", + "gsc_full": { + "summary": {"clicks": 80, "impressions": 900, "ctr": 0.09, "position": 6}, + "pages": [ + {"page": "https://ex.com/", "clicks": 60, "impressions": 400, "position": 4}, + {"page": "https://ex.com/loser", "clicks": 30, "impressions": 200, "position": 5}, + ], + "queries": [ + {"query": "widgets", "clicks": 15, "impressions": 180, "position": 4}, + {"query": "old query", "clicks": 5, "impressions": 40, "position": 12}, + ], + }, + "ga4_full": {"summary": {"sessions": 180, "users": 140}}, + } + + +def _google_mismatch() -> dict: + data = _payload()["google"].copy() + data["gsc"] = { + "daily": [ + {"page": "https://ex.com/", "date": "2026-06-01", "clicks": 2}, + {"page": "https://ex.com/mismatch-gsc", "date": "2026-06-01", "clicks": 1}, + ], + } + gsc = dict(data["gsc_full"]) + ga4 = dict(data["ga4_full"]) + gsc["by_page"] = { + "https://ex.com/mismatch-gsc": {"clicks": 20, "impressions": 100}, + "https://ex.com/ga4-only": {"clicks": 0, "impressions": 0}, + } + ga4["by_path"] = { + "/mismatch-gsc": {"sessions": 0}, + "/ga4-only": {"sessions": 25}, + "/ratio-high": {"sessions": 60}, + } + gsc["pages"] = list(gsc["pages"]) + [ + {"page": "https://ex.com/mismatch-gsc", "clicks": 20, "impressions": 100, "position": 5}, + {"page": "https://ex.com/ga4-only", "clicks": 0, "impressions": 0, "position": 10}, + {"page": "https://ex.com/ratio-high", "clicks": 10, "impressions": 50, "position": 6}, + ] + data["gsc_full"] = gsc + data["ga4_full"] = ga4 + return data + + +def _keyword_data() -> dict: + return { + "fetched_at": "2026-06-01", + "rows": [ + { + "keyword": "widgets", + "gsc_position": 8, + "gsc_clicks": 0, + "gsc_impressions": 500, + "gsc_url": "https://ex.com/", + "recommended_action": "Improve CTR", + "serp_features": ["ai_overview", "faq"], + "serp_estimated_competition": 45, + "is_branded": True, + "is_question": True, + "intent": "commercial", + "score": 10, + "traffic_potential": 200, + }, + { + "keyword": "repair service", + "gsc_position": 15, + "gsc_clicks": 0, + "gsc_impressions": 200, + "gsc_url": "https://ex.com/about", + "recommended_action": "Create content", + "serp_features": "local_pack", + "serp_estimated_competition": 70, + "is_branded": False, + "is_question": False, + "intent": "transactional", + }, + { + "keyword": "near page one", + "gsc_position": 11, + "gsc_clicks": 2, + "gsc_impressions": 100, + "gsc_url": "https://ex.com/near", + }, + { + "keyword": "bad position", + "gsc_position": "bad", + "gsc_clicks": 1, + "gsc_impressions": 10, + }, + ], + "cannibalisation": [ + { + "query": "widgets", + "pages": [ + {"url": "https://ex.com/a", "position": 5, "clicks": 3, "impressions": 50}, + {"url": "https://ex.com/b", "position": 8, "clicks": 1, "impressions": 20}, + "skip", + ], + }, + "skip", + ], + "query_page_misalignment": [{"keyword": "buy widgets", "url": "https://ex.com/wrong"}], + "semantic_keyword_clusters": [{"top_keyword": "widgets", "keywords": ["widgets"]}], + } + + +def _prior_keywords() -> dict: + return { + "rows": [ + {"keyword": "widgets", "gsc_position": 12, "gsc_impressions": 150}, + {"keyword": "repair service", "gsc_position": 8, "gsc_impressions": 100}, + {"keyword": "old top ten", "gsc_position": 9, "gsc_impressions": 80}, + {"keyword": "fell out", "gsc_position": 5, "gsc_impressions": 60}, + ], + } + + +def _gsc_links() -> dict: + return { + "top_linking_sites": [{"site": "partner.com", "link_count": 5}], + "top_linking_text": [{"anchor_text": "widgets", "link_count": 3}], + "sample_links": [ + { + "linking_site": "partner.com", + "source_page": "https://partner.com/page", + "target_page": "https://ex.com/", + "target_url_on_linking_page": "https://ex.com/", + "anchor_text": "Best widgets", + }, + { + "source_page": "https://www.other.org/ref", + "target_page": "https://ex.com/about", + "anchor_text": "About us", + }, + ], + "latest_links": [ + { + "linking_site": "fresh.com", + "target_page": "https://ex.com/new", + "anchor_text": "fresh link", + }, + ], + } + + +def _crawl_df() -> pd.DataFrame: + return pd.DataFrame([ + { + "url": "https://ex.com/", + "status": "200", + "title": "Home page with widgets", + "title_length": 25, + "meta_description": "desc", + "h1": "Home", + "word_count": 400, + "reading_level": 8, + "response_time_ms": 100, + "viewport_present": "true", + "viewport_content": "width=device-width", + "og_title": "OG Home", + "twitter_card": "summary", + "has_schema": "true", + "detected_language": "en", + "fetch_method": "static", + "content_excerpt": "Widgets are tools that means something useful.", + "html": "
  • item
    • ", + "page_analysis": json.dumps({ + "html_lang": "en", + "json_ld_types": ["Organization", "FAQPage"], + "hreflang_alternates": [ + {"hreflang": "en", "href": "https://ex.com/other"}, + {"hreflang": "fr", "href": "https://ex.com/fr"}, + ], + "browser": { + "console": [{"type": "error", "text": "Uncaught Error"}], + "page_errors": [{"message": "ReferenceError"}], + "failed_requests": [{"url": "https://ex.com/missing.js"}], + }, + }), + }, + { + "url": "https://ex.com/", + "status": "200", + "title": "Home rendered", + "h1": "Home R", + "word_count": 500, + "fetch_method": "rendered", + "page_analysis": "{}", + }, + { + "url": "https://ex.com/fr", + "status": "200", + "title": "French", + "detected_language": "fr", + "word_count": 300, + "page_analysis": json.dumps({ + "hreflang_alternates": [{"hreflang": "fr", "href": "https://ex.com/fr"}], + }), + }, + { + "url": "https://ex.com/other", + "status": "200", + "title": "Other EN", + "detected_language": "en", + "word_count": 250, + "page_analysis": json.dumps({ + "hreflang_alternates": [{"hreflang": "en", "href": "https://ex.com/other"}], + }), + }, + { + "url": "https://ex.com/short-crawl", + "status": "200", + "title": "Short", + "title_length": 5, + "word_count": 80, + "reading_level": "bad", + "response_time_ms": "bad", + "viewport_present": "true", + "viewport_content": "broken", + "og_title": "", + "twitter_card": "", + "has_schema": "true", + "page_analysis": json.dumps({"json_ld_types": []}), + }, + { + "url": "https://ex.com/slow-crawl", + "status": "200", + "title": "Slow page title here", + "title_length": 35, + "response_time_ms": 3000, + "word_count": 60, + "reading_level": 13, + "viewport_present": "false", + "page_analysis": json.dumps({"html_lang": ""}), + }, + { + "url": "https://ex.com/blog/how-to-fix-widgets", + "status": "200", + "title": "How to fix widgets step-by-step", + "h1": "How to fix widgets", + "word_count": 350, + "content_excerpt": "Posted by author on Monday. " + ("word " * 220), + "page_analysis": json.dumps({"json_ld_types": ["WebPage"]}), + }, + { + "url": "https://ex.com/guide/tutorial", + "status": "200", + "title": "Tutorial guide", + "word_count": 280, + "content_excerpt": "- step one\n- step two\nWidgets are devices.", + "html": "
    • one
    ", + "page_analysis": json.dumps({"json_ld_types": ["HowTo"]}), + }, + { + "url": "https://ex.com/redirect", + "status": "301", + "redirect_chain_length": 4, + "final_url": "https://ex.com/final", + "page_analysis": "{}", + }, + { + "url": "https://ex.com/href-dup", + "status": "200", + "page_analysis": json.dumps({ + "hreflang_alternates": [ + {"hreflang": "en", "href": "https://ex.com/href-dup"}, + {"hreflang": "en", "href": "https://ex.com/href-dup-2"}, + ], + }), + }, + { + "url": "https://ex.com/href-self", + "status": "200", + "page_analysis": json.dumps({ + "hreflang_alternates": [{"hreflang": "de", "href": "https://ex.com/de-only"}], + }), + }, + { + "url": "https://ex.com/console", + "status": "200", + "page_analysis": json.dumps({ + "console_errors": ["raw error", {"type": "warning", "message": "warn"}], + "js_errors": [{"level": "error", "text": "js fail"}], + }), + }, + { + "url": "https://ex.com/404", + "status": "404", + "word_count": 10, + "page_analysis": "{}", + }, + ]) + + +def _log_row() -> dict: + return { + "upload_id": 7, + "filename": "access.log", + "analysis": { + "top_paths": [ + {"path": "/orphan", "hits": 50}, + {"path": "/popular", "hits": 100}, + {"path": "/quiet", "hits": 5}, + ], + "googlebot_paths": [{"path": "/popular", "hits": 2}], + "paths_5xx": [], + "status_counts": {"500": 12, "503": 3}, + }, + } + + +def _compare_current() -> dict: + return { + "report_generated_at": "2026-06-07", + "google": _google_mismatch(), + "categories": [{"issues": [{"message": "New issue", "url": "https://ex.com/new"}]}], + "lighthouse_by_url": { + "https://ex.com/": {"performance": 70, "seo": 80}, + "https://ex.com/regressed": {"performance": 50, "seo": 60}, + }, + "links": [{"url": "https://ex.com/"}, {"url": "https://ex.com/new"}], + } + + +def _compare_baseline() -> dict: + return { + "report_generated_at": "2026-05-01", + "google": _prior_google(), + "categories": [{"issues": [{"message": "Old issue", "url": "https://ex.com/old"}]}], + "lighthouse_by_url": { + "https://ex.com/": {"performance": 80, "seo": 85}, + "https://ex.com/regressed": {"performance": 70, "seo": 75}, + }, + "links": [{"url": "https://ex.com/"}, {"url": "https://ex.com/removed"}], + } + + +def test_issue_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None: + empty_df = pd.DataFrame() + with patch.object(Ctx, "load_payload", return_value=None), patch.object(Ctx, "load_crawl_df", return_value=empty_df): + assert issue_mod.list_pages_title_too_short(conn, ctx, {})["total"] == 0 + assert issue_mod.list_pages_color_contrast_failures(conn, ctx, {})["error"] + assert issue_mod.list_orphan_hub_suggestions(conn, ctx, {})["error"] + assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["error"] + + with patch.object(Ctx, "load_payload", return_value={"content_urls": "bad"}): + assert issue_mod.list_pages_title_too_short(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value={"content_urls": {"title_short": "bad"}}): + out = issue_mod.list_pages_title_too_short(conn, ctx, {}) + assert out["total"] == 0 + + payload = _payload() + df = _crawl_df() + with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df): + assert issue_mod.list_pages_title_too_short(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_title_too_long(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_slow_response(conn, ctx, {"threshold_ms": "bad"})["total"] >= 1 + assert issue_mod.list_pages_missing_html_lang(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_invalid_viewport(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_high_reading_level(conn, ctx, {"min_reading_level": "bad"})["total"] >= 1 + assert issue_mod.list_pages_very_thin_content(conn, ctx, {"max_word_count": "bad"})["total"] >= 1 + assert issue_mod.list_hreflang_issue_pages(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_missing_og_tags(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_missing_twitter_cards(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_invalid_json_ld(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_mixed_language(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_orphan_hub_suggestions(conn, ctx, {})["total"] >= 0 + assert issue_mod.list_lighthouse_failure_lcp(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_lighthouse_failure_inp(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_lighthouse_failure_cls(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {"seo_threshold": "bad"})["total"] >= 1 + contrast = issue_mod.list_pages_color_contrast_failures(conn, ctx, {}) + assert contrast["total"] >= 0 + + empty_payload = {"hreflang_issue_urls": [], "issues": {"seo": "bad"}} + with patch.object(Ctx, "load_payload", return_value=empty_payload), patch.object(Ctx, "load_crawl_df", return_value=df): + assert issue_mod.list_hreflang_issue_pages(conn, ctx, {})["total"] >= 1 + + no_bucket = {"content_urls": {}, "social_coverage": {}} + with patch.object(Ctx, "load_payload", return_value=no_bucket), patch.object(Ctx, "load_crawl_df", return_value=df): + assert issue_mod.list_pages_title_too_short(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_missing_og_tags(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_missing_twitter_cards(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_very_thin_content(conn, ctx, {})["total"] >= 1 + + lh_payload = { + "lighthouse_failure_urls": {}, + "lighthouse_by_url": { + "https://ex.com/x": {"seo": 50, "audits": {"seo": {"score": 0.5, "title": "SEO"}}}, + "bad": "skip", + }, + } + with patch.object(Ctx, "load_payload", return_value=lh_payload): + assert issue_mod.list_lighthouse_failure_lcp(conn, ctx, {})["total"] == 0 + assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["total"] >= 1 + + no_lang = {"language_summary": {}} + with patch.object(Ctx, "load_payload", return_value=no_lang), patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert issue_mod.list_pages_mixed_language(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value={"orphan_urls": "bad", "graph_edges": []}), patch.object( + Ctx, "load_crawl_df", return_value=df, + ): + assert issue_mod.list_orphan_hub_suggestions(conn, ctx, {})["total"] == 0 + + +def test_google_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_google_full", return_value=None), patch.object(Ctx, "load_google", return_value=None): + assert google_mod.list_gsc_pages_by_impressions(conn, ctx, {})["missing"] is True + assert google_mod.get_gsc_query_trend(conn, ctx, {"query": "x"})["missing"] is True + assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com"})["missing"] is True + assert google_mod.get_ga4_path_trend(conn, ctx, {"path": "/"})["missing"] is True + + assert google_mod.get_gsc_query_trend(conn, ctx, {})["error"] == "query is required" + assert google_mod.get_gsc_page_trend(conn, ctx, {})["error"] == "url is required" + assert google_mod.get_ga4_path_trend(conn, ctx, {})["error"] == "path is required" + + google = _google_mismatch() + with patch.object(Ctx, "load_google_full", return_value=google), patch.object(Ctx, "load_google", return_value=google): + assert google_mod.list_gsc_pages_by_impressions(conn, ctx, {})["total"] >= 1 + assert google_mod.list_gsc_pages_by_clicks(conn, ctx, {})["total"] >= 1 + assert google_mod.list_gsc_queries_by_impressions(conn, ctx, {})["total"] >= 1 + assert google_mod.list_gsc_queries_by_clicks(conn, ctx, {})["total"] >= 1 + assert google_mod.list_gsc_ctr_underperformers(conn, ctx, {})["total"] >= 0 + assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] >= 1 + assert google_mod.list_ga4_pages_by_bounce_rate(conn, ctx, {})["total"] >= 1 + assert google_mod.list_ga4_pages_by_engagement_rate(conn, ctx, {})["total"] >= 1 + trend = google_mod.get_gsc_query_trend(conn, ctx, {"query": "widgets"}) + assert trend.get("daily") or trend.get("snapshot") + assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com/"})["daily"] + assert google_mod.get_ga4_path_trend(conn, ctx, {"url": "https://ex.com/alt"})["path"] == "/alt" + assert google_mod.list_gsc_ga4_mismatch_pages(conn, ctx, {})["total"] >= 1 + assert google_mod.list_gsc_pages_by_position_band(conn, ctx, {"min_position": "bad"})["total"] >= 1 + assert google_mod.get_gsc_site_benchmarks(conn, ctx, {})["page_count"] >= 1 + + top_key = {"gsc_full": {"top_pages": [{"page": "https://ex.com/top", "impressions": 99}]}} + with patch.object(Ctx, "load_google_full", return_value=top_key): + assert google_mod.list_gsc_pages_by_impressions(conn, ctx, {})["total"] == 1 + + no_daily = {"gsc_full": {"queries": [{"query": "only", "clicks": 1}]}} + with patch.object(Ctx, "load_google_full", return_value=no_daily): + snap = google_mod.get_gsc_query_trend(conn, ctx, {"query": "only"}) + assert snap.get("snapshot") + + with patch.object(Ctx, "load_keywords", return_value=_keyword_data()): + assert google_mod.list_gsc_branded_queries(conn, ctx, {})["total"] >= 1 + assert google_mod.list_gsc_non_branded_queries(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_keywords", return_value=None): + assert google_mod.list_gsc_branded_queries(conn, ctx, {})["missing"] is True + + current, prior = _google_mismatch(), _prior_google() + with patch.object(google_mod, "_load_google_pair", return_value=(current, prior)): + assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["total"] >= 0 + assert google_mod.list_gsc_decaying_queries(conn, ctx, {})["total"] >= 0 + assert google_mod.list_gsc_new_queries(conn, ctx, {})["total"] >= 1 + assert google_mod.compare_gsc_periods(conn, ctx, {})["gsc"]["clicks"]["delta"] != 0 + + with patch.object(google_mod, "_load_google_pair", return_value=(current, None)): + assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["missing"] is True + assert google_mod.compare_gsc_periods(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_google_pair", return_value=(current, None)), patch( + "website_profiling.integrations.google.store.read_prior_google_snapshot", + return_value=prior, + ): + cur, pr = google_mod._load_google_pair(ctx, conn) + assert pr is not None + + with patch.object(Ctx, "load_google_pair", return_value=(current, None)), patch( + "website_profiling.integrations.google.store.read_prior_google_snapshot", + side_effect=RuntimeError("fail"), + ): + conn.execute.return_value.fetchall.return_value = [{"data": prior}, {"data": prior}] + cur2, pr2 = google_mod._load_google_pair(ctx, conn) + assert pr2 is not None + + with patch.object(Ctx, "load_google_pair", return_value=(current, None)), patch( + "website_profiling.integrations.google.store.read_prior_google_snapshot", + side_effect=RuntimeError("fail"), + ), patch.object(conn, "execute", side_effect=RuntimeError("db fail")): + cur3, pr3 = google_mod._load_google_pair(ctx, conn) + assert pr3 is None + + no_prop = Ctx(property_id=None, report_id=1) + with patch.object(Ctx, "load_google_pair", return_value=(current, prior)): + cur4, pr4 = google_mod._load_google_pair(no_prop, conn) + assert pr4 is prior + + +def test_keyword_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None: + no_prop = Ctx(property_id=None, report_id=1) + assert kw_mod.list_keyword_rank_improvements(conn, no_prop, {})["missing"] is True + assert kw_mod.list_keywords_by_recommended_action(conn, ctx, {})["error"] + assert kw_mod.list_keywords_by_serp_feature(conn, ctx, {})["error"] + assert kw_mod.get_keyword_opportunity_score(conn, ctx, {})["error"] == "keyword is required" + assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {})["error"] == "keyword is required" + assert kw_mod.list_keywords_near_page_one(conn, ctx, {"min_position": "bad"})["error"] + + kw = _keyword_data() + prior = _prior_keywords() + payload = _payload() + with patch.object(Ctx, "load_keywords", return_value=kw), patch.object(Ctx, "load_payload", return_value=payload), patch( + "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property", + return_value=[kw, prior], + ): + assert kw_mod.list_keyword_rank_improvements(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keyword_rank_declines(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keywords_new_to_top_10(conn, ctx, {})["total"] >= 0 + assert kw_mod.list_keywords_fell_out_of_top_10(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_cannibalisation_queries(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_cannibalisation_urls(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_misaligned_queries(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keywords_by_recommended_action(conn, ctx, {"recommended_action": "Improve"})["total"] >= 1 + assert kw_mod.list_keywords_by_serp_feature(conn, ctx, {"serp_feature": "local"})["total"] >= 1 + assert kw_mod.list_semantic_cluster_queries(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_semantic_cluster_pages(conn, ctx, {})["total"] >= 1 + assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "widgets"})["opportunity_score"] > 0 + assert kw_mod.list_keywords_near_page_one(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keywords_high_impression_zero_click(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keywords_by_competition_band(conn, ctx, {})["total"] >= 1 + assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {"keyword": "widgets"})["keyword"] == "widgets" + assert kw_mod.list_keywords_with_ai_overview(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keywords_local_pack(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keywords_question_intent(conn, ctx, {})["total"] >= 1 + assert kw_mod.list_keywords_commercial_intent(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_keywords", return_value=None): + assert kw_mod.list_cannibalisation_queries(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_keywords", return_value={"rows": []}), patch( + "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property", + return_value=[{"rows": []}], + ): + assert kw_mod.list_keyword_rank_improvements(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_keywords", return_value=kw), patch( + "website_profiling.integrations.google.keyword_store.read_keyword_snapshots_for_property", + return_value=[prior], + ): + assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "missing"})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value={}), patch.object(Ctx, "load_keywords", return_value={"semantic_keyword_clusters": []}): + assert kw_mod.list_semantic_cluster_queries(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "x", "serp_estimated_competition": "bad"}]}): + assert kw_mod.list_keywords_by_competition_band(conn, ctx, {})["total"] == 0 + + +def test_backlink_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None: + no_prop = Ctx(property_id=None, report_id=1) + assert bl_mod.list_referring_domains(conn, no_prop, {})["error"] + assert bl_mod.list_backlinks_to_url(conn, ctx, {})["error"] == "url is required" + assert bl_mod.list_backlinks_from_domain(conn, ctx, {})["error"] == "domain is required" + + links = _gsc_links() + with patch.object(Ctx, "load_gsc_links", return_value=links): + assert bl_mod.list_referring_domains(conn, ctx, {})["total"] >= 1 + assert bl_mod.list_backlinks_by_anchor_text(conn, ctx, {"anchor_text": "widget"})["total"] >= 1 + assert bl_mod.list_backlinks_to_url(conn, ctx, {"url": "https://ex.com/"})["total"] >= 1 + assert bl_mod.list_backlinks_from_domain(conn, ctx, {"domain": "partner.com"})["total"] >= 1 + assert bl_mod.get_anchor_text_distribution(conn, ctx, {})["source"] == "top_linking_text" + + derived = { + "sample_links": [ + {"source_page": "https://www.example.com/x", "anchor_text": ""}, + {"source_page": "bad://", "anchor_text": "x"}, + ], + } + with patch.object(Ctx, "load_gsc_links", return_value=derived): + assert bl_mod.list_referring_domains(conn, ctx, {})["total"] >= 1 + assert bl_mod.get_anchor_text_distribution(conn, ctx, {})["source"] == "sample_links" + + with patch.object(Ctx, "load_gsc_links", return_value=None): + assert bl_mod.list_referring_domains(conn, ctx, {})["missing"] is True + + +def test_content_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None: + payload = _payload() + df = _crawl_df() + with patch.object(Ctx, "load_payload", return_value=None): + assert content_mod.get_text_content_analysis(conn, ctx, {})["missing"] is True + assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "x"})["error"] + + assert content_mod.list_pages_containing_keyword(conn, ctx, {})["error"] == "keyword is required" + + with patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df): + assert content_mod.get_text_content_analysis(conn, ctx, {})["missing"] is False + assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "widgets"})["total"] >= 1 + assert content_mod.list_pages_by_word_count_band(conn, ctx, {"min_word_count": "bad"})["band"]["min_word_count"] == 0 + assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["total"] >= 1 + assert content_mod.list_spell_check_issues(conn, ctx, {})["total"] >= 1 + assert content_mod.list_html_validation_issues(conn, ctx, {})["total"] >= 1 + assert content_mod.list_amp_validation_issues(conn, ctx, {})["total"] >= 1 + assert content_mod.list_pagination_issues(conn, ctx, {})["total"] >= 1 + assert content_mod.list_schema_errors_by_type(conn, ctx, {"schema_type": "faq"})["total"] >= 1 + assert content_mod.list_pages_missing_article_schema(conn, ctx, {})["total"] >= 1 + + fallback_payload = {"content_analytics": {"keyword_index": [{"word": "repair", "top_pages": []}]}} + with patch.object(Ctx, "load_payload", return_value=fallback_payload): + assert content_mod.get_text_content_analysis(conn, ctx, {})["note"] + + cat_payload = { + "optional_audit_urls": {}, + "categories": [{"issues": [{"message": "custom audit needle found on page"}]}], + } + with patch.object(Ctx, "load_payload", return_value=cat_payload): + assert content_mod.list_spell_check_issues(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value={"categories": []}), patch.object(Ctx, "load_crawl_df", return_value=df): + assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "widgets"})["total"] >= 1 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert content_mod.list_pages_by_word_count_band(conn, ctx, {})["missing"] is True + assert content_mod.list_pages_missing_article_schema(conn, ctx, {})["missing"] is True + + +def test_link_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None: + with patch.object(Ctx, "load_payload", return_value=None): + assert link_mod.list_outbound_links(conn, ctx, {})["error"] + assert link_mod.list_internal_links_from_url(conn, ctx, {"url": "https://ex.com/"})["error"] + assert link_mod.list_internal_links_to_url(conn, ctx, {"url": "https://ex.com/"})["error"] + assert link_mod.list_links_by_rel_nofollow(conn, ctx, {})["error"] + assert link_mod.list_pagerank_low_pages(conn, ctx, {})["error"] + + assert link_mod.list_internal_links_from_url(conn, ctx, {})["error"] == "url is required" + assert link_mod.list_internal_links_to_url(conn, ctx, {})["error"] == "url is required" + + payload = _payload() + with patch.object(Ctx, "load_payload", return_value=payload): + assert link_mod.list_outbound_links(conn, ctx, {})["total"] >= 1 + assert link_mod.list_internal_links_from_url(conn, ctx, {"url": "https://ex.com/"})["total"] >= 1 + assert link_mod.list_internal_links_to_url(conn, ctx, {"url": "https://ex.com/"})["total"] >= 1 + assert link_mod.list_links_by_rel_nofollow(conn, ctx, {})["total"] >= 1 + assert link_mod.list_links_by_rel_nofollow(conn, ctx, {"rel": "ugc"})["total"] >= 1 + assert link_mod.list_pagerank_low_pages(conn, ctx, {"max_pagerank": "bad"})["total"] >= 1 + + graph_payload = { + "start_url": "https://ex.com/", + "graph_edges": [[0, 1], {"from": "https://ex.com/a", "to": "https://other.com/x"}], + "top_pages": [{"url": "https://ex.com/x", "pagerank": "bad"}, {"url": "https://ex.com/y", "pagerank": 0.001}], + } + with patch.object(Ctx, "load_payload", return_value=graph_payload): + assert link_mod.list_outbound_links(conn, ctx, {})["total"] >= 1 + assert link_mod.list_pagerank_low_pages(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_payload", return_value={"top_pages": []}): + assert link_mod.list_pagerank_low_pages(conn, ctx, {})["missing"] is True + + +def test_indexation_lists_all_paths(conn: MagicMock, ctx: Ctx) -> None: + payload = _payload() + df = _crawl_df() + with patch.object(Ctx, "load_payload", return_value=None): + assert idx_mod.list_indexation_submitted_not_indexed(conn, ctx, {})["error"] + assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["error"] + + with patch.object(Ctx, "load_payload", return_value={"indexation_coverage": "bad"}): + assert idx_mod.list_sitemap_urls_not_in_crawl(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value=payload): + assert idx_mod.list_indexation_submitted_not_indexed(conn, ctx, {})["total"] >= 1 + assert idx_mod.list_indexation_indexed_not_submitted(conn, ctx, {})["total"] >= 1 + assert idx_mod.list_sitemap_urls_not_in_crawl(conn, ctx, {})["source"] + assert idx_mod.list_crawl_urls_not_in_sitemap(conn, ctx, {})["source"] + + no_prop = Ctx(property_id=None, report_id=1) + assert idx_mod.list_log_paths_by_hits(conn, no_prop, {})["error"] + + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=None): + assert idx_mod.list_log_paths_by_hits(conn, ctx, {})["missing"] is True + + log = _log_row() + orphan_payload = {**payload, "orphan_urls": ["https://ex.com/orphan"]} + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log), patch.object( + Ctx, "load_payload", return_value=orphan_payload, + ): + assert idx_mod.list_log_paths_by_hits(conn, ctx, {})["total"] >= 1 + assert idx_mod.list_log_5xx_paths(conn, ctx, {})["total"] >= 1 + assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": "bad"})["total"] >= 0 + assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {"min_hits": "bad"})["total"] >= 0 + + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log), patch.object( + Ctx, "load_payload", return_value={"orphan_urls": []}, + ): + assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["note"] + + with patch.object(Ctx, "load_crawl_df", return_value=df): + assert idx_mod.list_redirect_chains_by_length(conn, ctx, {"min_length": "bad"})["total"] >= 1 + assert idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})["missing"] is True + + +def test_compare_list_tools_all_paths(conn: MagicMock, ctx: Ctx) -> None: + err = {"error": "baseline required"} + with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(None, None, None, None, err)): + assert cmp_mod.list_compare_new_issues(conn, ctx, {})["error"] + assert cmp_mod.list_compare_resolved_issues(conn, ctx, {})["error"] + assert cmp_mod.list_compare_new_urls(conn, ctx, {})["error"] + assert cmp_mod.list_compare_removed_urls(conn, ctx, {})["error"] + assert cmp_mod.list_compare_lighthouse_regressions(conn, ctx, {})["error"] + assert cmp_mod.list_compare_traffic_losers(conn, ctx, {})["error"] + + current, baseline = _compare_current(), _compare_baseline() + with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(current, baseline, 2, 1, None)): + assert cmp_mod.list_compare_new_issues(conn, ctx, {})["total"] >= 0 + assert cmp_mod.list_compare_resolved_issues(conn, ctx, {})["total"] >= 0 + assert cmp_mod.list_compare_new_urls(conn, ctx, {})["total"] >= 1 + assert cmp_mod.list_compare_removed_urls(conn, ctx, {})["total"] >= 1 + assert cmp_mod.list_compare_lighthouse_regressions(conn, ctx, {"min_regression": "bad"})["total"] >= 1 + assert cmp_mod.list_compare_traffic_losers(conn, ctx, {})["total"] >= 1 + + no_google = dict(current) + no_google.pop("google") + base_no_google = dict(baseline) + base_no_google.pop("google") + with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(no_google, base_no_google, 2, 1, None)), patch.object( + Ctx, "load_google_full", return_value=None, + ), patch.object(Ctx, "load_google", return_value=None): + assert cmp_mod.list_compare_traffic_losers(conn, ctx, {})["missing"] is True + + +def test_geo_list_tools_all_paths(conn: MagicMock, ctx: Ctx) -> None: + df = _crawl_df() + payload = _payload() + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert geo_list_mod.list_pages_missing_howto_schema(conn, ctx, {})["missing"] is True + assert geo_list_mod.list_pages_ai_citation_signals(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_crawl_df", return_value=df): + assert geo_list_mod.list_pages_missing_howto_schema(conn, ctx, {})["total"] >= 1 + assert geo_list_mod.list_pages_ai_citation_signals(conn, ctx, {"min_score": "bad"})["total"] >= 1 + + with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch( + "website_profiling.tools.audit_tools.geo_list_tools._fetch_llms_txt", + return_value={"found": False}, + ): + assert geo_list_mod.list_pages_missing_llms_txt_reference(conn, ctx, {})["missing"] is True + + llms = {"found": True, "url": "https://ex.com/llms.txt", "preview": "https://ex.com/\nMore docs"} + with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch( + "website_profiling.tools.audit_tools.geo_list_tools._fetch_llms_txt", return_value=llms, + ), patch.object(Ctx, "load_payload", return_value=payload), patch.object(Ctx, "load_crawl_df", return_value=df): + missing = geo_list_mod.list_pages_missing_llms_txt_reference(conn, ctx, {}) + assert missing["total"] >= 1 + + with patch.object(Ctx, "resolve_property_domain", return_value=""): + assert geo_list_mod.list_robots_blocked_ai_crawlers(conn, ctx, {})["error"] + + robots = "User-agent: GPTBot\nDisallow: /\nUser-agent: *\nDisallow: /private" + with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch.object( + geo_list_mod, "_parse_robots_txt", return_value=robots, + ): + blocked = geo_list_mod.list_robots_blocked_ai_crawlers(conn, ctx, {}) + assert blocked["total"] >= 1 + + with patch.object(Ctx, "resolve_property_domain", return_value="ex.com"), patch.object( + geo_list_mod, "_parse_robots_txt", return_value="", + ): + assert geo_list_mod.list_robots_blocked_ai_crawlers(conn, ctx, {})["missing"] is True + + with patch("website_profiling.tools.audit_tools.geo_list_tools.requests.get", side_effect=requests.RequestException("fail")): + assert geo_list_mod._parse_robots_txt("ex.com") == "" + + mock_resp = MagicMock(status_code=404, text="") + with patch("website_profiling.tools.audit_tools.geo_list_tools.requests.get", return_value=mock_resp): + assert geo_list_mod._parse_robots_txt("ex.com") == "" + + +def test_crawl_console_and_js_handlers(conn: MagicMock, ctx: Ctx) -> None: + assert crawl_mod.list_pages_console_errors_by_type(conn, ctx, {})["error"] == "error_type is required" + + df = _crawl_df() + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert crawl_mod.list_pages_console_errors_by_type(conn, ctx, {"error_type": "error"})["total"] == 0 + assert crawl_mod.list_pages_js_rendering_delta(conn, ctx, {})["note"] + + with patch.object(Ctx, "load_crawl_df", return_value=df): + console = crawl_mod.list_pages_console_errors_by_type(conn, ctx, {"error_type": "error"}) + assert console["total"] >= 1 + page_err = crawl_mod.list_pages_console_errors_by_type(conn, ctx, {"error_type": "page_error"}) + assert page_err["total"] >= 1 + js = crawl_mod.list_pages_js_rendering_delta(conn, ctx, {}) + assert js["total"] >= 1 + assert js["provenance"] == "Crawl" + + js_only = pd.DataFrame([ + {"url": "https://ex.com/js", "status": "200", "fetch_method": "static", "title": "A", "word_count": 10, "h1": "A"}, + {"url": "https://ex.com/js", "status": "200", "fetch_method": "javascript", "title": "B", "word_count": 100, "h1": "B"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=js_only): + assert crawl_mod.list_pages_js_rendering_delta(conn, ctx, {})["total"] == 1 + + +def test_batch100_coverage_gaps(conn: MagicMock, ctx: Ctx) -> None: + """Hit remaining branches for 100% line coverage on batch-100 modules.""" + empty = pd.DataFrame() + df = _crawl_df() + + # issue_lists helpers and dead paths + with patch.object(Ctx, "load_payload", return_value=None): + assert issue_mod._payload_list_key(conn, ctx, {}, "missing_key")["error"] + with patch.object(Ctx, "load_payload", return_value={"flat_list": [{"url": "x"}]}): + assert issue_mod._payload_list_key(conn, ctx, {}, "flat_list")["total"] == 1 + with patch.object(Ctx, "load_payload", return_value={"scalar_key": 123}): + assert issue_mod._payload_list_key(conn, ctx, {}, "scalar_key")["missing"] is True + with patch.object(Ctx, "load_payload", return_value={"issues": {"seo": [{"type": "missing_title", "url": "u"}]}}): + assert issue_mod._issues_by_type(conn, ctx, {}, "missing_title")["total"] == 1 + with patch.object(Ctx, "load_payload", return_value={"issues": "bad"}): + assert issue_mod._issues_by_type(conn, ctx, {}, "x")["total"] == 0 + + crawl_only = pd.DataFrame([ + {"url": "https://ex.com/long", "status": "200", "title_length": 65, "title": "T" * 65}, + {"url": "https://ex.com/slow2", "status": "200", "response_time_ms": 5000, "title": "Slow"}, + {"url": "https://ex.com/nolang2", "status": "200", "title": "X", "page_analysis": "{}"}, + {"url": "https://ex.com/vp", "status": "200", "viewport_present": "true", "viewport_content": "initial-scale=1", "page_analysis": "{}"}, + {"url": "https://ex.com/read", "status": "200", "reading_level": 14, "title": "Hard"}, + ]) + with patch.object(Ctx, "load_payload", return_value={"content_urls": {}}), patch.object(Ctx, "load_crawl_df", return_value=crawl_only): + assert issue_mod.list_pages_title_too_long(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_slow_response(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_missing_html_lang(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_invalid_viewport(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_pages_high_reading_level(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_payload", return_value={"hreflang_issue_urls": []}), patch.object(Ctx, "load_crawl_df", return_value=empty): + assert issue_mod.list_hreflang_issue_pages(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value=None), patch.object(Ctx, "load_crawl_df", return_value=empty): + assert issue_mod.list_pages_missing_og_tags(conn, ctx, {})["error"] + assert issue_mod.list_pages_missing_twitter_cards(conn, ctx, {})["error"] + + no_dom = {"language_summary": {"counts": {}}} + with patch.object(Ctx, "load_payload", return_value=no_dom), patch.object(Ctx, "load_crawl_df", return_value=df): + mixed = issue_mod.list_pages_mixed_language(conn, ctx, {}) + assert mixed["total"] == 0 + + lh_edge = { + "lighthouse_failure_urls": {}, + "lighthouse_by_url": { + "https://ex.com/lcp2": {"lcp": 4.0, "cwv_failures": "lcp"}, + "https://ex.com/inp2": {"inp": 500, "top_failures": [{"id": "inp-slow"}]}, + "https://ex.com/cls2": {"cls": 0.25, "cwv_failures": "cls"}, + "https://ex.com/audit": {"lcp": 1.0, "audits": {"lcp": {"score": 0.5, "title": "LCP audit"}}}, + "bad": "skip", + }, + } + with patch.object(Ctx, "load_payload", return_value=lh_edge): + assert issue_mod.list_lighthouse_failure_lcp(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_lighthouse_failure_inp(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_lighthouse_failure_cls(conn, ctx, {})["total"] >= 1 + assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["total"] == 0 + with patch.object(Ctx, "load_payload", return_value={"lighthouse_by_url": {"https://ex.com/badseo": {"seo": "bad"}}}): + assert issue_mod.list_lighthouse_failure_seo(conn, ctx, {})["total"] == 0 + + # google_lists gaps + no_google = None + google_handlers = [ + google_mod.list_gsc_pages_by_impressions, + google_mod.list_gsc_pages_by_clicks, + google_mod.list_gsc_queries_by_impressions, + google_mod.list_gsc_queries_by_clicks, + google_mod.list_gsc_ctr_underperformers, + google_mod.list_ga4_landing_pages, + google_mod.list_ga4_pages_by_bounce_rate, + google_mod.list_ga4_pages_by_engagement_rate, + google_mod.list_gsc_ga4_mismatch_pages, + google_mod.list_gsc_pages_by_position_band, + google_mod.get_gsc_site_benchmarks, + ] + with patch.object(Ctx, "load_google_full", return_value=no_google), patch.object(Ctx, "load_google", return_value=no_google): + for handler in google_handlers: + assert handler(conn, ctx, {}).get("missing") is True + + assert google_mod._gsc_rows(None, "pages") == [] + assert google_mod._gsc_rows({"gsc_full": {"pages": "bad"}}, "pages") == [] + + decay_curr = {"gsc_full": {"pages": [{"page": "https://ex.com/p", "clicks": 5, "impressions": 100, "position": 8}]}} + decay_prior = {"gsc_full": {"pages": [{"page": "https://ex.com/p", "clicks": 20, "impressions": 200, "position": 5}]}} + assert google_mod._gsc_deltas( + google_mod._gsc_rows(decay_curr, "pages"), + google_mod._gsc_rows(decay_prior, "pages"), + ("page",), + decay=True, + ) + assert google_mod._gsc_deltas( + google_mod._gsc_rows(decay_curr, "pages"), + google_mod._gsc_rows(decay_prior, "pages"), + ("page",), + decay=False, + ) == [] + + with patch.object(google_mod, "_load_google_pair", return_value=(decay_curr, decay_prior)): + assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["total"] >= 1 + assert google_mod.list_gsc_decaying_queries(conn, ctx, {"limit": 5})["total"] == 0 + + with patch.object(google_mod, "_load_google_pair", return_value=(None, decay_prior)): + assert google_mod.list_gsc_decaying_queries(conn, ctx, {})["missing"] is True + assert google_mod.list_gsc_new_queries(conn, ctx, {})["missing"] is True + + ga4_by_path = {"ga4_full": {"by_path": {"/only": {"sessions": 5, "bounceRate": 0.5, "engagementRate": 0.4}}}} + with patch.object(Ctx, "load_google_full", return_value=ga4_by_path): + assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] == 1 + assert google_mod.list_ga4_pages_by_bounce_rate(conn, ctx, {})["total"] == 1 + assert google_mod.list_ga4_pages_by_engagement_rate(conn, ctx, {})["total"] == 1 + + daily_dims = { + "gsc": {"daily": [{"dimensions": {"page": "https://ex.com/dim"}, "clicks": 1}, "skip"]}, + "ga4": {"daily": "bad"}, + } + with patch.object(Ctx, "load_google_full", return_value=daily_dims): + assert google_mod._daily_series(daily_dims, "ga4", "path", "/") == [] + assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com/dim"})["daily"] + + high_ctr_only = {"gsc_full": {"pages": [{"page": "https://ex.com/g", "clicks": 50, "impressions": 100, "ctr": "50%", "position": 3}]}} + with patch.object(Ctx, "load_google_full", return_value=high_ctr_only): + assert google_mod.list_gsc_ctr_underperformers(conn, ctx, {})["total"] == 0 + + # keyword_lists gaps + kw = _keyword_data() + with patch.object(Ctx, "load_keywords", return_value=None), patch( + "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property", + return_value=[kw], + ): + cur, prior = kw_mod._load_keyword_pair(ctx, conn) + assert cur is kw and prior is None + + entered_prior = { + "rows": [{"keyword": "entered", "gsc_position": 15, "gsc_impressions": 10}], + } + entered_curr = { + "rows": [{"keyword": "entered", "gsc_position": 8, "gsc_impressions": 20}], + } + assert kw_mod._top_ten_transitions(entered_curr, entered_prior, entered=True) + fell_curr = {"rows": [{"keyword": "fell", "gsc_position": 15, "gsc_impressions": 5}]} + fell_prior = {"rows": [{"keyword": "fell", "gsc_position": 5, "gsc_impressions": 50}]} + assert kw_mod._top_ten_transitions(fell_curr, fell_prior, entered=False) + + no_prop = Ctx(property_id=None, report_id=1) + assert kw_mod.list_cannibalisation_urls(conn, no_prop, {})["missing"] is True + assert kw_mod.list_misaligned_queries(conn, no_prop, {})["missing"] is True + assert kw_mod.list_semantic_cluster_pages(conn, no_prop, {})["missing"] is True + assert kw_mod.get_keyword_opportunity_score(conn, no_prop, {"keyword": "x"})["missing"] is True + assert kw_mod.get_keyword_serp_snapshot(conn, no_prop, {"keyword": "x"})["missing"] is True + + with patch.object(Ctx, "load_keywords", return_value=kw), patch.object(Ctx, "load_payload", return_value={}): + assert kw_mod._keyword_bucket(conn, ctx, {}, key="semantic_keyword_clusters", item_key="clusters")["total"] >= 1 + + with patch.object(Ctx, "load_keywords", return_value=None): + assert kw_mod._filter_keywords(conn, ctx, {}, lambda r: True)["missing"] is True + assert kw_mod._pair_delta_tool(conn, ctx, {}, builder=lambda a, b: [], item_key="keywords")["missing"] is True + + with patch.object(Ctx, "load_keywords", return_value={"rows": []}), patch( + "website_profiling.tools.audit_tools.keyword_lists.read_keyword_snapshots_for_property", + return_value=[], + ): + assert kw_mod._load_keyword_pair(ctx, conn) == ({"rows": []}, None) + + with patch.object(Ctx, "load_keywords", return_value=kw), patch.object(Ctx, "load_payload", return_value={}): + assert kw_mod._semantic_clusters(ctx, conn) + + assert kw_mod._keyword_rows(None) == [] + assert kw_mod._position({"gsc_position": 0}) is None + assert kw_mod._position({"gsc_position": "bad"}) is None + + with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "x"}]}): + assert kw_mod.list_keywords_by_competition_band(conn, ctx, {"min_competition": "bad"})["error"] + + # backlink_lists error branches + no_prop = Ctx(property_id=None, report_id=1) + for handler, args in [ + (bl_mod.list_backlinks_by_anchor_text, {}), + (bl_mod.list_backlinks_to_url, {"url": "https://ex.com"}), + (bl_mod.list_backlinks_from_domain, {"domain": "x.com"}), + ]: + assert handler(conn, no_prop, args)["error"] + with patch.object(Ctx, "load_gsc_links", return_value=None): + assert bl_mod.list_backlinks_by_anchor_text(conn, ctx, {})["missing"] is True + assert bl_mod.list_backlinks_to_url(conn, ctx, {"url": "https://ex.com"})["missing"] is True + assert bl_mod.list_backlinks_from_domain(conn, ctx, {"domain": "x.com"})["missing"] is True + assert bl_mod.get_anchor_text_distribution(conn, ctx, {})["missing"] is True + with patch("website_profiling.tools.audit_tools.backlink_lists.urlparse", side_effect=ValueError("bad")): + assert bl_mod._norm_domain("bad://") == "bad://" + + # content_lists gaps + with patch.object(Ctx, "load_payload", return_value=None): + assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["error"] + assert content_mod.list_schema_errors_by_type(conn, ctx, {})["error"] + + cat_issues = { + "optional_audit_urls": {}, + "categories": [ + { + "issues": [ + {"message": "spell issue on page"}, + {"message": "html markup broken"}, + {"message": "amp validation failed"}, + {"message": "pagination rel=next broken"}, + {"message": "needle custom audit"}, + ], + }, + ], + } + with patch.object(Ctx, "load_payload", return_value=cat_issues): + assert content_mod.list_spell_check_issues(conn, ctx, {})["total"] >= 1 + assert content_mod.list_html_validation_issues(conn, ctx, {})["total"] >= 1 + assert content_mod.list_amp_validation_issues(conn, ctx, {})["total"] >= 1 + assert content_mod.list_pagination_issues(conn, ctx, {})["total"] >= 1 + + dup_bad = {"content_duplicates": ["skip", {"member_urls": "bad", "representative_url": "https://ex.com/a"}]} + with patch.object(Ctx, "load_payload", return_value=dup_bad): + assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["total"] == 0 + + schema_payload = {"rich_results_validation": ["skip", {"status": "fail", "type": "Product"}]} + with patch.object(Ctx, "load_payload", return_value=schema_payload): + assert content_mod.list_schema_errors_by_type(conn, ctx, {})["total"] == 1 + assert content_mod.list_schema_errors_by_type(conn, ctx, {"schema_type": "product"})["total"] == 1 + + article_df = pd.DataFrame([ + { + "url": "https://ex.com/blog/my-post", + "status": "200", + "title": "Post", + "content_excerpt": "Posted by author on Monday. " + ("word " * 210), + "page_analysis": json.dumps({"json_ld_types": ["WebPage"]}), + }, + { + "url": "https://ex.com/with-schema", + "status": "200", + "title": "Article", + "content_excerpt": "short", + "page_analysis": json.dumps({"json_ld_types": ["NewsArticle"]}), + }, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=article_df): + assert content_mod.list_pages_missing_article_schema(conn, ctx, {})["total"] >= 1 + + kw_payload = { + "text_content_analysis": { + "keyword_index": [{"word": "term", "top_pages": ["skip", ("https://ex.com/t", 1)]}], + }, + } + with patch.object(Ctx, "load_payload", return_value=kw_payload), patch.object(Ctx, "load_crawl_df", return_value=empty): + assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "term"})["total"] == 1 + + # link_lists gaps + graph_only = {"graph_edges": ["skip", [1, 2]], "top_pages": ["skip", {"url": "https://ex.com/z", "pagerank": None}]} + with patch.object(Ctx, "load_payload", return_value=graph_only): + assert link_mod._load_link_edges(graph_only) + assert link_mod._pagerank_rows(graph_only) == [] + + # indexation_lists gaps + assert idx_mod._cap_indexation_urls("bad", {})["total"] == 0 + with patch.object(Ctx, "load_payload", return_value=_payload()): + cov_err = idx_mod.list_indexation_indexed_not_submitted(conn, ctx, {}) + assert cov_err["total"] >= 1 + + log_non_list = { + "analysis": { + "top_paths": "bad", + "paths_5xx": "bad", + "googlebot_paths": ["skip"], + "status_counts": {"500": 12, "503": 3}, + }, + } + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log_non_list), patch.object( + Ctx, "load_payload", return_value=_payload(), + ): + assert idx_mod.list_log_paths_by_hits(conn, ctx, {})["total"] == 0 + assert idx_mod.list_log_5xx_paths(conn, ctx, {})["total"] >= 1 + assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {})["total"] >= 0 + + orphan_log = { + **_payload(), + "orphan_urls": ["https://ex.com/orphan"], + "links": [{"url": "https://ex.com/orphan"}], + } + log_row = { + "analysis": { + "top_paths": [{"path": "/orphan", "hits": 50}, {"path": "/other", "hits": 3}], + "googlebot_paths": [{"path": "/orphan", "hits": 0}], + }, + } + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=log_row), patch.object( + Ctx, "load_payload", return_value=orphan_log, + ): + assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["total"] >= 1 + assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": 10, "max_googlebot_hits": 0})["total"] >= 1 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame([{"url": "https://ex.com/r", "status": "301", "redirect_chain_length": "bad"}])): + assert idx_mod.list_redirect_chains_by_length(conn, ctx, {})["total"] == 0 + + href_df = pd.DataFrame([ + {"url": "https://ex.com/a", "status": "200", "page_analysis": json.dumps({ + "hreflang_alternates": [{"href": "https://ex.com/b", "hreflang": "en"}], + })}, + {"url": "https://ex.com/b", "status": "200", "page_analysis": json.dumps({ + "hreflang_alternates": [{"href": "https://ex.com/c", "hreflang": "de"}], + })}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=href_df): + gaps = idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {}) + assert gaps["total"] >= 1 + + with patch("website_profiling.tools.audit_tools.indexation_lists.url_to_path", side_effect=RuntimeError("bad")): + assert idx_mod._norm_path("https://ex.com/x") == "https://ex.com/x" + + # compare_list_tools line 156 (skip non-losers) + winner_current = { + "report_generated_at": "2026-06-07", + "google": {"gsc_full": {"pages": [{"page": "https://ex.com/win", "clicks": 100, "impressions": 200}]}}, + } + winner_baseline = { + "report_generated_at": "2026-05-01", + "google": {"gsc_full": {"pages": [{"page": "https://ex.com/win", "clicks": 10, "impressions": 50}]}}, + } + with patch("website_profiling.tools.audit_tools.compare_list_tools.load_compare_pair", return_value=(winner_current, winner_baseline, 2, 1, None)): + losers = cmp_mod.list_compare_traffic_losers(conn, ctx, {}) + assert losers["total"] == 0 + + # geo_list_tools gaps + low_score_df = pd.DataFrame([{ + "url": "https://ex.com/low", + "status": "200", + "title": "Low", + "content_excerpt": "tiny", + "word_count": 10, + "page_analysis": "{}", + }]) + with patch.object(Ctx, "load_crawl_df", return_value=low_score_df): + assert geo_list_mod.list_pages_ai_citation_signals(conn, ctx, {"min_score": 50})["total"] == 0 + + assert geo_list_mod._parse_robots_txt("") == "" + ok_resp = MagicMock(status_code=200, text="User-agent: *\nAllow: /\n# comment\nUser-agent: ClaudeBot\nDisallow: /") + with patch("website_profiling.tools.audit_tools.geo_list_tools.requests.get", return_value=ok_resp): + robots = geo_list_mod._parse_robots_txt("ex.com") + assert "User-agent" in robots + assert geo_list_mod._agent_blocked(robots, "ClaudeBot") is True + + # --- final line coverage targets --- + assert bl_mod._load_links(Ctx(property_id=None), conn) is None + assert bl_mod.get_anchor_text_distribution(conn, Ctx(property_id=None), {})["error"] + + with patch.object(Ctx, "load_payload", return_value=None): + assert content_mod.list_spell_check_issues(conn, ctx, {})["error"] + + optional_cat = { + "categories": [ + "skip", + {"issues": ["skip", {"message": "needle custom audit on page"}]}, + ], + } + with patch.object(Ctx, "load_payload", return_value=optional_cat): + assert content_mod.list_pagination_issues(conn, ctx, {})["total"] >= 0 + + with patch.object(Ctx, "load_payload", return_value={"content_duplicates": "bad"}): + assert content_mod.list_duplicate_content_pairs(conn, ctx, {})["total"] == 0 + with patch.object(Ctx, "load_payload", return_value={"rich_results_validation": "bad"}): + assert content_mod.list_schema_errors_by_type(conn, ctx, {})["total"] == 0 + + article_types = pd.DataFrame([{ + "url": "https://ex.com/story-longform", + "status": "200", + "title": "Story", + "content_excerpt": "Posted by author on Monday. " + ("word " * 210), + "page_analysis": json.dumps({"json_ld_types": ["WebPage"]}), + }]) + with patch.object(Ctx, "load_crawl_df", return_value=article_types): + content_mod.list_pages_missing_article_schema(conn, ctx, {}) + + kw_skip = { + "text_content_analysis": { + "keyword_index": ["skip", {"word": "needle", "top_pages": [{"url": "https://ex.com/n", "count": 1}]}], + }, + } + with patch.object(Ctx, "load_payload", return_value=kw_skip), patch.object(Ctx, "load_crawl_df", return_value=empty): + assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "needle"})["total"] == 1 + + link_empty = {"link_edges": "bad", "links": ["skip", {"url": "https://ex.com/p", "pagerank": 0.5}]} + with patch.object(Ctx, "load_payload", return_value=link_empty): + assert link_mod._load_link_edges(link_empty) == [] + assert link_mod._pagerank_rows(link_empty)[0]["url"] == "https://ex.com/p" + + improving = {"gsc_full": {"pages": [{"page": "https://ex.com/up", "clicks": 30, "impressions": 300, "position": 3}]}} + declining = {"gsc_full": {"pages": [{"page": "https://ex.com/up", "clicks": 10, "impressions": 100, "position": 5}]}} + assert google_mod._gsc_deltas( + google_mod._gsc_rows(improving, "pages"), + google_mod._gsc_rows(declining, "pages"), + ("page",), + decay=True, + ) == [] + + with patch.object(google_mod, "_load_google_pair", return_value=(None, declining)): + assert google_mod.list_gsc_decaying_pages(conn, ctx, {})["missing"] is True + assert google_mod.list_gsc_new_queries(conn, ctx, {"limit": 1})["missing"] is True + + ga4_empty = {"ga4_full": {"top_pages": [], "by_path": {}}} + with patch.object(Ctx, "load_google_full", return_value=ga4_empty): + assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] == 0 + + mismatch_data = { + "gsc_full": { + "by_page": {"https://ex.com/ratio": {"clicks": 10, "impressions": 50}}, + "pages": [], + }, + "ga4_full": {"by_path": {"/ratio": {"sessions": 40}}}, + } + with patch.object(Ctx, "load_google_full", return_value=mismatch_data): + assert google_mod.list_gsc_ga4_mismatch_pages(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_google_full", return_value={"gsc_full": {"pages": []}}): + assert google_mod.get_gsc_page_trend(conn, ctx, {"url": "https://ex.com/missing"})["missing"] is True + + with patch.object(Ctx, "load_keywords", return_value=None): + assert google_mod.list_gsc_branded_queries(conn, ctx, {})["missing"] is True + assert google_mod.list_gsc_non_branded_queries(conn, ctx, {})["missing"] is True + + with patch.object(google_mod, "_load_google_pair", return_value=(_google_mismatch(), None)): + assert google_mod.compare_gsc_periods(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_payload", return_value={"issues": {"seo": "not-list"}}): + assert issue_mod._issues_by_type(conn, ctx, {}, "x")["total"] == 0 + + bad_crawl = pd.DataFrame([ + {"url": "https://ex.com/badrt", "status": "200", "response_time_ms": "x", "title": "X"}, + {"url": "https://ex.com/badread", "status": "200", "reading_level": "x", "title": "X"}, + ]) + with patch.object(Ctx, "load_payload", return_value={"content_urls": {}}), patch.object(Ctx, "load_crawl_df", return_value=bad_crawl): + assert issue_mod.list_pages_slow_response(conn, ctx, {})["total"] == 0 + assert issue_mod.list_pages_high_reading_level(conn, ctx, {})["total"] == 0 + + lh_skip = { + "lighthouse_by_url": { + "https://ex.com/seo-fail2": {"seo": 40}, + "skip": "x", + }, + } + with patch.object(Ctx, "load_payload", return_value=lh_skip): + assert issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "seo")["total"] >= 1 + assert issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "lcp")["total"] == 0 + + with patch.object(Ctx, "load_payload", return_value={"indexation_coverage": {"lists": {}, "lists_total": {}}}): + idx_mod.list_indexation_submitted_not_indexed(conn, ctx, {}) + idx_mod.list_crawl_urls_not_in_sitemap(conn, ctx, {}) + + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()), patch.object( + Ctx, "load_payload", return_value=_payload(), + ): + assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": "bad"})["paths"] is not None + + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()), patch.object( + Ctx, "load_payload", return_value=None, + ): + assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["error"] + + href_skip = pd.DataFrame([ + {"url": "https://ex.com/a", "status": "200", "page_analysis": json.dumps({ + "hreflang_alternates": ["skip", {"href": "https://ex.com/b", "hreflang": "en"}], + })}, + {"url": "https://ex.com/b", "status": "200", "page_analysis": "{}"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=href_skip): + assert idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame([{"url": "https://ex.com/r2", "status": "301", "redirect_chain_length": 2}])): + idx_mod.list_redirect_chains_by_length(conn, ctx, {"min_length": "bad"}) + + skip_kw = {"rows": [{"keyword": "skip-top10", "gsc_position": 8}]} + skip_prior = {"rows": [{"keyword": "skip-top10", "gsc_position": 5}]} + assert kw_mod._top_ten_transitions(skip_kw, skip_prior, entered=True) == [] + skip_fell = {"rows": [{"keyword": "fell2", "gsc_position": 20}]} + skip_fell_prior = {"rows": [{"keyword": "fell2", "gsc_position": 6}]} + assert kw_mod._top_ten_transitions(skip_fell, skip_fell_prior, entered=False) + + assert kw_mod._rank_delta_rows( + {"rows": [{"keyword": "x", "gsc_position": None}]}, + {"rows": [{"keyword": "x", "gsc_position": 8}]}, + improved=True, + ) == [] + assert kw_mod._rank_delta_rows( + {"rows": [{"keyword": "y", "gsc_position": 10}]}, + {"rows": [{"keyword": "y", "gsc_position": 8}]}, + improved=True, + ) == [] + + with patch.object(Ctx, "load_keywords", return_value=None), patch.object(Ctx, "load_payload", return_value={"semantic_keyword_clusters": [{"keywords": ["a"]}]}): + assert kw_mod._semantic_clusters(ctx, conn) + + with patch.object(Ctx, "load_payload", return_value={"semantic_keyword_clusters": []}), patch.object(Ctx, "load_keywords", return_value=None): + assert kw_mod.list_semantic_cluster_pages(conn, ctx, {})["missing"] is True + + with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "widgets", "gsc_position": 5}]}): + assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "widgets"})["opportunity_score"] >= 0 + assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {"keyword": "missing-kw"})["missing"] is True + + # --- remaining uncovered line targets --- + with patch.object(Ctx, "load_payload", return_value=None): + assert issue_mod._issues_by_type(conn, ctx, {}, "missing_title")["error"] + + with patch.object(Ctx, "load_payload", return_value=None): + assert issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "lcp")["error"] + + lh_bad_cwv = { + "lighthouse_by_url": { + "https://ex.com/bad-cwv": {"lcp": "n/a", "cwv_failures": "lcp", "top_failures": []}, + "https://ex.com/bad-seo": {"seo": "bad"}, + }, + } + with patch.object(Ctx, "load_payload", return_value=lh_bad_cwv): + issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "lcp") + issue_mod._lighthouse_failure_bucket(conn, ctx, {}, "seo") + + under_ctr = { + "gsc_full": { + "pages": [ + {"page": "https://ex.com/high", "clicks": 50, "impressions": 500, "ctr": 0.08, "position": 3}, + {"page": "https://ex.com/low-ctr", "clicks": 1, "impressions": 500, "ctr": 0.01, "position": 5}, + ], + }, + } + with patch.object(Ctx, "load_google_full", return_value=under_ctr): + assert google_mod.list_gsc_ctr_underperformers(conn, ctx, {})["total"] >= 1 + + decay_curr = {"gsc_full": {"queries": [{"query": "decay-q", "clicks": 1, "impressions": 50, "position": 12}]}} + decay_prior = {"gsc_full": {"queries": [{"query": "decay-q", "clicks": 10, "impressions": 200, "position": 5}]}} + with patch.object(google_mod, "_load_google_pair", return_value=(decay_curr, None)): + assert google_mod.list_gsc_decaying_queries(conn, ctx, {})["missing"] is True + assert google_mod.list_gsc_new_queries(conn, ctx, {})["missing"] is True + with patch.object(google_mod, "_load_google_pair", return_value=(None, decay_prior)): + assert google_mod.compare_gsc_periods(conn, ctx, {})["missing"] is True + + ga4_bad_pages = {"ga4_full": {"top_pages": "bad", "by_path": {"/x": {"sessions": 1}}}} + with patch.object(Ctx, "load_google_full", return_value=ga4_bad_pages): + assert google_mod.list_ga4_landing_pages(conn, ctx, {})["total"] == 0 + + assert google_mod._daily_series(None, "gsc", "page", "/") == [] + + with patch.object(Ctx, "load_google_full", return_value={"gsc_full": {"queries": [{"query": "snap-q", "clicks": 2}]}}): + snap = google_mod.get_gsc_query_trend(conn, ctx, {"query": "snap-q"}) + assert snap.get("missing") is True and snap.get("snapshot") + fallback = google_mod.get_gsc_query_trend(conn, ctx, {"query": "not-in-gsc"}) + assert fallback.get("daily") == [] + + with patch.object(Ctx, "load_google_full", return_value=_google_mismatch()): + assert google_mod.list_gsc_pages_by_position_band(conn, ctx, {"max_position": object()})["total"] >= 0 + + mismatch_pages_only = { + "gsc_full": { + "pages": [{"page": "https://ex.com/p-only", "clicks": 15, "impressions": 80, "position": 4}], + }, + "ga4_full": {"by_path": {"/p-only": {"sessions": 0}}}, + } + with patch.object(Ctx, "load_google_full", return_value=mismatch_pages_only): + assert google_mod.list_gsc_ga4_mismatch_pages(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_google_full", return_value=_google_mismatch()): + assert google_mod.list_gsc_pages_by_position_band(conn, ctx, {"min_position": object()})["total"] >= 0 + + fell_still_top = {"rows": [{"keyword": "still-top", "gsc_position": 8}]} + fell_still_prior = {"rows": [{"keyword": "still-top", "gsc_position": 6}]} + assert kw_mod._top_ten_transitions(fell_still_top, fell_still_prior, entered=False) == [] + + assert kw_mod.list_keywords_near_page_one(conn, Ctx(property_id=None, report_id=1), {})["missing"] is True + + cann_empty_url = { + "cannibalisation": [{"query": "q", "pages": [{"url": "", "position": 1}, {"url": "https://ex.com/c", "position": 2}]}], + } + with patch.object(Ctx, "load_keywords", return_value=cann_empty_url): + assert kw_mod.list_cannibalisation_urls(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_keywords", return_value=None): + assert kw_mod.list_cannibalisation_urls(conn, ctx, {})["missing"] is True + assert kw_mod.get_keyword_opportunity_score(conn, ctx, {"keyword": "x"})["missing"] is True + assert kw_mod.get_keyword_serp_snapshot(conn, ctx, {"keyword": "x"})["missing"] is True + + custom_audit = { + "optional_audit_urls": {}, + "categories": [{"issues": [{"message": "custom foo audit detected on page"}]}], + } + with patch.object(Ctx, "load_payload", return_value=custom_audit): + assert content_mod._optional_audit_urls(conn, ctx, {}, "foo")["total"] >= 1 + + kw_index = { + "text_content_analysis": { + "keyword_index": [ + {"word": "unrelated", "top_pages": [{"url": "https://ex.com/x", "count": 1}]}, + {"word": "needleword", "top_pages": [{"url": "https://ex.com/y", "count": 2}]}, + ], + }, + } + with patch.object(Ctx, "load_payload", return_value=kw_index), patch.object(Ctx, "load_crawl_df", return_value=empty): + assert content_mod.list_pages_containing_keyword(conn, ctx, {"keyword": "needle"})["total"] == 1 + + article_str_types = pd.DataFrame([{ + "url": "https://ex.com/blog/posted-by-author", + "status": "200", + "title": "Story", + "content_excerpt": "Posted by author on Monday. " + ("word " * 210), + "page_analysis": json.dumps({"json_ld_types": "WebPage"}), + }]) + with patch.object(Ctx, "load_crawl_df", return_value=article_str_types): + content_mod.list_pages_missing_article_schema(conn, ctx, {}) + + no_cov_payload = {k: v for k, v in _payload().items() if k != "indexation_coverage"} + with patch.object(Ctx, "load_payload", return_value=no_cov_payload): + assert idx_mod.list_indexation_indexed_not_submitted(conn, ctx, {})["error"] + assert idx_mod.list_crawl_urls_not_in_sitemap(conn, ctx, {})["error"] + + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()): + assert idx_mod.list_log_5xx_paths(conn, Ctx(property_id=None, report_id=1), {})["error"] + assert idx_mod.list_log_googlebot_low_crawl(conn, Ctx(property_id=None, report_id=1), {})["error"] + + bot_log = { + "analysis": { + "top_paths": ["skip", {"path": "/popular", "hits": 100}, {"path": "/crawled", "hits": 50}], + "googlebot_paths": [{"path": "/popular", "hits": 2}, {"path": "/crawled", "hits": 1}], + }, + } + bot_payload = { + **_payload(), + "links": [{"url": "https://ex.com/crawled"}, {"url": "https://ex.com/popular"}], + } + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=bot_log), patch.object( + Ctx, "load_payload", return_value=bot_payload, + ): + assert idx_mod.list_log_googlebot_low_crawl(conn, ctx, {"min_hits": 20, "max_googlebot_hits": 5})["total"] == 0 + + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=_log_row()): + assert idx_mod.list_log_orphan_high_traffic(conn, Ctx(property_id=None, report_id=1), {})["error"] + + orphan_payload = { + **_payload(), + "orphan_urls": ["", "https://ex.com/orphan"], + } + orphan_log2 = { + "analysis": { + "top_paths": ["skip", {"path": "/orphan", "hits": 50}], + }, + } + with patch("website_profiling.tools.audit_tools.indexation_lists._load_log_analysis", return_value=orphan_log2), patch.object( + Ctx, "load_payload", return_value=orphan_payload, + ): + assert idx_mod.list_log_orphan_high_traffic(conn, ctx, {})["total"] >= 1 + + with patch.object(Ctx, "load_crawl_df", return_value=pd.DataFrame()): + assert idx_mod.list_redirect_chains_by_length(conn, ctx, {})["pages"] == [] + + href_empty_url = pd.DataFrame([ + {"url": "", "status": "200", "page_analysis": "{}"}, + {"url": "https://ex.com/a", "status": "200", "page_analysis": json.dumps({ + "hreflang_alternates": [{"href": "https://ex.com/b", "hreflang": "en"}], + })}, + {"url": "https://ex.com/b", "status": "200", "page_analysis": "{}"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=href_empty_url): + idx_mod.list_hreflang_reciprocal_gaps(conn, ctx, {}) + + bad_graph = {"link_edges": [], "graph_edges": "bad", "top_pages": "bad", "links": [{"url": "https://ex.com/p", "pagerank": 0.3}]} + with patch.object(Ctx, "load_payload", return_value=bad_graph): + assert link_mod._load_link_edges(bad_graph) == [] + assert link_mod._pagerank_rows(bad_graph) == [] + diff --git a/tests/test_audit_tools_expanded.py b/tests/test_audit_tools_expanded.py index 8cc772e8..ead7e11f 100644 --- a/tests/test_audit_tools_expanded.py +++ b/tests/test_audit_tools_expanded.py @@ -178,7 +178,7 @@ def conn() -> MagicMock: def test_handler_schema_parity() -> None: names = {t["name"] for t in TOOL_DEFINITIONS} assert names == tool_handler_names() - assert len(TOOL_DEFINITIONS) == 240 + assert len(TOOL_DEFINITIONS) == 340 def test_slice_helpers() -> None: diff --git a/tests/test_log_parser.py b/tests/test_log_parser.py index 8d32165d..c2add630 100644 --- a/tests/test_log_parser.py +++ b/tests/test_log_parser.py @@ -10,6 +10,14 @@ def test_parse_access_log_lines_counts_googlebot() -> None: assert out["top_paths"][0]["path"] == "/page" +def test_parse_access_log_lines_tracks_5xx_paths() -> None: + lines = [ + '127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /broken HTTP/1.1" 503 0 "-" "curl/8.0"', + ] + out = parse_access_log_lines(lines) + assert out["paths_5xx"] == [{"path": "/broken", "hits": 1}] + + def test_parse_access_log_lines_skips_blank_and_comments() -> None: lines = ["", "# comment", "not-a-log-line"] out = parse_access_log_lines(lines) diff --git a/tests/test_mcp_registry.py b/tests/test_mcp_registry.py index 4fa53225..c693d500 100644 --- a/tests/test_mcp_registry.py +++ b/tests/test_mcp_registry.py @@ -13,7 +13,7 @@ def test_tool_definitions_schema() -> None: - assert len(TOOL_DEFINITIONS) == 240 + assert len(TOOL_DEFINITIONS) == 340 for tool in TOOL_DEFINITIONS: assert tool.get("name") assert tool.get("description") diff --git a/tests/test_mcp_server_helpers.py b/tests/test_mcp_server_helpers.py index eddb9f76..ca2c1428 100644 --- a/tests/test_mcp_server_helpers.py +++ b/tests/test_mcp_server_helpers.py @@ -62,7 +62,7 @@ def test_read_glossary_excerpt_missing(monkeypatch) -> None: def test_tools_catalog_json_includes_security_tools() -> None: with patch.dict(os.environ, {"WP_MCP_DOMAIN": "full"}): catalog = json.loads(mcp_server._tools_catalog_json()) - assert catalog["tool_count"] >= 240 + assert catalog["tool_count"] >= 340 assert "get_security_findings" in catalog["domains"]["security"] assert "get_geo_readiness_score" in catalog["domains"]["geo"] assert "get_gsc_url_inspection" in catalog["domains"]["integrations"] @@ -173,7 +173,7 @@ async def __aexit__(self, *_args): assert captured["name"] == "site-audit-full" assert captured["ran"] is True tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type] - assert len(tools) >= 240 + assert len(tools) >= 340 resources = asyncio.run(captured["list_resources"]()) # type: ignore[arg-type] assert any(r["uri"] == "audit://property/7" for r in resources) assert any(r["uri"] == "audit://domains" for r in resources) @@ -183,6 +183,8 @@ async def __aexit__(self, *_args): assert content[0]["text"] == json.dumps({"ok": True}, indent=2, default=str) read_text = asyncio.run(captured["read_resource"]("audit://tools")) # type: ignore[arg-type] assert read_text.startswith("{") + domains_text = asyncio.run(captured["read_resource"]("audit://domains")) # type: ignore[arg-type] + assert "current_mcp_domain" in domains_text def test_mcp_call_tool_rejects_tools_outside_domain(monkeypatch) -> None: @@ -244,10 +246,27 @@ async def __aexit__(self, *_args): with patch.dict(os.environ, {"WP_MCP_DOMAIN": "core"}, clear=False): mcp_server.main() + tools = asyncio.run(captured["list_tools"]()) # type: ignore[arg-type] + assert len(tools) < 340 blocked = asyncio.run(captured["call_tool"]("export_audit_report", {"format": "pdf"})) # type: ignore[arg-type] assert "not exposed" in blocked[0]["text"] +def test_mcp_core_server_main() -> None: + with patch("website_profiling.mcp.core_server.run_domain_server") as mock_run: + from website_profiling.mcp import core_server + core_server.main() + mock_run.assert_called_once_with("core") + + +def test_mcp_domain_server_sets_env() -> None: + with patch("website_profiling.mcp.domain_server.main") as mock_main: + from website_profiling.mcp.domain_server import run_domain_server + run_domain_server("google") + assert os.environ.get("WP_MCP_DOMAIN") == "google" + mock_main.assert_called_once() + + def test_mcp_package_main(monkeypatch) -> None: with patch("website_profiling.mcp.server.main") as mock_main: runpy.run_module("website_profiling.mcp", run_name="__main__") diff --git a/tests/test_tools_gate100_coverage.py b/tests/test_tools_gate100_coverage.py new file mode 100644 index 00000000..c547f88d --- /dev/null +++ b/tests/test_tools_gate100_coverage.py @@ -0,0 +1,556 @@ +"""Coverage for batch-100 foundation modules (data_coverage, insight, router, registry).""" +from __future__ import annotations + +import os +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from website_profiling.tools.audit_tools import insight_helpers as ih +from website_profiling.tools.audit_tools.context import AuditToolContext as Ctx +from website_profiling.tools.audit_tools import ( + crawl as crawl_mod, + data_coverage as dc_mod, + google as google_mod, + insight_tools as insight_mod, + keywords as kw_mod, + registry, + router_tools as router_mod, +) +from website_profiling.tools.audit_tools.tool_domains import ( + CANONICAL_DOMAINS, + TIER_0_TOOLS, + classify_tool_domain, + domains_catalog, + tool_names_for_mcp_bundle, + tool_names_for_tier, + tools_by_domain, +) +from website_profiling.tools.audit_tools.tool_selector import ( + apply_tool_cap, + chat_tool_max, + compact_tool_list, + select_tools_for_turn, +) + + +@pytest.fixture +def conn() -> MagicMock: + return MagicMock() + + +@pytest.fixture +def ctx() -> Ctx: + return Ctx(property_id=1, report_id=1) + + +def test_context_load_google_full_and_pair_fallbacks(conn: MagicMock, ctx: Ctx) -> None: + embedded = {"gsc": {"summary": {"clicks": 1}}, "fetched_at": "2026-01-01"} + prior = {"gsc_full": {"summary": {"clicks": 0}}} + + with patch( + "website_profiling.tools.audit_tools.context.read_google_data_full", + return_value=None, + ), patch( + "website_profiling.tools.audit_tools.context.read_report_payload", + return_value={"google": embedded}, + ): + assert ctx.load_google_full(conn) == embedded + + with patch( + "website_profiling.tools.audit_tools.context.read_google_data_full", + return_value=None, + ), patch( + "website_profiling.tools.audit_tools.context.read_prior_google_snapshot", + return_value=prior, + ), patch( + "website_profiling.tools.audit_tools.context.read_report_payload", + return_value={"google": embedded}, + ): + current, prior_out = ctx.load_google_pair(conn) + assert current == embedded + assert prior_out == prior + + with patch( + "website_profiling.tools.audit_tools.context.read_google_data_full", + return_value={"gsc_full": {"summary": {}}}, + ): + assert ctx.load_google_full(conn) == {"gsc_full": {"summary": {}}} + + +def test_data_coverage_report_all_branches(conn: MagicMock, ctx: Ctx) -> None: + assert dc_mod.get_data_coverage_report(conn, Ctx(property_id=None), {})["error"] + + with patch("website_profiling.tools.audit_tools.data_coverage.get_property_by_id", return_value=None): + assert dc_mod.get_data_coverage_report(conn, ctx, {})["error"] == "property not found" + + prop = {"google_refresh_token": "tok"} + payload = { + "image_inventory": [{"url": "x"}], + "axe_audit_summary": {"violations": 1}, + "rich_results_validation": {"ok": True}, + "text_content_analysis": {"words": 100}, + "semantic_keyword_clusters": [{"k": "a"}], + "log_analysis": {"paths": []}, + } + google = { + "gsc": {"summary": {"clicks": 10}}, + "ga4": {"summary": {"sessions": 20}}, + } + keywords = {"rows": [{"keyword": "a"}], "fetched_at": "2026-01-01"} + gsc_links = { + "sample_links": [{"url": "x"}], + "third_party_overlays": [{"source": "moz"}], + } + google_full = {"gsc_full": {"summary": {}}, "ga4_full": {"summary": {}}} + + with patch("website_profiling.tools.audit_tools.data_coverage.get_property_by_id", return_value=prop), patch.object( + Ctx, "load_payload", return_value=payload, + ), patch.object(Ctx, "load_google", return_value=google), patch.object( + Ctx, "load_keywords", return_value=keywords, + ), patch.object(Ctx, "load_gsc_links", return_value=gsc_links), patch.object( + Ctx, "load_google_full", return_value=google_full, + ), patch( + "website_profiling.integrations.google.store.read_prior_google_snapshot", + return_value={"gsc": {}}, + ): + result = dc_mod.get_data_coverage_report(conn, ctx, {}) + assert result["missing_count"] == 0 + assert len(result["checks"]) >= 10 + + sparse_prop = {"id": 1} + with patch("website_profiling.tools.audit_tools.data_coverage.get_property_by_id", return_value=sparse_prop), patch.object( + Ctx, "load_payload", return_value={}, + ), patch.object(Ctx, "load_google", return_value=None), patch.object( + Ctx, "load_keywords", return_value=None, + ), patch.object(Ctx, "load_gsc_links", return_value=None), patch.object( + Ctx, "load_google_full", return_value=None, + ), patch( + "website_profiling.integrations.google.store.read_prior_google_snapshot", + return_value=None, + ): + sparse = dc_mod.get_data_coverage_report(conn, ctx, {}) + assert sparse["missing_count"] > 0 + assert sparse["checks"][0]["config_hint"] + + +def test_google_series_and_page_queries(conn: MagicMock, ctx: Ctx) -> None: + assert google_mod.get_gsc_daily_trend(conn, ctx, {})["missing"] + assert google_mod.get_ga4_daily_trend(conn, ctx, {})["missing"] + assert google_mod.get_ga4_by_device(conn, ctx, {})["missing"] + assert google_mod.get_ga4_by_channel(conn, ctx, {})["missing"] + assert google_mod.get_gsc_page_queries(conn, ctx, {})["error"] == "url is required" + with patch.object(Ctx, "load_google_full", return_value=None), patch.object(Ctx, "load_google", return_value=None): + assert google_mod.get_gsc_page_queries(conn, ctx, {"url": "https://ex.com/"})["missing"] + + raw = { + "gsc_full": { + "by_page": { + "https://ex.com/": { + "queries": [{"query": "q1", "clicks": 1}], + }, + }, + }, + "fetched_at": "2026-01-01", + } + with patch.object(Ctx, "load_google_full", return_value=raw): + ok = google_mod.get_gsc_page_queries(conn, ctx, {"url": "https://ex.com/"}) + assert ok["total"] == 1 + + google_data = { + "gsc": {"daily": [{"date": "2026-01-01", "clicks": 1}]}, + "ga4": {"daily": [], "by_device": [{"device": "mobile"}], "by_channel": [{"channel": "organic"}]}, + "fetched_at": "2026-01-01", + "date_range": "28d", + } + with patch.object(Ctx, "load_google", return_value=google_data): + assert google_mod.get_gsc_daily_trend(conn, ctx, {})["provenance"] == "Search Console" + assert google_mod.get_ga4_daily_trend(conn, ctx, {})["provenance"] == "Google Analytics 4" + assert google_mod.get_ga4_by_device(conn, ctx, {})["by_device"] + assert google_mod.get_ga4_by_channel(conn, ctx, {})["by_channel"] + + +def test_insight_helpers_all_branches() -> None: + assert ih._num("bad", 5.0) == 5.0 + assert ih.classify_opportunity_quadrant( + {"position": 10, "impressions": 200}, {"sessions": 20, "engagementRate": 0.6}, + ) == "high_impact" + assert ih.classify_opportunity_quadrant( + {"position": 10, "impressions": 200}, {"sessions": 0, "engagementRate": 0.1}, + ) == "worth_optimizing" + assert ih.classify_opportunity_quadrant( + {"position": 1, "impressions": 10}, {"sessions": 100, "engagementRate": 0.9}, + ) == "good_but_capped" + assert ih.classify_opportunity_quadrant({"position": 1, "impressions": 1}, None) == "low_priority" + + assert ih.traffic_health_ratio({}, {})["diagnosis"] == "no_data" + low = ih.traffic_health_ratio({"clicks": 100}, {"sessions": 10}) + assert low["diagnosis"] == "tracking_gap" + high = ih.traffic_health_ratio({"clicks": 10}, {"sessions": 100}) + assert high["diagnosis"] == "filter_issue" + + gsc_pages = { + "https://ex.com/a": {"impressions": 500, "clicks": 10, "position": 8, "ctr": 0.02}, + "https://ex.com/skip": "not-a-dict", + "https://ex.com/low": {"impressions": 5, "clicks": 0, "position": 50, "ctr": 0.001}, + } + ga4_paths = { + "/a": {"full_url": "https://ex.com/a", "sessions": 15, "engagementRate": 0.55}, + "/by-path": {"sessions": 8, "engagementRate": 0.4}, + "bad": "skip", + } + rows = ih.blend_landing_pages(gsc_pages, ga4_paths, limit=5, min_impressions=100) + assert rows[0]["quadrant"] == "high_impact" + assert rows[0]["ga4_sessions"] == 15 + path_match = ih.blend_landing_pages( + {"https://ex.com/by-path": {"impressions": 200, "clicks": 1, "position": 15, "ctr": 0.01}}, + ga4_paths, + limit=5, + min_impressions=0, + ) + assert path_match[0]["ga4_sessions"] == 8 + + payload = { + "categories": [ + "bad", + { + "id": "onpage", + "issues": [ + {"url": "https://ex.com/page", "priority": "High", "message": "missing title"}, + {"url": "https://other.com", "priority": "Low", "message": "skip"}, + "bad-issue", + {"priority": "Medium", "message": "site-wide"}, + ], + }, + ], + } + flags = ih.page_issue_flags("https://ex.com/page", payload) + assert flags[0]["message"] == "missing title" + assert any(f.get("message") == "site-wide" for f in flags) + + green = ih.composite_page_score( + {"position": 5}, + {"engagementRate": 0.8}, + {"position": 10}, + {"engagementRate": 0.5}, + [], + {"performance": 90, "seo": 95}, + ) + assert green["band"] == "green" + + amber = ih.composite_page_score( + {"position": 20}, + {"engagementRate": 0.5}, + {"position": 10}, + {"engagementRate": 0.5}, + [{"priority": "High"}], + None, + ) + assert amber["band"] == "amber" + + red = ih.composite_page_score( + {"position": 30}, + {"engagementRate": 0.1}, + {"position": 10}, + {"engagementRate": 0.5}, + [{"priority": "Critical"}, {"priority": "Critical"}], + {"performance": 30, "seo": 50}, + ) + assert red["band"] == "red" + + +def test_insight_tools_dispatch(conn: MagicMock, ctx: Ctx) -> None: + assert insight_mod.get_landing_page_blended_table(conn, ctx, {})["missing"] + + google_full = { + "gsc_full": { + "by_page": {"https://ex.com/": {"impressions": 500, "clicks": 5, "position": 12, "ctr": 0.01}}, + "top_pages": [{"page": "https://ex.com/", "impressions": 500, "clicks": 5, "position": 12}], + }, + "ga4_full": {"by_path": {"/": {"sessions": 20, "engagementRate": 0.6, "full_url": "https://ex.com/"}}}, + "fetched_at": "2026-01-01", + } + payload = { + "categories": [{"id": "c", "issues": [{"url": "https://ex.com/", "priority": "High", "message": "m"}]}], + "lighthouse_by_url": { + "https://ex.com/": {"performance": 80, "seo": 90}, + "https://ex.com": {"performance": 70, "seo": 80}, + }, + "top_pages": [{"url": "https://ex.com/", "title": "Home"}], + } + + with patch.object(Ctx, "load_google_full", return_value={ + "gsc_full": {"top_pages": [{"page": "https://ex.com/", "impressions": 100, "clicks": 1, "position": 10}]}, + "ga4_full": {"by_path": {}}, + "fetched_at": "2026-01-01", + }): + top_pages_only = insight_mod.get_landing_page_blended_table(conn, ctx, {}) + assert top_pages_only["total"] >= 0 + + with patch.object(Ctx, "load_google_full", return_value=google_full): + blended = insight_mod.get_landing_page_blended_table(conn, ctx, {}) + assert blended["total"] >= 0 + matrix = insight_mod.get_opportunity_matrix(conn, ctx, {}) + assert "counts" in matrix + + with patch.object(Ctx, "load_google_full", return_value=None), patch.object( + Ctx, "load_google", return_value={"gsc": {"summary": {"clicks": 1}}, "ga4": {"summary": {"sessions": 2}}}, + ): + health = insight_mod.get_traffic_health_check(conn, ctx, {}) + assert "diagnosis" in health + + assert insight_mod.get_landing_page_full_diagnosis(conn, ctx, {})["error"] == "url is required" + with patch.object(Ctx, "load_payload", return_value=None): + assert insight_mod.get_landing_page_full_diagnosis(conn, ctx, {"url": "https://ex.com/"})["missing"] + + with patch.object(Ctx, "load_payload", return_value=payload), patch.object( + Ctx, "load_google_full", return_value=google_full, + ): + diag = insight_mod.get_landing_page_full_diagnosis(conn, ctx, {"url": "https://ex.com/"}) + assert diag["url"] == "https://ex.com/" + assert "diagnosis" in diag + + with patch.object(Ctx, "load_google_full", return_value=None), patch.object(Ctx, "load_google", return_value=None): + assert insight_mod.get_traffic_health_check(conn, ctx, {})["missing"] + + with patch.object(insight_mod, "get_landing_page_blended_table", return_value={"error": "no google data", "missing": True}): + assert insight_mod.get_opportunity_matrix(conn, ctx, {})["missing"] + + with patch.object(Ctx, "load_payload", return_value={ + "lighthouse_by_url": {"https://ex.com": {"performance": 70, "seo": 80}}, + "top_pages": [], + }), patch.object(Ctx, "load_google_full", return_value=google_full): + slash_diag = insight_mod.get_landing_page_full_diagnosis(conn, ctx, {"url": "https://ex.com/"}) + assert slash_diag["lighthouse"]["performance"] == 70 + + with patch("website_profiling.tools.audit_tools.insight_tools.list_issues", return_value={"error": "boom"}): + assert insight_mod.get_issue_to_traffic_map(conn, ctx, {})["error"] == "boom" + + with patch("website_profiling.tools.audit_tools.insight_tools.list_issues", return_value={ + "issues": ["bad", { + "url": "https://ex.com/x", + "priority": "High", + "category": "onpage", + "message": "issue", + "impact_score": 10, + "gsc_clicks": 5, + "ga4_sessions": 3, + }], + "total": 1, + "truncated": False, + }): + mapped2 = insight_mod.get_issue_to_traffic_map(conn, ctx, {}) + assert mapped2["total"] == 1 + + issues_payload = { + "categories": [{ + "id": "c", + "issues": [{ + "url": "https://ex.com/x", + "priority": "High", + "category": "onpage", + "message": "issue", + "impact_score": 10, + "gsc_clicks": 5, + "ga4_sessions": 3, + }, "bad"], + }], + } + with patch.object(Ctx, "load_payload", return_value=issues_payload): + mapped = insight_mod.get_issue_to_traffic_map(conn, ctx, {}) + assert mapped["total"] == 1 + + +def test_router_tools_workflows(conn: MagicMock, ctx: Ctx) -> None: + assert router_mod.search_audit_tools(conn, ctx, {})["error"] == "query is required" + found = router_mod.search_audit_tools(conn, ctx, {"query": "broken links", "limit": 3}) + assert found["total"] >= 1 + assert found["tool_names"] + + domains = router_mod.list_tool_domains(conn, ctx, {}) + assert domains["domains"] + assert domains["domain_tool_counts"] + + with patch.object(router_mod, "_dispatch", return_value={"ok": True}) as dispatch: + traffic = router_mod.run_insight_workflow(conn, ctx, {"type": "traffic"}) + assert traffic["steps"][0]["tool"] == "get_traffic_health_check" + landing = router_mod.run_insight_workflow(conn, ctx, {"type": "landing_pages"}) + assert len(landing["steps"]) == 2 + default = router_mod.run_insight_workflow(conn, ctx, {"type": "priorities"}) + assert len(default["steps"]) == 2 + + tech = router_mod.run_technical_workflow(conn, ctx, {"baseline_report_id": 99}) + assert any(s["tool"] == "compare_issue_deltas" for s in tech["steps"]) + + assert router_mod.run_keyword_workflow(conn, Ctx(property_id=None), {})["error"] + kw = router_mod.run_keyword_workflow(conn, ctx, {}) + assert kw["workflow"] == "keyword" + assert len(kw["steps"]) == 3 + assert dispatch.call_count >= 6 + + assert router_mod.run_domain_agent(conn, ctx, {})["error"] == "task is required" + + with patch( + "website_profiling.tools.audit_tools.registry.search_tools", + return_value=[], + ), patch( + "website_profiling.tools.audit_tools.registry.tool_names_for_domain", + return_value=["get_schema_coverage", "list_broken_links"], + ), patch.object(router_mod, "_dispatch", return_value={"ok": True}): + domain_only = router_mod.run_domain_agent(conn, ctx, {"task": "schema", "domain": "schema"}) + assert domain_only["tools_used"] + + with patch( + "website_profiling.tools.audit_tools.registry.search_tools", + return_value=[{"name": "get_schema_coverage"}], + ), patch( + "website_profiling.tools.audit_tools.registry.tool_names_for_domain", + return_value=["get_schema_coverage"], + ), patch.object(router_mod, "_dispatch", return_value={"ok": True}) as dispatch2: + in_pool = router_mod.run_domain_agent(conn, ctx, {"task": "schema audit", "domain": "schema", "max_steps": 2}) + assert in_pool["tools_used"] == ["get_schema_coverage"] + assert dispatch2.call_count == 1 + + with patch( + "website_profiling.tools.audit_tools.registry.search_tools", + return_value=[{"name": "get_report_summary"}], + ), patch( + "website_profiling.tools.audit_tools.registry.tool_names_for_domain", + return_value=[], + ), patch.object(router_mod, "_dispatch", return_value={"ok": True}): + global_pick = router_mod.run_domain_agent(conn, ctx, {"task": "report overview", "domain": "schema", "max_steps": 1}) + assert global_pick["tools_used"] == ["get_report_summary"] + + with patch( + "website_profiling.tools.audit_tools.registry.search_tools", + return_value=[], + ), patch( + "website_profiling.tools.audit_tools.registry.tool_names_for_domain", + return_value=["get_schema_coverage"], + ), patch.object(router_mod, "_dispatch", return_value={"ok": True}): + domain_fallback = router_mod.run_domain_agent(conn, ctx, {"task": "schema", "domain": "schema"}) + assert domain_fallback["tools_used"] == ["get_schema_coverage"] + + with patch( + "website_profiling.tools.audit_tools.registry.search_tools", + return_value=[ + {"name": "list_broken_links"}, + {"name": "get_schema_coverage"}, + ], + ), patch.object(router_mod, "_dispatch", return_value={"ok": True}): + no_domain = router_mod.run_domain_agent(conn, ctx, {"task": "broken links audit", "max_steps": 1}) + assert no_domain["tools_used"] == ["list_broken_links"] + + +def test_registry_helpers_and_validation_errors() -> None: + assert registry.tool_definition("get_report_summary") is not None + assert registry.tool_definition("__missing__") is None + assert registry.tool_names_for_tier(0) + assert registry.list_domains_catalog() + assert registry.search_tools("") == [] + assert registry.search_tools("get_report_summary", limit=5)[0]["name"] == "get_report_summary" + + filtered = registry.openai_tools_schema({"get_report_summary"}) + assert len(filtered) == 1 + assert filtered[0]["function"]["name"] == "get_report_summary" + + with patch.object(registry, "_TOOL_HANDLERS", {"a": lambda *a, **k: {}}): + with patch.object(registry, "TOOL_DEFINITIONS", [{"name": "b", "description": "", "inputSchema": {}}]): + errors = registry.validate_tool_registry() + assert any("handler/catalog mismatch" in e for e in errors) + + with patch.object(registry, "_TOOL_HANDLERS", {"a": lambda *a, **k: {}}): + with patch.object(registry, "TOOL_DEFINITIONS", [{"name": "a", "description": "", "inputSchema": {}}]): + with patch.object(registry, "_TOOL_META", {"b": {"domain": "core", "tier": 1}}): + errors = registry.validate_tool_registry() + assert any("handler/meta mismatch" in e for e in errors) + + all_handlers = registry.tool_handler_names() + missing_t0 = next(iter(registry.tier0_tool_names())) + with patch.object(registry, "tool_handler_names", return_value=all_handlers - {missing_t0}): + errors = registry.validate_tool_registry() + assert any("tier0 tools missing" in e for e in errors) + + +def test_tool_domains_classify_and_catalog() -> None: + with patch( + "website_profiling.tools.audit_tools.tool_domains.TIER_0_TOOLS", + frozenset({"synthetic_tier0_tool"}), + ): + assert classify_tool_domain("synthetic_tier0_tool") == "core" + + assert classify_tool_domain("get_landing_page_custom") == "insight" + assert classify_tool_domain("get_page_ctr_detail") == "ctr" + assert classify_tool_domain("list_competitor_domains") == "backlinks" + assert classify_tool_domain("list_compare_new_issues") == "drift" + + meta = registry.tool_meta() + meta_with_unknown = {**meta, "orphan_tool": {"domain": "noncanonical_domain", "tier": 1}} + by_domain = tools_by_domain(meta_with_unknown) + assert "noncanonical_domain" in by_domain + + tier1 = tool_names_for_tier(meta, 1) + assert tier1 + full_bundle = tool_names_for_mcp_bundle(meta, "full") + assert len(full_bundle) == len(meta) + core_bundle = tool_names_for_mcp_bundle(meta, "core") + assert TIER_0_TOOLS <= core_bundle + + catalog = domains_catalog(meta) + assert catalog + assert all(row["domain"] in CANONICAL_DOMAINS for row in catalog) + assert domains_catalog({}) == [] + + +def test_tool_selector_edge_cases() -> None: + with patch.dict(os.environ, {"CHAT_TOOL_MAX": "not-a-number"}): + assert chat_tool_max() >= len(TIER_0_TOOLS) + 1 + + history = [ + {"role": "assistant", "content": "hi"}, + {"role": "user", "content": "show gsc clicks and ga4 landing pages"}, + ] + names = select_tools_for_turn("show gsc clicks", history=history) + assert "get_google_summary" in names or "get_gsc_top_queries" in names + + with_extra = select_tools_for_turn("hello", extra_names={"search_audit_tools"}) + assert "search_audit_tools" in with_extra + + capped = apply_tool_cap(set(f"tool_{i}" for i in range(200)) | set(TIER_0_TOOLS), 50) + assert len(capped) <= 50 + assert TIER_0_TOOLS <= capped + + text = compact_tool_list({"b_tool", "a_tool"}) + assert text.startswith("- a_tool") + + +def test_keywords_brand_and_intent(conn: MagicMock, ctx: Ctx) -> None: + assert kw_mod.get_brand_keyword_split(conn, Ctx(property_id=None), {})["error"] + with patch.object(Ctx, "load_keywords", return_value=None): + assert kw_mod.get_brand_keyword_split(conn, ctx, {})["missing"] + with patch.object(Ctx, "load_keywords", return_value={ + "brand_name": "Acme", + "rows": [ + {"keyword": "acme shoes", "is_branded": True}, + {"keyword": "buy shoes", "is_branded": False}, + ], + }): + split = kw_mod.get_brand_keyword_split(conn, ctx, {}) + assert split["branded_count"] == 1 + assert split["non_branded_count"] == 1 + + assert kw_mod.list_keywords_by_intent(conn, ctx, {})["error"] == "intent is required" + with patch.object(Ctx, "load_keywords", return_value={"rows": [{"keyword": "a", "intent": "informational"}]}): + assert kw_mod.list_keywords_by_intent(conn, ctx, {"intent": "informational"})["total"] == 1 + + +def test_crawl_js_delta_skips_empty_url(conn: MagicMock, ctx: Ctx) -> None: + df = pd.DataFrame([ + {"url": "", "fetch_method": "static", "word_count": 100, "title": "A", "h1": "H"}, + {"url": "https://ex.com/", "fetch_method": "static", "word_count": 100, "title": "A", "h1": "H"}, + {"url": "https://ex.com/", "fetch_method": "rendered", "word_count": 200, "title": "B", "h1": "H2"}, + ]) + with patch.object(Ctx, "load_crawl_df", return_value=df): + result = crawl_mod.list_pages_js_rendering_delta(conn, ctx, {}) + assert result["total"] == 1