+
+
+
+
+
+
+ Site Audit — Open Source SEO Crawl & Audit
+ Free, self-hosted — no vendor paywalls.
+
+ 
+ 
+ 
+ 
+ Quick start · + Features · + Structure · + Contributing · + Docs · + License +
+ +--- + # Site Audit -Open-source technical SEO crawl and audit UI (Next.js + Python + PostgreSQL). +**Open Source SEO Crawl & Audit** — self-hosted UI built with **Next.js + Python + PostgreSQL**. + +Repository: [codefrydev/WebsiteProfiling](https://github.com/codefrydev/WebsiteProfiling) ## Overview @@ -8,6 +46,87 @@ Open-source technical SEO crawl and audit UI (Next.js + Python + PostgreSQL). **Goal** — A free, self-hosted audit you control: crawl your sites, see honest technical SEO issues, connect Search Console and Analytics when you want, and export reports for clients — without a vendor sitting between you and the data. +## Features + +
+  + Site crawl + Static & JS rendering, sitemap export, crawl maps + |
+
+  + Technical audit + Issues, Lighthouse, on-page checks, workbooks + |
+
+  + Integrations + Google Search Console, GA4, Bing Webmaster + |
+
+  + Self-hosted + Docker or local dev — your data stays yours + |
+
+ 
+
+ {error.message || strings.app.failedHint} +
++ {useGscMode ? vo.keywordOpportunitiesGscHint : vo.keywordOpportunitiesHint} +
+ + {!hasGscEnrichment && !hasGoogleConnected ? ( ++ {ke.dataStatus.noGscDetail} +
+ ) : null} + + {(useGscMode || showCrawlColumns || showSiteTerms) && ( +{vo.keywordOpportunitiesHint}
-{label}
++ {shortCategoryLabel(cat)} +
+{cat.score}
+ {cat.issueCount > 0 ? ( ++ {format(issueLabel, { count: cat.issueCount })} +
+ ) : null} ++ {vh.deleteConfirmTitle} +
++ {group.crawlOnly + ? format(vh.deleteConfirmCrawlOnly, { + name: group.domainName, + count: group.urlCount.toLocaleString(), + }) + : format(vh.deleteConfirmBody, { name: group.domainName })} +
+{vh.crawlUrlLabel}
+ e.stopPropagation()} + className="inline-flex max-w-full items-center gap-1 text-xs sm:text-sm text-link hover:underline" + title={group.crawlUrl} + > + {group.crawlUrl} ++ {vh.crawlConfigLabel} +
+{crawlConfigSegments.join(' · ')}
++ {strings.views.overview.dataSourcesLabel} +
+{vh.urlCountLabel}
++ {group.urlCount.toLocaleString()} +
+{vh.titleCoverageLabel}
++ {group.titleCoverage != null ? `${group.titleCoverage}%` : sj.emDash} +
++ {vh.lastCrawlLabel}: + {group.lastCrawl || sj.emDash} +
+{vh.avgWordCountLabel}
++ {group.avgWordCount != null ? group.avgWordCount.toLocaleString() : sj.emDash} +
+{vh.thinPagesLabel}
++ {group.thinPages != null ? group.thinPages.toLocaleString() : sj.emDash} +
++ {vh.crawlOnlyHint} +
+{vh.categoryScoresLabel}
+{vh.urlCountLabel}
+{group.urlCount.toLocaleString()}
+ {group.medianWordCount != null ? ( ++ {vh.medianWordsLabel}: {group.medianWordCount.toLocaleString()} +
+ ) : null} + {group.medianResponseMs != null ? ( ++ {format(vh.responseTimeValue, { ms: group.medianResponseMs.toLocaleString() })} +
+ ) : null} +
+
{group.totalIssues.toLocaleString()}
+
+
+ {vh.perfScoreLabel} + {group.perfScore ?? sj.emDash} +
++ {vh.seoScoreLabel} + {group.seoScore ?? sj.emDash} +
++ {vh.trendUrgentLabel}: {trends.urgentCount} +
+ ) : null} +{vh.seoSignalsLabel}
+{vh.lastCrawlLabel}
++ {group.lastCrawl || sj.emDash} +
+{vh.lastAuditLabel}
++ {group.lastAudit || sj.emDash} +
+{vh.crawlDurationLabel}
++ {format(vh.crawlDurationValue, { seconds: group.crawlDurationS.toLocaleString() })} +
+{vh.crawlTrendsLabel}
+ {trends.hasCrawlTrendLines ? ( +{vh.crawlTrendsNeedHistory}
+ )} +{vh.trendsLabel}
+ {trends.hasAuditTrendLines ? ( +{vh.trendsNeedHistory}
+ )} +
{vca.totalCrawled}{' '}
- {Number(summary.total_urls || 0).toLocaleString()}
+ {crawledCount.toLocaleString()}
{summary.success_rate != null && (
<> · {summary.success_rate}% {vca.returned2xx}
diff --git a/web/src/views/Home.tsx b/web/src/views/Home.tsx
index 38fe082c..870f120b 100644
--- a/web/src/views/Home.tsx
+++ b/web/src/views/Home.tsx
@@ -1,36 +1,20 @@
-import { Building2, ChevronDown, ExternalLink, Globe, ArrowRight, Search, Trash2 } from 'lucide-react';
+import { Building2, ChevronDown, Search } from 'lucide-react';
import { useMemo, useState, useEffect, useCallback } from 'react';
import AppLogo from '@/components/AppLogo';
import { PageLayout, Card } from '../components';
-import HealthSparkline from '@/components/HealthSparkline';
+import PortfolioPropertyCard from '@/components/portfolio/PortfolioPropertyCard';
+import { healthScoreClass, portfolioCardKey } from '@/components/portfolio/portfolioCardUtils';
import { Skeleton, SkeletonDomainCard } from '../components/Skeleton';
import { useReport } from '../context/useReport';
import { format, strings } from '../lib/strings';
import { extractHostname } from '@/lib/domainSlug';
import { apiUrl, reportApi } from '../lib/publicBase';
-import type { PortfolioGroup, ReportCategory, ViewProps } from '@/types';
-
-function scoreFromCategories(categories: ReportCategory[] = []): number | null {
- const numeric = (categories || [])
- .map((c) => Number(c?.score))
- .filter((n) => Number.isFinite(n));
- if (!numeric.length) return null;
- const avg = numeric.reduce((a, b) => a + b, 0) / numeric.length;
- return Math.round(avg);
-}
-
-function toLocalDateTime(value: string | null | undefined): string {
- if (!value) return '';
- const d = new Date(value);
- if (Number.isNaN(d.getTime())) return '';
- return d.toLocaleString();
-}
-
-function healthScoreClass(score: number): string {
- if (score >= 80) return 'text-emerald-700 dark:text-emerald-400';
- if (score >= 60) return 'text-amber-700 dark:text-amber-400';
- return 'text-rose-700 dark:text-rose-400';
-}
+import {
+ parsePortfolioAuditHistory,
+ type PortfolioAuditHistoryPoint,
+} from '@/lib/portfolioAuditHistory';
+import type { PortfolioCrawlHistoryPoint } from '@/types/api';
+import type { PortfolioGroup, ViewProps } from '@/types';
function portfolioRootDomain(group: PortfolioGroup): string {
const host = extractHostname(group.crawlUrl) || group.domainName.trim().toLowerCase();
@@ -51,7 +35,12 @@ export default function Home({ onNavigate }: ViewProps) {
const [pendingDeleteKey, setPendingDeleteKey] = useState {vh.subtitle} {vh.totalBrandsLabel} {portfolioTotals.totalBrands.toLocaleString()} {vh.totalUrlsLabel} {portfolioTotals.totalUrls.toLocaleString()} {vh.subtitle} {vh.avgHealthLabel}
- {portfolioTotals.avgHealth ?? sj.emDash}
- {vh.totalBrandsLabel} {portfolioTotals.totalBrands.toLocaleString()} {vh.totalUrlsLabel} {portfolioTotals.totalUrls.toLocaleString()} {vh.avgHealthLabel}
+ {portfolioTotals.avgHealth ?? sj.emDash}
+
- {vh.deleteConfirmTitle}
-
- {group.crawlOnly
- ? format(vh.deleteConfirmCrawlOnly, {
- name: group.domainName,
- count: group.urlCount.toLocaleString(),
- })
- : format(vh.deleteConfirmBody, { name: group.domainName })}
- {vh.crawlUrlLabel} {vh.urlCountLabel} {group.urlCount.toLocaleString()} {vh.lastCrawlLabel} {group.lastCrawl || sj.emDash} {vh.statusBreakdownLabel}
- {iss.llm_recommendation || iss.recommendation || emDash}
-
- {vi.ruleRecommendation}:
- {iss.recommendation}
-
+ {iss.llm_recommendation || iss.recommendation || emDash}
+
+ {vi.ruleRecommendation}:
+ {iss.recommendation}
+ {vi.noMatches} {vj.emptyFiltered} {vlh.multiCompareHint} {f.message || strings.common.emDash} {f.message || strings.common.emDash} {description} {vtca.noKeywordData}
+ {keywordsChartPagination.total.toLocaleString()} terms
+ {vtca.noKeywordData} {sj.noData}
+
@@ -228,52 +218,52 @@ export default function Home({ onNavigate }: ViewProps) {
-
-
- {vh.title}
-
-
-
-
-
) : (
diff --git a/web/src/views/Issues.tsx b/web/src/views/Issues.tsx
index d2510194..f5f41835 100644
--- a/web/src/views/Issues.tsx
+++ b/web/src/views/Issues.tsx
@@ -1,11 +1,12 @@
-import { useState, useMemo } from 'react';
+import { useState, useMemo, useEffect } from 'react';
import { Bar, Doughnut } from 'react-chartjs-2';
import type { TooltipItem } from 'chart.js';
-import { AlertTriangle, AlertCircle, Info, ChevronDown, ChevronRight, ExternalLink, Flame, BarChart2, ListChecks } from 'lucide-react';
+import { AlertTriangle, AlertCircle, Info, ExternalLink, Flame, BarChart2, ListChecks } from 'lucide-react';
import { useReport } from '../context/useReport';
import { useOptionalPipeline } from '../context/PipelineContext';
import { strings, format } from '../lib/strings';
-import { PageLayout, PageHeader, Card, Badge, ViewTabs } from '../components';
+import { PageLayout, PageHeader, Card, Badge, ViewTabs, ViewTabPanel, Button } from '../components';
+import { paginateSlice, PAGE_SIZE } from '@/components/google/tableUtils';
import UrlInspectorButton from '@/components/UrlInspectorButton';
import IssueTaskBoard from '@/components/issues/IssueTaskBoard';
import IssueAiFixButton from '@/components/issues/IssueAiFixButton';
@@ -38,85 +39,56 @@ interface CategoryIssueItem {
issue: ReportIssue;
}
-interface CategorySectionProps {
- category: string;
- items: CategoryIssueItem[];
- defaultOpen?: boolean;
+interface IssueCardProps {
+ item: CategoryIssueItem;
vi: (typeof strings.views)['issues'];
emDash: string;
}
-function CategorySection({ category, items, defaultOpen = false, vi, emDash }: CategorySectionProps) {
- const [open, setOpen] = useState(defaultOpen);
+function IssueCard({ item, vi, emDash }: IssueCardProps) {
+ const iss = item.issue;
+ const p = normalizePriority(iss.priority);
+ const cfg = PRIORITY_CONFIG[p];
+ const Icon = PRIORITY_ICONS[p];
return (
-
-
{deleteError ? (
@@ -299,201 +289,59 @@ export default function Home({ onNavigate }: ViewProps) {
{groupedPortfolio.map(({ rootDomain, items }) => {
const collapsed = collapsedGroups.has(rootDomain);
return (
-
+
-
-
{vh.title}
+
+
-
-
-
+
+
+
+
+
+
- {items.map((group) => {
- const cardKey = portfolioCardKey(group);
- const confirmOpen = pendingDeleteKey === cardKey;
- const isDeleting = deletingKey === cardKey;
- return (
-
+ ) : null}
+
-
- ) : null}
-
- );
+ {items.map((group) => {
+ const cardKey = portfolioCardKey(group);
+ return (
+
-
- );
- })}
-
-
-
-
-
- {confirmOpen ? (
-
-
- ) : null}
-
-
-
-
-
-
-
-
-
-
-
-
+ {!collapsed ? (
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 2xx {group.statusCounts.s2xx}
-
-
- 3xx {group.statusCounts.s3xx}
-
-
- 4xx {group.statusCounts.s4xx}
-
-
- 5xx {group.statusCounts.s5xx}
-
- {group.statusCounts.other > 0 && (
-
- {format(vh.otherStatusPill, { count: group.statusCounts.other })}
-
- )}
-
-
-
-
- {open && (
-
- {items.map((item, i) => {
- const iss = item.issue;
- const p = normalizePriority(iss.priority);
- const cfg = PRIORITY_CONFIG[p];
- const Icon = PRIORITY_ICONS[p];
- return (
-
)}
@@ -434,17 +418,56 @@ export default function Issues({ searchQuery = '' }: ViewProps) {
-
- );
- })}
+
-
-
-
- {iss.message || emDash}
- {iss.url && (
-
-
- {iss.url}
-
- )}
-
-
- {vi.fixRecommendation}
-
+
);
}
@@ -126,11 +98,13 @@ export default function Issues({ searchQuery = '' }: ViewProps) {
const pipeline = useOptionalPipeline();
const propertyId = Number(pipeline?.configState.active_property_id || 0) || null;
const vi = strings.views.issues;
+ const vlp = vi.pagination;
const sj = strings.common;
const priorityOrder = PRIORITY_ORDER;
const [issuesTab, setIssuesTab] = useState<'audit' | 'board'>('audit');
const [priorityFilter, setPriorityFilter] = useState(sj.all);
- const [categoryFilter, setCategoryFilter] = useState(sj.all);
+ const [activeCategory, setActiveCategory] = useState
+
+
+
- )}
+ {iss.message || emDash}
+ {iss.url && (
+
+
+ {iss.url}
+
+ )}
+
+
{vi.fixRecommendation}
+
- {Object.entries(grouped).map(([cat, items], idx) => (
-
))}
diff --git a/web/src/views/JavaScriptErrors.tsx b/web/src/views/JavaScriptErrors.tsx
index 0f3c6be9..dbb855e8 100644
--- a/web/src/views/JavaScriptErrors.tsx
+++ b/web/src/views/JavaScriptErrors.tsx
@@ -1,6 +1,6 @@
'use client';
-import { useMemo, useState, Fragment } from 'react';
+import { useMemo, useState, useEffect, Fragment } from 'react';
import Link from 'next/link';
import { useSearchParams } from 'next/navigation';
import { useUrlTab } from '@/hooks/useUrlTab';
@@ -8,6 +8,7 @@ import { Bug, ChevronDown, ChevronRight, ExternalLink, BarChart3, List } from 'l
import { useReport } from '../context/useReport';
import { strings, format } from '../lib/strings';
import { PageLayout, PageHeader, Card, Button, StatCard, Select, Table, TableHead, TableHeadCell, TableBody, TableRow, TableCell, ViewTabs, ViewTabPanel } from '../components';
+import { paginateSlice, PAGE_SIZE } from '@/components/google/tableUtils';
import type { ViewTabItem } from '../components';
import type { ViewProps } from '@/types';
import {
@@ -32,9 +33,11 @@ export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) {
const trailingQuery = searchParams.toString() ? `?${searchParams.toString()}` : '';
const [typeFilter, setTypeFilter] = useState
+
+ ) : null}
+
+ {totalPages > 1 ? (
+ {format(vlp.showingSlice, { from, to, total: activeTotal })}
+
+ {vlp.pageOf}{' '}
+ {safePage} {vlp.of}{' '}
+ {totalPages}
+
+ ({format(vlp.rowsPerPage, { n: PAGE_SIZE })})
+
+
+
+
+
+
+ ) : null}
+
-
+ {filteredRowsTotal > 0 ? (
+
-
+
-
-
+ {row.message}
+
+
+
+ ) : null}
+
+
+
+ {row.stack}
+
+
+
+ ) : null}
+
)}
diff --git a/web/src/views/Lighthouse.tsx b/web/src/views/Lighthouse.tsx
index 9fb1cd32..4e715060 100644
--- a/web/src/views/Lighthouse.tsx
+++ b/web/src/views/Lighthouse.tsx
@@ -1,4 +1,4 @@
-import { useState, useMemo, useRef } from 'react';
+import { useState, useMemo, useRef, useEffect } from 'react';
import type {
LighthouseDiagnostic,
LighthouseFailure,
@@ -18,7 +18,8 @@ import {
} from '../lib/domainSlug';
import { goToPipeline } from '../lib/pipelineReturn';
import { strings, format } from '../lib/strings';
-import { PageLayout, PageHeader, Card, Button, ViewTabs, Select } from '../components';
+import { PageLayout, PageHeader, Card, Button, ViewTabs, ViewTabPanel, Select } from '../components';
+import { paginateSlice, PAGE_SIZE } from '@/components/google/tableUtils';
import type { ViewTabItem } from '../components';
import {
CATEGORIES, CATEGORY_LABELS, METRIC_THRESHOLDS, IMPACT_GROUPS, QUICK_WINS,
@@ -26,7 +27,7 @@ import {
import {
ScoreRing,
ThresholdBar,
- DiagnosticGroup,
+ DiagnosticItem,
QuickWinCard,
MultiPageTable,
LhAuditExpandable,
@@ -43,7 +44,10 @@ export default function Lighthouse({ searchQuery = '' }: ViewProps) {
const { data, startUrlByRunId } = useReport();
const searchParams = useSearchParams();
const detailRef = useRef
+
+ {errorsTotalPages > 1 ? (
+ {format(vjp.showingSlice, { from: errorsFrom, to: errorsTo, total: filteredRowsTotal })}
+
+ {vjp.pageOf}{' '}
+ {safeErrorsPage} {vjp.of}{' '}
+ {errorsTotalPages}
+
+ ({format(vjp.rowsPerPage, { n: PAGE_SIZE })})
+
+
+
+
+
+
+ ) : null}
+
)}
@@ -449,19 +521,56 @@ export default function Lighthouse({ searchQuery = '' }: ViewProps) {
) : diagnosticsForGroups.length === 0 ? (
+
+ ) : null}
+
+ {(selectedPageSummary.human_summary_full || selectedPageSummary.human_summary) ? (
+
+ {vlh.categoriesSection}
+
+
+ {CATEGORIES.map(({ id, label }) => {
+ const pageCs = selectedPageSummary.category_scores || {};
+ const score = pageCs[id] != null ? Number(pageCs[id]) : null;
+ return
+ {vlh.summary}
+
+ {selectedPageSummary.human_summary_full || selectedPageSummary.human_summary}
+
+
- {IMPACT_GROUPS.map((group) => {
- const items = groupedDiagnostics[group.id] || [];
- if (items.length === 0) return null;
- return (
-
diff --git a/web/src/views/Links.tsx b/web/src/views/Links.tsx
index ec403441..fbca9033 100644
--- a/web/src/views/Links.tsx
+++ b/web/src/views/Links.tsx
@@ -38,6 +38,7 @@ import { exportLinksCsv } from '@/utils/linkExport';
import { useOptionalPipeline } from '../context/PipelineContext';
import AiSuggestionButton from '@/components/ai/AiSuggestionButton';
import { buildTechnicalLinkIssueContext } from '@/lib/fixSuggestionContext';
+import { crawledUrlCount } from '@/lib/crawlCounts';
const EXPLORER_TABS = ['urls', 'anchors'] as const;
type ExplorerTabId = (typeof EXPLORER_TABS)[number];
@@ -97,6 +98,7 @@ export default function Links({ searchQuery = '' }: ViewProps) {
const tableRef = useRef
+ {impactGroupTabs.length > 1 ? (
+
)}
+
+ ) : null}
+
+ {diagnosticTotalPages > 1 ? (
+ {format(vlp.showingSlice, { from: diagnosticFrom, to: diagnosticTo, total: activeDiagnosticTotal })}
+
+ {vlp.pageOf}{' '}
+ {safeDiagnosticPage} {vlp.of}{' '}
+ {diagnosticTotalPages}
+
+ ({format(vlp.rowsPerPage, { n: PAGE_SIZE })})
+
+
+
+
+
+
+ ) : null}
+ ,
- badge: links.length > 0 ? links.length : null,
+ badge: crawledCount > 0 ? crawledCount : null,
},
];
if (hasLinkAttributes) {
@@ -145,7 +147,7 @@ export default function Links({ searchQuery = '' }: ViewProps) {
});
}
return items;
- }, [vl.tabs, links.length, hasLinkAttributes, data?.inlink_anchor_matrix?.length]);
+ }, [vl.tabs, crawledCount, hasLinkAttributes, data?.inlink_anchor_matrix?.length]);
const inspectParam = searchParams.get('inspect');
const tabParam = searchParams.get('tab');
diff --git a/web/src/views/Security.tsx b/web/src/views/Security.tsx
index 084ffe05..35984ee9 100644
--- a/web/src/views/Security.tsx
+++ b/web/src/views/Security.tsx
@@ -1,11 +1,12 @@
-import { useState, useMemo } from 'react';
+import { useState, useMemo, useEffect } from 'react';
import { useUrlTab } from '@/hooks/useUrlTab';
import { Bar, Doughnut } from 'react-chartjs-2';
import type { TooltipItem } from 'chart.js';
import { Shield, Flame, AlertTriangle, AlertCircle, Info, ExternalLink, BarChart3, List } from 'lucide-react';
import { useReport } from '../context/useReport';
import { strings, format } from '../lib/strings';
-import { PageLayout, PageHeader, Card, Badge, ViewTabs, ViewTabPanel } from '../components';
+import { PageLayout, PageHeader, Card, Badge, ViewTabs, ViewTabPanel, Button } from '../components';
+import { paginateSlice, PAGE_SIZE } from '@/components/google/tableUtils';
import type { ViewTabItem } from '../components';
import { palette } from '../utils/chartPalette';
import { registerChartJsBase, barOptionsHorizontal } from '../utils/chartJsDefaults';
@@ -97,6 +98,7 @@ export default function Security({ searchQuery = '' }: ViewProps) {
const { data } = useReport();
const [severityFilter, setSeverityFilter] = useState('All');
const [activeTab, setActiveTab] = useUrlTab(SECURITY_TABS, 'findings');
+ const [findingsPage, setFindingsPage] = useState(1);
const q = (searchQuery || '').toLowerCase().trim();
@@ -154,6 +156,44 @@ export default function Security({ searchQuery = '' }: ViewProps) {
}, []);
const vs = strings.views.security;
+ const vsp = vs.pagination;
+
+ const filteredFindings = useMemo(() => {
+ let list: SecurityFinding[] = allFindings;
+ if (severityFilter !== 'All') {
+ list = list.filter((f) => (f.severity || 'Info') === severityFilter);
+ }
+ if (q) {
+ list = list.filter((f) => {
+ const url = (f.url || '').toLowerCase();
+ const msg = (f.message || '').toLowerCase();
+ const rec = (f.recommendation || '').toLowerCase();
+ const typ = securityFindingLabel(f.finding_type).toLowerCase();
+ return url.includes(q) || msg.includes(q) || rec.includes(q) || typ.includes(q);
+ });
+ }
+ return [...list].sort((a, b) => {
+ const ao = (SEVERITY_CONFIG[(a.severity || 'Info') as SeverityKey] ?? SEVERITY_CONFIG.Info).order;
+ const bo = (SEVERITY_CONFIG[(b.severity || 'Info') as SeverityKey] ?? SEVERITY_CONFIG.Info).order;
+ return ao - bo;
+ });
+ }, [allFindings, severityFilter, q]);
+
+ const {
+ slice: visibleFindings,
+ page: safeFindingsPage,
+ totalPages: findingsTotalPages,
+ total: filteredFindingsTotal,
+ from: findingsFrom,
+ to: findingsTo,
+ } = useMemo(
+ () => paginateSlice(filteredFindings, findingsPage, PAGE_SIZE),
+ [filteredFindings, findingsPage],
+ );
+
+ useEffect(() => {
+ setFindingsPage(1);
+ }, [severityFilter, q]);
const tabItems = useMemo((): ViewTabItem[] => {
const chartCount = allFindings.length > 0 ? (typeLabels.length > 0 ? 2 : 1) : 0;
@@ -180,26 +220,6 @@ export default function Security({ searchQuery = '' }: ViewProps) {
return acc;
}, {});
- let findings: SecurityFinding[] = allFindings;
- if (severityFilter !== 'All') {
- findings = findings.filter((f) => (f.severity || 'Info') === severityFilter);
- }
- if (q) {
- findings = findings.filter((f) => {
- const url = (f.url || '').toLowerCase();
- const msg = (f.message || '').toLowerCase();
- const rec = (f.recommendation || '').toLowerCase();
- const typ = securityFindingLabel(f.finding_type).toLowerCase();
- return url.includes(q) || msg.includes(q) || rec.includes(q) || typ.includes(q);
- });
- }
-
- findings = [...findings].sort((a, b) => {
- const ao = (SEVERITY_CONFIG[(a.severity || 'Info') as SeverityKey] ?? SEVERITY_CONFIG.Info).order;
- const bo = (SEVERITY_CONFIG[(b.severity || 'Info') as SeverityKey] ?? SEVERITY_CONFIG.Info).order;
- return ao - bo;
- });
-
return (
@@ -318,49 +338,86 @@ export default function Security({ searchQuery = '' }: ViewProps) {
- {findings.map((f, i) => {
- const sev = (f.severity || 'Info') as SeverityKey;
- const cfg = SEVERITY_CONFIG[sev] ?? SEVERITY_CONFIG.Info;
- const Icon = cfg.icon;
- return (
-
-
-
-
+
)}
diff --git a/web/src/views/TextContentAnalysis.tsx b/web/src/views/TextContentAnalysis.tsx
new file mode 100644
index 00000000..5d231508
--- /dev/null
+++ b/web/src/views/TextContentAnalysis.tsx
@@ -0,0 +1,757 @@
+'use client';
+
+import type { Chart, TooltipItem } from 'chart.js';
+import { Fragment, useState, useMemo, useEffect, type ComponentType, type ReactNode } from 'react';
+import { useUrlTab } from '@/hooks/useUrlTab';
+import type {
+ ContentAnalyticsData,
+ TextContentAnalysisData,
+ TextContentKeywordEntry,
+ TopicCluster,
+ ViewProps,
+} from '@/types';
+import { filterTopicClusters } from '@/lib/semanticTextHygiene';
+import { buildByPageTextRows } from '@/lib/textContentAnalysis';
+import { anyChartOptions } from '../utils/chartOptions';
+import { Chart as ChartJS, CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend } from 'chart.js';
+import { Bar } from 'react-chartjs-2';
+import {
+ BookOpen,
+ FileText,
+ BarChart2,
+ Tag,
+ Layers,
+ Sparkles,
+ ChevronDown,
+ ChevronRight,
+ LayoutDashboard,
+ Key,
+ AlignLeft,
+ Globe,
+} from 'lucide-react';
+import { useReport } from '../context/useReport';
+import { strings, format } from '../lib/strings';
+import {
+ PageLayout,
+ PageHeader,
+ Card,
+ Table,
+ TableHead,
+ TableHeadCell,
+ TableBody,
+ TableRow,
+ TableCell,
+ ViewTabs,
+ ViewTabPanel,
+ Button,
+} from '../components';
+import type { ViewTabItem } from '../components';
+import SortablePaginatedTable from '../components/google/SortablePaginatedTable';
+import { PAGE_SIZE, paginateSlice } from '../components/google/tableUtils';
+import { palette, PALETTE_CATEGORICAL } from '../utils/chartPalette';
+import {
+ getGridColor,
+ getChartTitleColor,
+ getChartCanvasTextColor,
+} from '../utils/chartJsDefaults';
+
+ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend);
+
+const TEXT_TABS = ['overview', 'keywords', 'analytics', 'topics'] as const;
+type TextTabId = (typeof TEXT_TABS)[number];
+
+const EMPTY_CA: ContentAnalyticsData = {};
+const EMPTY_TCA: TextContentAnalysisData = {};
+
+const barValueLabelsPlugin = {
+ id: 'tcaBarLabels',
+ afterDatasetsDraw(chart: Chart) {
+ const ctx = chart.ctx;
+ const isHorizontal = chart.options.indexAxis === 'y';
+ const pad = 6;
+ ctx.save();
+ ctx.font = '11px system-ui, sans-serif';
+ ctx.textBaseline = 'middle';
+ (chart.data.datasets || []).forEach((dataset, dsi: number) => {
+ const meta = chart.getDatasetMeta(dsi);
+ if (!meta?.data?.length || !dataset?.data) return;
+ meta.data.forEach((bar, i: number) => {
+ const value = dataset.data[i];
+ if (value == null || value === 0) return;
+ const label = Number(value).toLocaleString();
+ if (isHorizontal) {
+ const textWidth = ctx.measureText(label).width;
+ const fitsOutside = bar.x + pad + textWidth <= chart.chartArea.right;
+ if (fitsOutside) {
+ ctx.textAlign = 'left';
+ ctx.fillStyle = getChartCanvasTextColor();
+ ctx.fillText(label, bar.x + pad, bar.y);
+ } else {
+ ctx.textAlign = 'right';
+ ctx.fillStyle = '#ffffff';
+ ctx.fillText(label, bar.x - pad, bar.y);
+ }
+ } else {
+ ctx.textAlign = 'center';
+ ctx.fillStyle = getChartCanvasTextColor();
+ ctx.fillText(label, bar.x, bar.y - 12);
+ }
+ });
+ });
+ ctx.restore();
+ },
+};
+
+function barOpts(xTitle?: string) {
+ const pagesWord = strings.common.pages;
+ return anyChartOptions({
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: { legend: { display: false } },
+ scales: {
+ x: { grid: { color: getGridColor() }, ...(xTitle ? { title: { display: true, text: xTitle } } : {}) },
+ y: { grid: { color: getGridColor() }, beginAtZero: true, title: { display: true, text: pagesWord } },
+ },
+ });
+}
+
+function barOptsH(xTitle?: string) {
+ const freq = strings.charts.axisFrequency;
+ return anyChartOptions({
+ indexAxis: 'y',
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: { legend: { display: false } },
+ scales: {
+ x: {
+ grid: { color: getGridColor() },
+ beginAtZero: true,
+ grace: '10%',
+ title: { display: true, text: xTitle ?? freq },
+ },
+ y: { grid: { color: getGridColor() } },
+ },
+ });
+}
+
+function SectionHeader({
+ icon,
+ title,
+ description,
+}: {
+ icon: ComponentType<{ className?: string }>;
+ title: ReactNode;
+ description?: ReactNode;
+}) {
+ const Icon = icon;
+ return (
+
+ {visibleFindings.map((f, i) => {
+ const sev = (f.severity || 'Info') as SeverityKey;
+ const cfg = SEVERITY_CONFIG[sev] ?? SEVERITY_CONFIG.Info;
+ const Icon = cfg.icon;
+ return (
+
+ {filteredFindingsTotal > 0 ? (
+
+
-
- {securityFindingLabel(f.finding_type)}
-
- {f.url && (
-
- {f.url}
-
+ );
+ })}
+
+
+ {vs.recommendation}
+
+ {f.recommendation}
+
)}
+
+
+ ) : null}
+
- );
- })}
+ {findingsTotalPages > 1 ? (
+ {format(vsp.showingSlice, { from: findingsFrom, to: findingsTo, total: filteredFindingsTotal })}
+
+ {vsp.pageOf}{' '}
+ {safeFindingsPage} {vsp.of}{' '}
+ {findingsTotalPages}
+
+ ({format(vsp.rowsPerPage, { n: PAGE_SIZE })})
+
-
-
- {vs.recommendation}
-
- {f.recommendation}
-
- )}
-
+
+
+
+ ) : null}
+
+
+ );
+}
+
+function KeywordIndexTable({
+ rows,
+ vtca,
+ sj,
+}: {
+ rows: TextContentKeywordEntry[];
+ vtca: (typeof strings.views)['textContentAnalysis'];
+ sj: typeof strings.common;
+}) {
+ const [expanded, setExpanded] = useState
+
+
+
+ {title}
+ {description ?
+
+ );
+}
+
+export default function TextContentAnalysis({ searchQuery = '' }: ViewProps) {
+ const vtca = strings.views.textContentAnalysis;
+ const sj = strings.common;
+ const ch = strings.charts;
+ const { data } = useReport();
+ const [activeTab, setActiveTab] = useUrlTab(TEXT_TABS, 'overview');
+ const [keywordsChartPage, setKeywordsChartPage] = useState(1);
+
+ const tca: TextContentAnalysisData = data?.text_content_analysis ?? EMPTY_TCA;
+ const ca: ContentAnalyticsData = data?.content_analytics ?? EMPTY_CA;
+ const vocab = tca.vocabulary_stats ?? {};
+ const wcStats = ca.word_count_stats ?? {};
+ const keywordIndex = tca.keyword_index ?? [];
+
+ const keywordIndexFiltered = useMemo(() => {
+ const q = (searchQuery || '').trim().toLowerCase();
+ if (!q) return keywordIndex;
+ return keywordIndex.filter(
+ (k) =>
+ k.word.toLowerCase().includes(q) ||
+ (k.top_pages ?? []).some((p) => p.url.toLowerCase().includes(q)),
+ );
+ }, [keywordIndex, searchQuery]);
+
+ const keywordsChartPagination = useMemo(
+ () => paginateSlice(keywordIndexFiltered, keywordsChartPage, PAGE_SIZE),
+ [keywordIndexFiltered, keywordsChartPage],
+ );
+
+ useEffect(() => {
+ setKeywordsChartPage(1);
+ }, [keywordIndexFiltered]);
+
+ useEffect(() => {
+ setKeywordsChartPage((p) => Math.min(Math.max(1, p), keywordsChartPagination.totalPages));
+ }, [keywordsChartPagination.totalPages]);
+
+ const keywordsChart = useMemo(() => {
+ const slice = keywordsChartPagination.slice;
+ if (slice.length === 0) return null;
+ return {
+ labels: slice.map((k) => k.word),
+ values: slice.map((k) => k.total_count),
+ };
+ }, [keywordsChartPagination.slice]);
+
+ const keywordsChartHeightPx = keywordsChart
+ ? Math.max(320, keywordsChart.labels.length * 28)
+ : 320;
+
+ const histChart = useMemo(() => {
+ const hist = tca.keyword_frequency_histogram;
+ if (!hist) return null;
+ const labels = [vtca.histBucket1, vtca.histBucket2, vtca.histBucket6, vtca.histBucket21];
+ const keys = ['1', '2-5', '6-20', '21+'];
+ const values = keys.map((k) => Number(hist[k]) || 0);
+ if (values.every((v) => v === 0)) return null;
+ return { labels, values };
+ }, [tca.keyword_frequency_histogram, vtca]);
+
+ const byPageRows = useMemo(
+ () => buildByPageTextRows(data?.links, searchQuery),
+ [data?.links, searchQuery],
+ );
+
+ const languageMlChart = useMemo(() => {
+ const c = data?.language_summary?.counts || {};
+ const entries = Object.entries(c)
+ .sort((a, b) => Number(b[1]) - Number(a[1]))
+ .slice(0, 15);
+ if (entries.length === 0) return null;
+ return { labels: entries.map((x) => x[0]), values: entries.map((x) => Number(x[1])) };
+ }, [data?.language_summary?.counts]);
+
+ const nerSiteChart = useMemo(() => {
+ const lc = data?.ner_site_summary?.label_counts;
+ if (!lc || typeof lc !== 'object') return null;
+ const entries = Object.entries(lc)
+ .sort((a, b) => Number(b[1]) - Number(a[1]))
+ .slice(0, 15);
+ if (entries.length === 0) return null;
+ return { labels: entries.map((x) => x[0]), values: entries.map((x) => Number(x[1])) };
+ }, [data?.ner_site_summary?.label_counts]);
+
+ const tokenClusters = useMemo(
+ () => filterTopicClusters(data?.keyword_opportunities?.token_topic_clusters ?? []),
+ [data?.keyword_opportunities?.token_topic_clusters],
+ );
+
+ const semanticClusters = useMemo(
+ () => filterTopicClusters(data?.semantic_keyword_clusters ?? []),
+ [data?.semantic_keyword_clusters],
+ );
+
+ const wcDist = ca.word_count_distribution ?? {};
+ const rlDist = ca.reading_level_distribution ?? {};
+ const crDist = ca.content_ratio_distribution ?? {};
+ const wcLabels = Object.keys(wcDist);
+ const wcValues = Object.values(wcDist).map(Number);
+ const rlLabels = Object.keys(rlDist);
+ const rlValues = Object.values(rlDist).map(Number);
+ const crLabels = Object.keys(crDist);
+ const crValues = Object.values(crDist).map(Number);
+
+ const wcPercLabels = vtca.wcPercLabels;
+ const wcPercRaw = [wcStats.min, wcStats.p25, wcStats.median, wcStats.mean, wcStats.p75, wcStats.max];
+ const wcPercValues = wcPercRaw.map((v) => (v != null && !Number.isNaN(Number(v)) ? Number(v) : null));
+ const hasWcPercBar = wcPercValues.every((v) => v != null) && (wcStats.max ?? 0) > 0;
+
+ const tabItems = useMemo((): ViewTabItem[] => [
+ { id: 'overview', label: vtca.tabs.overview, icon:
+
+
+
+
+
+
+ {vtca.thTopPages}
+
+
+ {row.top_pages!.map((p) => (
+
+
+
+
+ {vtca.uniqueTerms}
+ {vocab.unique_terms ?? sj.emDash}
+ {vtca.pagesWithKeywords}
+ {vocab.pages_with_keywords ?? sj.emDash}
+
+
+
+ {wcStats.mean != null ? Math.round(wcStats.mean).toLocaleString() : sj.emDash}
+
+ {vtca.perPage}
+
+
+
+ {wcStats.median != null ? Math.round(wcStats.median).toLocaleString() : sj.emDash}
+
+ {vtca.perPage}
+
+
+
+ {vtca.avgTermsPerPage}
+ {vocab.avg_terms_per_page ?? sj.emDash}
+ {vtca.totalOccurrences}
+
+ {vocab.total_term_occurrences != null ? vocab.total_term_occurrences.toLocaleString() : sj.emDash}
+
+
+
+
+
+ {vtca.keywordFrequencyHist}
+
+
+
+
+
+
+ {vtca.topKeywordsChart}
+
+
+ {keywordsChartPagination.total > 0 ? (
+
+
+ ) : null}
+
+
+ {keywordsChartPagination.totalPages > 1 ? (
+
+ {format(vtca.pagination.showingSlice, {
+ from: keywordsChartPagination.from,
+ to: keywordsChartPagination.to,
+ total: keywordsChartPagination.total,
+ })}
+
+
+ {vtca.pagination.pageOf}{' '}
+ {keywordsChartPagination.page}{' '}
+ {vtca.pagination.of}{' '}
+ {keywordsChartPagination.totalPages}
+
+ ({format(vtca.pagination.rowsPerPage, { n: PAGE_SIZE })})
+
+
+
+
+
+
+ ) : null}
+
+
+ {vtca.wordCountDist}
+
+ {wcLabels.length > 0 ? (
+
+ {sj.noData}
+ )}
+ {vtca.readingLevelDist}
+
+ {rlLabels.length > 0 ? (
+
+ {sj.noData}
+ )}
+ {vtca.contentHtmlRatio}
+
+ {crLabels.length > 0 ? (
+
+ {sj.noData}
+ )}
+ {vtca.wordCountLadder}
+
+
+
+
+ {vtca.languageMix}
+
+
+
+
+ {vtca.entityLabels}
+
+
+
+
+ {vtca.parentTopicsToken}
+
+
+
+
+
+
+ {vtca.parentTopicsSemantic}
+
+
+
+
+