diff --git a/apps/web/lib/extract-urls.test.ts b/apps/web/lib/extract-urls.test.ts new file mode 100644 index 000000000..10291f739 --- /dev/null +++ b/apps/web/lib/extract-urls.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "bun:test" +import { extractUrls } from "./url-helpers" + +describe("extractUrls", () => { + it("extracts bare, markdown, and angle-bracket links", () => { + const { urls } = extractUrls( + "see https://a.example/one, [docs](https://b.example/two) and ", + ) + expect(urls).toEqual([ + "https://a.example/one", + "https://b.example/two", + "https://c.example/three", + ]) + }) + + it("normalizes scheme-less URLs", () => { + const { urls } = extractUrls("check supermemory.ai for details") + expect(urls).toEqual(["https://supermemory.ai"]) + }) + + it("does not extract URLs from email addresses", () => { + const result = extractUrls("email me at john.doe@example.com") + expect(result.urls).toEqual([]) + expect(result.duplicates).toBe(0) + }) + + it("keeps real URLs while skipping emails in the same text", () => { + const { urls } = extractUrls( + "email john.doe@example.com or visit https://supermemory.ai", + ) + expect(urls).toEqual(["https://supermemory.ai"]) + }) + + it("skips multiple email addresses", () => { + const { urls } = extractUrls( + "contacts: a.person@foo.example, b.person@bar.example", + ) + expect(urls).toEqual([]) + }) + + it("strips trailing punctuation", () => { + const { urls } = extractUrls("read https://example.com/post.") + expect(urls).toEqual(["https://example.com/post"]) + }) + + it("dedupes URLs that differ only by scheme/host case or trailing slash", () => { + const { urls, duplicates } = extractUrls( + "HTTPS://EXAMPLE.COM/docs https://example.com/docs https://example.com/docs/", + ) + expect(urls).toHaveLength(1) + expect(duplicates).toBe(2) + }) + + it("keeps URLs whose paths differ only by case", () => { + const { urls, duplicates } = extractUrls( + "https://example.com/Page and https://example.com/page", + ) + expect(urls).toEqual([ + "https://example.com/Page", + "https://example.com/page", + ]) + expect(duplicates).toBe(0) + }) + + it("returns nothing for plain text", () => { + expect(extractUrls("no links here").urls).toEqual([]) + expect(extractUrls("").urls).toEqual([]) + }) +}) diff --git a/apps/web/lib/url-helpers.ts b/apps/web/lib/url-helpers.ts index ea5bdec21..dd5269a43 100644 --- a/apps/web/lib/url-helpers.ts +++ b/apps/web/lib/url-helpers.ts @@ -96,12 +96,20 @@ export const extractUrls = ( const unwrapped = text .replace(MARKDOWN_LINK_REGEX, " $1 ") .replace(ANGLE_LINK_REGEX, " $1 ") - const matches = unwrapped.match(URL_TOKEN_REGEX) ?? [] const seen = new Set() const urls: string[] = [] let duplicates = 0 - for (const match of matches) { - let trimmed = match.trim().replace(/[.,;!]+$/, "") + for (const match of unwrapped.matchAll(URL_TOKEN_REGEX)) { + const start = match.index ?? 0 + const end = start + match[0].length + const before = start > 0 ? (unwrapped[start - 1] ?? "") : "" + const after = end < unwrapped.length ? (unwrapped[end] ?? "") : "" + // Skip email addresses: a domain-shaped token ending at "@" is the + // local part, one starting right after "@" is the mail domain. Also + // skip matches that begin mid-token (e.g. after "_", which the + // hostname charset can't include) — those aren't standalone URLs. + if (after === "@" || before === "@" || /[\w.-]/.test(before)) continue + let trimmed = match[0].trim().replace(/[.,;!]+$/, "") const opens = (trimmed.match(/\(/g) ?? []).length const closes = (trimmed.match(/\)/g) ?? []).length if (closes > opens && trimmed.endsWith(")")) { @@ -109,7 +117,15 @@ export const extractUrls = ( } const normalized = normalizeUrl(trimmed) if (!isValidUrl(normalized)) continue - const key = normalized.toLowerCase().replace(/\/+$/, "") + // Dedupe on the parsed URL so the scheme and host compare + // case-insensitively while the path/query — which are case-sensitive + // resources — stay distinct. + const parsed = new URL(normalized) + const key = + `${parsed.origin}${parsed.pathname}${parsed.search}${parsed.hash}`.replace( + /\/+$/, + "", + ) if (seen.has(key)) { duplicates++ continue