From e42ae9d87a3207c1f139171f6e66955370f2b2cb Mon Sep 17 00:00:00 2001 From: Rohit Ghumare <48523873+rohitg00@users.noreply.github.com> Date: Tue, 28 Apr 2026 11:16:20 +0100 Subject: [PATCH] chore: edge cache headers, agent-aware robots, UA bot-block middleware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cuts Vercel hobby edge-request burn on both skillkit (Vite static) and skillkit-docs (Next.js fumadocs) projects. Site stays human-first AND agent-first — Firecrawl, Context7, Crawl4AI, OpenClaw, Hermes, ChatGPT-User, Claude-User, PerplexityBot explicitly allowed; training crawlers and SEO scrapers blocked. skillkit (Vite, docs/skillkit): - vercel.json: Cache-Control on assets (1d browser, 7d edge, 30d SWR; immutable for hashed /assets/*), HTML (5min browser, 1d edge, 7d SWR), and /api JSON - redirects with has user-agent: deflect known training+SEO bots to /robots.txt; negative lookahead on source prevents /robots.txt redirect loop - public/robots.txt: explicit allow + deny lists skillkit-docs (Next.js, docs/fumadocs): - next.config.mjs: headers() block for /_next/static (1y immutable), assets (1d/7d/30d), /docs/* and / (5min/1d/7d) - src/middleware.ts: UA-aware allow/deny pipeline; allowed agents pass through, blocked bots get 403 with cache header so Vercel edge serves the rejection cheaply - public/robots.txt: same allow/deny list as marketing site --- docs/fumadocs/next.config.mjs | 28 +++++++ docs/fumadocs/public/robots.txt | 138 ++++++++++++++++++++++++++++++++ docs/fumadocs/src/middleware.ts | 22 +++++ docs/skillkit/public/robots.txt | 138 ++++++++++++++++++++++++++++++++ docs/skillkit/vercel.json | 42 ++++++++++ 5 files changed, 368 insertions(+) create mode 100644 docs/fumadocs/public/robots.txt create mode 100644 docs/fumadocs/src/middleware.ts create mode 100644 docs/skillkit/public/robots.txt diff --git a/docs/fumadocs/next.config.mjs b/docs/fumadocs/next.config.mjs index c87d1246..06dcd602 100644 --- a/docs/fumadocs/next.config.mjs +++ b/docs/fumadocs/next.config.mjs @@ -8,6 +8,34 @@ const config = { trailingSlash: false, skipTrailingSlashRedirect: true, assetPrefix: 'https://skillkit-docs.vercel.app', + async headers() { + return [ + { + source: '/:path*.(css|js|png|jpg|jpeg|svg|webp|woff2|woff|ttf|ico)', + headers: [ + { key: 'Cache-Control', value: 'public, max-age=86400, s-maxage=604800, stale-while-revalidate=2592000' }, + ], + }, + { + source: '/_next/static/:path*', + headers: [ + { key: 'Cache-Control', value: 'public, max-age=31536000, immutable' }, + ], + }, + { + source: '/docs/:path*', + headers: [ + { key: 'Cache-Control', value: 'public, max-age=300, s-maxage=86400, stale-while-revalidate=604800' }, + ], + }, + { + source: '/', + headers: [ + { key: 'Cache-Control', value: 'public, max-age=300, s-maxage=86400, stale-while-revalidate=604800' }, + ], + }, + ]; + }, }; export default withMDX(config); diff --git a/docs/fumadocs/public/robots.txt b/docs/fumadocs/public/robots.txt new file mode 100644 index 00000000..7a6c650f --- /dev/null +++ b/docs/fumadocs/public/robots.txt @@ -0,0 +1,138 @@ +# Humans + agents welcome. Training crawlers + SEO scrapers blocked. + +User-agent: Googlebot +Allow: / + +User-agent: Bingbot +Allow: / + +User-agent: DuckDuckBot +Allow: / + +User-agent: Applebot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Perplexity-User +Allow: / + +User-agent: Claude-User +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: FirecrawlAgent +Allow: / + +User-agent: firecrawl +Allow: / + +User-agent: Context7Bot +Allow: / + +User-agent: Crawl4AI +Allow: / + +User-agent: Clawdbot +Allow: / + +User-agent: OpenClaw +Allow: / + +User-agent: Hermes +Allow: / + +User-agent: GPTBot +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: Applebot-Extended +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: FacebookBot +Disallow: / + +User-agent: Meta-ExternalAgent +Disallow: / + +User-agent: cohere-ai +Disallow: / + +User-agent: Diffbot +Disallow: / + +User-agent: ImagesiftBot +Disallow: / + +User-agent: Omgilibot +Disallow: / + +User-agent: peer39_crawler +Disallow: / + +User-agent: YouBot +Disallow: / + +User-agent: Timpibot +Disallow: / + +User-agent: ICC-Crawler +Disallow: / + +User-agent: AhrefsBot +Disallow: / + +User-agent: SemrushBot +Disallow: / + +User-agent: MJ12bot +Disallow: / + +User-agent: DotBot +Disallow: / + +User-agent: PetalBot +Disallow: / + +User-agent: BLEXBot +Disallow: / + +User-agent: MegaIndex +Disallow: / + +User-agent: SeznamBot +Disallow: / + +User-agent: DataForSeoBot +Disallow: / + +User-agent: * +Allow: / + +Sitemap: https://skillkit-docs.vercel.app/sitemap.xml diff --git a/docs/fumadocs/src/middleware.ts b/docs/fumadocs/src/middleware.ts new file mode 100644 index 00000000..6a1a7fb4 --- /dev/null +++ b/docs/fumadocs/src/middleware.ts @@ -0,0 +1,22 @@ +import { NextResponse } from 'next/server'; +import type { NextRequest } from 'next/server'; + +const BLOCK = /GPTBot|ClaudeBot|anthropic-ai|CCBot|Google-Extended|Applebot-Extended|Bytespider|Amazonbot|Meta-ExternalAgent|cohere-ai|Diffbot|ImagesiftBot|Omgilibot|peer39_crawler|YouBot|Timpibot|ICC-Crawler|AhrefsBot|SemrushBot|MJ12bot|DotBot|PetalBot|BLEXBot|MegaIndex|SeznamBot|DataForSeoBot/i; + +const ALLOW = /Googlebot|Bingbot|DuckDuckBot|Applebot(?!-Extended)|ChatGPT-User|OAI-SearchBot|PerplexityBot|Perplexity-User|Claude-User|Claude-SearchBot|FirecrawlAgent|firecrawl|Context7Bot|Crawl4AI|Clawdbot|OpenClaw|Hermes/i; + +export function middleware(req: NextRequest) { + const ua = req.headers.get('user-agent') || ''; + if (ALLOW.test(ua)) return NextResponse.next(); + if (BLOCK.test(ua)) { + return new NextResponse('disallowed by robots.txt', { + status: 403, + headers: { 'Cache-Control': 'public, max-age=86400' }, + }); + } + return NextResponse.next(); +} + +export const config = { + matcher: '/((?!_next/static|_next/image|favicon|robots\\.txt|sitemap\\.xml).*)', +}; diff --git a/docs/skillkit/public/robots.txt b/docs/skillkit/public/robots.txt new file mode 100644 index 00000000..b83202ae --- /dev/null +++ b/docs/skillkit/public/robots.txt @@ -0,0 +1,138 @@ +# Humans + agents welcome. Training crawlers + SEO scrapers blocked. + +User-agent: Googlebot +Allow: / + +User-agent: Bingbot +Allow: / + +User-agent: DuckDuckBot +Allow: / + +User-agent: Applebot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Perplexity-User +Allow: / + +User-agent: Claude-User +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: FirecrawlAgent +Allow: / + +User-agent: firecrawl +Allow: / + +User-agent: Context7Bot +Allow: / + +User-agent: Crawl4AI +Allow: / + +User-agent: Clawdbot +Allow: / + +User-agent: OpenClaw +Allow: / + +User-agent: Hermes +Allow: / + +User-agent: GPTBot +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: Applebot-Extended +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: FacebookBot +Disallow: / + +User-agent: Meta-ExternalAgent +Disallow: / + +User-agent: cohere-ai +Disallow: / + +User-agent: Diffbot +Disallow: / + +User-agent: ImagesiftBot +Disallow: / + +User-agent: Omgilibot +Disallow: / + +User-agent: peer39_crawler +Disallow: / + +User-agent: YouBot +Disallow: / + +User-agent: Timpibot +Disallow: / + +User-agent: ICC-Crawler +Disallow: / + +User-agent: AhrefsBot +Disallow: / + +User-agent: SemrushBot +Disallow: / + +User-agent: MJ12bot +Disallow: / + +User-agent: DotBot +Disallow: / + +User-agent: PetalBot +Disallow: / + +User-agent: BLEXBot +Disallow: / + +User-agent: MegaIndex +Disallow: / + +User-agent: SeznamBot +Disallow: / + +User-agent: DataForSeoBot +Disallow: / + +User-agent: * +Allow: / + +Sitemap: https://skillkit.dev/sitemap.xml diff --git a/docs/skillkit/vercel.json b/docs/skillkit/vercel.json index 06e9d5bf..fc43099b 100644 --- a/docs/skillkit/vercel.json +++ b/docs/skillkit/vercel.json @@ -21,5 +21,47 @@ "source": "/docs", "destination": "https://skillkit-docs.vercel.app/docs" } + ], + "headers": [ + { + "source": "/(.*)\\.(css|js|png|jpg|jpeg|svg|webp|woff2|woff|ttf|ico|mp4)", + "headers": [ + { "key": "Cache-Control", "value": "public, max-age=86400, s-maxage=604800, stale-while-revalidate=2592000" } + ] + }, + { + "source": "/assets/(.*)", + "headers": [ + { "key": "Cache-Control", "value": "public, max-age=31536000, immutable" } + ] + }, + { + "source": "/(.*)\\.html", + "headers": [ + { "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" } + ] + }, + { + "source": "/", + "headers": [ + { "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" } + ] + }, + { + "source": "/api", + "headers": [ + { "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" } + ] + } + ], + "redirects": [ + { + "source": "/((?!robots\\.txt$).*)", + "has": [ + { "type": "header", "key": "user-agent", "value": "(?i).*(GPTBot|ClaudeBot|anthropic-ai|CCBot|Google-Extended|Applebot-Extended|Bytespider|Amazonbot|Meta-ExternalAgent|cohere-ai|Diffbot|ImagesiftBot|Omgilibot|peer39_crawler|YouBot|Timpibot|ICC-Crawler|AhrefsBot|SemrushBot|MJ12bot|DotBot|PetalBot|BLEXBot|MegaIndex|SeznamBot|DataForSeoBot).*" } + ], + "destination": "/robots.txt", + "permanent": false + } ] }