From 2fe4de3db9564eebabd329bfc11ad68782dcfb09 Mon Sep 17 00:00:00 2001
From: Jason Doyle <46789294+Jason-Doyle@users.noreply.github.com>
Date: Thu, 24 Sep 2026 22:10:20 -0700
Subject: [PATCH] Use standards-compliant robots directives

---
 site/scripts/validate-production.mjs | 5 +----
 site/src/pages/robots.txt.ts         | 3 +--
 site/tests/site.spec.ts              | 4 +---
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/site/scripts/validate-production.mjs b/site/scripts/validate-production.mjs
index 5774084..0d2ab75 100644
--- a/site/scripts/validate-production.mjs
+++ b/site/scripts/validate-production.mjs
@@ -39,10 +39,7 @@ assert.equal(robots.status, 200);
 const robotsText = await robots.text();
 assert.match(robotsText, /User-agent: \*/);
 assert.match(robotsText, /Allow: \//);
-assert.match(
-  robotsText,
-  /Content-Signal: search=yes, ai-input=yes, ai-train=yes/,
-);
+assert.doesNotMatch(robotsText, /Disallow:/);
 
 for (const userAgent of [
   "OAI-SearchBot",
diff --git a/site/src/pages/robots.txt.ts b/site/src/pages/robots.txt.ts
index 4ace82a..40cef81 100644
--- a/site/src/pages/robots.txt.ts
+++ b/site/src/pages/robots.txt.ts
@@ -5,10 +5,9 @@ export const prerender = true;
 export function GET() {
   return new Response(
     [
-      "# ThimbleDB permits search indexing, real-time AI input, and model training.",
+      "# ThimbleDB permits crawling and search indexing.",
       "User-agent: *",
       "Allow: /",
-      "Content-Signal: search=yes, ai-input=yes, ai-train=yes",
       `Sitemap: ${site.url}/sitemap-index.xml`,
       "",
     ].join("\n"),
diff --git a/site/tests/site.spec.ts b/site/tests/site.spec.ts
index fdc09e0..2da94d5 100644
--- a/site/tests/site.spec.ts
+++ b/site/tests/site.spec.ts
@@ -96,9 +96,7 @@ test("AI discovery routes publish explicit access and decision content", async (
   const robotsText = await robots.text();
   expect(robotsText).toContain("User-agent: *");
   expect(robotsText).toContain("Allow: /");
-  expect(robotsText).toContain(
-    "Content-Signal: search=yes, ai-input=yes, ai-train=yes",
-  );
+  expect(robotsText).not.toContain("Disallow:");
 
   const llms = await request.get("/llms.txt");
   expect(llms.ok()).toBe(true);