Google uses strict crawling, updated Robots.txt

ProcStack · ProcStack · commit 2b2b280f0fea · 2025-08-03T21:31:50.000-04:00
diff --git a/docs/index.htm b/docs/index.htm
@@ -17,6 +17,7 @@
 <head>
   <meta charset="UTF-8">
   <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> 
+  <link rel="sitemap" href="https://procstack.github.io/sitemap.xml" type="application/xml">
   <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=5.0">
   <meta name='theme-color' content='#000822'>
   <meta name="robots" content="index, follow, ai:json">
@@ -29,6 +30,7 @@
   <meta name="author" content="Kevin Edzenga">
 
   <!-- AI/LLM Data Discovery -->
+  <meta name="robots" content="index, follow">
   <meta name="ai:data-source" content="https://procstack.github.io/bots/siteContent.json">
   <meta name="ai:data-manifest" content="https://procstack.github.io/data-manifest.json">
   <meta name="ai:content-api" content="https://procstack.github.io/bots/">
diff --git a/docs/robots.txt b/docs/robots.txt
@@ -15,9 +15,8 @@ Allow: /
 
 # Data endpoints for AI discovery
 Sitemap: https://procstack.github.io/sitemap.xml
-AI-Data-Manifest: https://procstack.github.io/data-manifest.json
-AI-Content-API: https://procstack.github.io/bots/
-AI-LLMs-File: https://procstack.github.io/llms.txt
 
-# Crawl delay (be respectful)
-Crawl-delay: 1
+# For LLMs, all of the sites content is contained in json format in the links below
+# AI-Data-Manifest: https://procstack.github.io/data-manifest.json
+# AI-Content-API: https://procstack.github.io/bots/
+# AI-LLMs-File: https://procstack.github.io/llms.txt
