Add robots.txt and structured data for SEO (#3610)

Add docs/robots.txt pointing crawlers to the sitemap. Add JSON-LD
structured data to main.html: WebSite + Organization schema on the
homepage (helps Google show sitelinks and knowledge panel), and
BreadcrumbList schema on all other pages (helps Google show breadcrumb
trails in search results). Also update the Twitter social link to x.com.

AI Assistance: Claude

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: peterschmidt85

docs/overrides/main.html

@@ -5,6 +5,85 @@
 <link rel="preload" href="{{ 'assets/fonts/GeistMono-Variable.woff2' | url }}" as="font" type="font/woff2" crossorigin>
 <link rel="preload" href="{{ 'assets/fonts/GeistPixel-Square.woff2' | url }}" as="font" type="font/woff2" crossorigin>
 <link rel="preload" href="{{ 'assets/fonts/GeistPixel-Circle.woff2' | url }}" as="font" type="font/woff2" crossorigin>
+{#
+  Structured data (JSON-LD) for SEO.
+  - Homepage gets WebSite + Organization schema (helps Google show sitelinks and knowledge panel).
+  - All other pages get BreadcrumbList schema (helps Google show breadcrumb trails in results).
+  Breadcrumb URLs are resolved via _find_leaf_url because MkDocs nav sections don't have
+  their own URLs — we use the first descendant page's URL as a proxy.
+  Dedup by title to collapse nav levels duplicated by plugins (e.g. the blog plugin nests
+  "Blog" inside "Blog"). The current page is omitted when its title matches the last
+  ancestor (e.g. /examples/ is both the "Examples" section index and the page itself).
+#}
+{% macro _find_leaf_url(nav_item) -%}
+  {%- if nav_item.url -%}
+    /{{ nav_item.url }}
+  {%- elif nav_item.children -%}
+    {{ _find_leaf_url(nav_item.children | first) }}
+  {%- endif -%}
+{%- endmacro %}
+{% if page.is_homepage %}
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@graph": [
+    {
+      "@type": "WebSite",
+      "name": "dstack",
+      "url": "https://dstack.ai/"
+    },
+    {
+      "@type": "Organization",
+      "name": "dstack",
+      "url": "https://dstack.ai/",
+      "logo": "https://dstack.ai/assets/images/dstack-logo-notext.svg",
+      "sameAs": [
+        "https://github.com/dstackai/dstack",
+        "https://discord.gg/u8SmfwPpMd",
+        "https://x.com/dstackai",
+        "https://www.linkedin.com/company/dstackai"
+      ]
+    }
+  ]
+}
+</script>
+{% elif page.ancestors | length > 0 %}
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@type": "BreadcrumbList",
+  "itemListElement": [
+    {
+      "@type": "ListItem",
+      "position": 1,
+      "name": "Home",
+      "item": "https://dstack.ai/"
+    }
+    {%- set pos = namespace(val=2) -%}
+    {%- set prev = namespace(title="") -%}
+    {%- for ancestor in page.ancestors | reverse -%}
+    {%- if ancestor.title != prev.title -%}
+    ,{
+      "@type": "ListItem",
+      "position": {{ pos.val }},
+      "name": "{{ ancestor.title | e }}",
+      "item": "https://dstack.ai{{ _find_leaf_url(ancestor) | trim }}"
+    }
+    {%- set pos.val = pos.val + 1 -%}
+    {%- endif -%}
+    {%- set prev.title = ancestor.title -%}
+    {%- endfor -%}
+    {%- if page.title != prev.title -%}
+    ,{
+      "@type": "ListItem",
+      "position": {{ pos.val }},
+      "name": "{{ page.title | e }}"
+    }
+    {%- endif -%}
+  ]
+}
+</script>
+{% endif %}
 {% endblock %}
 
 {% block container %}

docs/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://dstack.ai/sitemap.xml

mkdocs.yml

@@ -212,7 +212,7 @@ extra:
     - icon: /fontawesome/brands/discord
       link: https://discord.gg/u8SmfwPpMd
     - icon: /simple/x
-      link: https://twitter.com/dstackai
+      link: https://x.com/dstackai
     - icon: /fontawesome/brands/linkedin
       link: https://www.linkedin.com/company/dstackai
   status: