fix(seo): answer Markdown OPTIONS on unknown paths and list every accepted knowledge-base file type (#9101)

Author: waleedlatif1

apps/sim/app/markdown-not-found/route.ts

@@ -35,3 +35,4 @@ export const POST = notFound
 export const PUT = notFound
 export const PATCH = notFound
 export const DELETE = notFound
+export const OPTIONS = notFound

apps/sim/lib/compare/data/competitors/dust.ts

@@ -54,7 +54,7 @@ export const dustProfile: CompetitorProfile = {
     {
       question: 'Sim vs Dust for building company knowledge agents: which is better?',
       answer:
-        "It depends on where the agent needs to live. Dust connects agents to sources such as Slack, Notion, Google Drive, Confluence, and GitHub and serves them in its web chat, Slack, and Microsoft Teams, with access scoped by Spaces. Sim's knowledge base combines vector and full-text search over 12 document formats with connectors that keep sources such as Google Drive, Confluence, and Slack in sync, and a knowledge agent can be published as a chat interface, a REST API, or an MCP server. Dust suits a managed assistant used mainly in chat; Sim suits a knowledge agent that also has to feed workflows or other systems.",
+        "It depends on where the agent needs to live. Dust connects agents to sources such as Slack, Notion, Google Drive, Confluence, and GitHub and serves them in its web chat, Slack, and Microsoft Teams, with access scoped by Spaces. Sim's knowledge base combines vector and full-text search over 15 file types with connectors that keep sources such as Google Drive, Confluence, and Slack in sync, and a knowledge agent can be published as a chat interface, a REST API, or an MCP server. Dust suits a managed assistant used mainly in chat; Sim suits a knowledge agent that also has to feed workflows or other systems.",
     },
     {
       question: 'How does Sim compare to Dust for shared team AI assistants?',

apps/sim/lib/compare/data/sim.ts

@@ -41,9 +41,9 @@ export const simProfile: CompetitorProfile = {
     {
       title: 'Hybrid semantic + keyword knowledge base',
       description:
-        'Built-in RAG with pgvector embeddings and a generated tsvector column for combined vector + full-text search, plus a token-based chunker with configurable chunk size/overlap and 12 supported document formats (csv, doc, docx, html, json, jsonl, md, pdf, pptx, txt, xlsx, yaml).',
+        'Built-in RAG with pgvector embeddings and a generated tsvector column for combined vector + full-text search, plus a token-based chunker with configurable chunk size/overlap and 15 supported file types (csv, doc, docx, htm, html, json, jsonl, md, pdf, pptx, txt, xls, xlsx, yaml, yml).',
       shortDescription:
-        'Combines vector and full-text search with configurable chunking across 12 document formats.',
+        'Combines vector and full-text search with configurable chunking across 15 file types.',
       source: {
         url: 'https://github.com/simstudioai/sim/blob/main/packages/db/schema.ts',
         label: 'Sim codebase: KB schema + file-parsers',
@@ -406,8 +406,8 @@ export const simProfile: CompetitorProfile = {
         ],
       },
       knowledgeBaseRag: {
-        value: `Yes: hybrid vector (pgvector) plus full-text (tsvector) search knowledge base, 12 supported document formats (csv, doc, docx, html, json, jsonl, md, pdf, pptx, txt, xlsx, yaml), configurable chunking, plus ${CONNECTOR_COUNT_LABEL} connectors that continuously sync external sources (Google Drive, Confluence, Slack, Gmail, GitHub, HubSpot, Linear, Jira, and more) into the knowledge base rather than a one-shot upload`,
-        shortValue: `Hybrid vector + full-text search, 12 document formats, ${CONNECTOR_COUNT_LABEL} connectors`,
+        value: `Yes: hybrid vector (pgvector) plus full-text (tsvector) search knowledge base, 15 supported file types (csv, doc, docx, htm, html, json, jsonl, md, pdf, pptx, txt, xls, xlsx, yaml, yml), configurable chunking, plus ${CONNECTOR_COUNT_LABEL} connectors that continuously sync external sources (Google Drive, Confluence, Slack, Gmail, GitHub, HubSpot, Linear, Jira, and more) into the knowledge base rather than a one-shot upload`,
+        shortValue: `Hybrid vector + full-text search, 15 file types, ${CONNECTOR_COUNT_LABEL} connectors`,
         confidence: 'verified',
         sources: [
           {

apps/sim/scripts/test-mcp-host-e2e.ts

@@ -169,17 +169,19 @@ try {
     })
   }
 
-  await check('an unknown page answers a Markdown POST in Markdown, not 405', async () => {
-    const { response } = await request(
-      '/not-a-real-page',
-      'python-requests/2.32.3',
-      baseUrl.host,
-      'POST',
-      MARKDOWN_ACCEPT
-    )
-    assert.equal(response.status, 404)
-    assert.equal(response.headers.get('content-type'), 'text/markdown; charset=utf-8')
-  })
+  for (const method of ['POST', 'OPTIONS']) {
+    await check(`an unknown page answers a Markdown ${method} in Markdown`, async () => {
+      const { response } = await request(
+        '/not-a-real-page',
+        'python-requests/2.32.3',
+        baseUrl.host,
+        method,
+        MARKDOWN_ACCEPT
+      )
+      assert.equal(response.status, 404)
+      assert.equal(response.headers.get('content-type'), 'text/markdown; charset=utf-8')
+    })
+  }
 
   await check('a client that refuses Markdown with q=0 gets the HTML 404', async () => {
     const { response } = await request(