improvement(library): single JSON-LD source, Markdown articles, and IndexNow pings (#9355)

* improvement(library): single JSON-LD source, Markdown articles, and IndexNow pings

* improvement(library): link Andrew's author profile and surface the docs history fetch outcome in build logs

* fix(library): gate IndexNow on the ECS cutover, cover unpublished and renamed pages, and give customer stories speakable hooks

* fix(docs): fetch commit history from the repository Vercel names, since its clone has no origin remote

* fix(library): noindex direct Markdown article URLs on self-hosted deployments and diff IndexNow from the last submitted revision

Author: waleedlatif1

.github/workflows/ci.yml

@@ -569,6 +569,9 @@ jobs:
       needs.promote.result == 'success' &&
       needs.trigger-upload.result == 'success' &&
       needs.promote.outputs.promoted == 'true'
+    outputs:
+      # The new app serves traffic; indexnow keys off this, not the later Trigger.dev promotion.
+      cutover: ${{ steps.cutover.outcome }}
     # Mostly waiting on the ECS cutover after a dependency install: the smallest runner is enough.
     runs-on: *runner-2vcpu
     # Leave setup/promotion headroom above the 70-minute cutover poll.
@@ -596,6 +599,7 @@ jobs:
       # An unchanged tag may belong to a failed or still-running earlier deploy.
       # Verify its latest cutover rather than treating tag equality as success.
       - name: Wait for ECS traffic cutover
+        id: cutover
         env:
           OVERALL_TIMEOUT: 4200
           APP_IMAGE_CHANGED: ${{ needs.promote.outputs.app_image_changed }}
@@ -636,6 +640,59 @@ jobs:
           fi
           bunx trigger.dev@4.5.16 promote "$VERSION" "${TARGET_ARGS[@]}"
 
+  # Production only, once the new app serves traffic: submit the library and comparison URLs this
+  # push changed to IndexNow. Best effort: nothing depends on it, and a failure never fails the run.
+  indexnow:
+    name: indexnow
+    needs: [promote, trigger-promote]
+    if: >-
+      !cancelled() &&
+      github.event_name == 'push' &&
+      github.ref == 'refs/heads/main' &&
+      needs.promote.outputs.promoted == 'true' &&
+      needs.trigger-promote.outputs.cutover == 'success'
+    runs-on: *runner-2vcpu
+    timeout-minutes: 5
+    continue-on-error: true
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.4.4
+
+      # Diff from the head of the last earlier run whose submission succeeded, not the previous
+      # push: a push that never deployed, or whose submission failed, would otherwise drop its
+      # changed URLs for good. Falls back to the previous push when no such run is in view. The
+      # diff needs only the two trees, so that one commit is fetched alone; an all-zero `before`
+      # (the branch's first push) has no base, which the script skips.
+      - name: Submit changed URLs to IndexNow
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PUSH_BEFORE: ${{ github.event.before }}
+        run: |
+          BASE_SHA=""
+          while read -r run_id head_sha; do
+            conclusion="$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100" \
+              --jq '.jobs[] | select(.name == "indexnow") | .conclusion')"
+            if [[ "$conclusion" == "success" ]]; then
+              BASE_SHA="$head_sha"
+              break
+            fi
+          done < <(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?branch=main&event=push&per_page=30" \
+            --jq ".workflow_runs[] | select(.id < $GITHUB_RUN_ID) | \"\\(.id) \\(.head_sha)\"")
+          BASE_SHA="${BASE_SHA:-$PUSH_BEFORE}"
+          echo "Submitting URLs changed in $BASE_SHA..$GITHUB_SHA"
+          if [[ ! "$BASE_SHA" =~ ^0+$ ]]; then
+            git fetch --no-tags --depth=1 origin "$BASE_SHA"
+          fi
+          bun scripts/ping-indexnow.ts "$BASE_SHA" "$GITHUB_SHA"
+
   # Build ARM64 images for GHCR (main branch only, runs in parallel with
   # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64
   # are applied by ghcr after the gate, so a failing run

apps/docs/scripts/fetch-git-history.ts

@@ -3,6 +3,8 @@
  * shallow clone (every file untouched since the boundary commit would get that commit's date), so
  * the docs sitemap shipped with no `lastModified` at all. Fetch the commit history before the
  * build. `--filter=blob:none` skips file contents: `git log --name-only` needs commits and trees.
+ * Vercel's clone has no `origin` remote, so the history comes from the public repository URL its
+ * system environment names, pinned to the commit being built.
  *
  * A failed fetch leaves the clone shallow, so pages ship undated as before. That is logged rather
  * than failing the deploy, because dates are an enhancement and the docs must still ship.
@@ -11,7 +13,8 @@ import { execFileSync } from 'node:child_process'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 
-const logger = createLogger('DocsGitHistory')
+/** A build step's outcome belongs in the build log, which the production default (errors only) would hide. */
+const logger = createLogger('DocsGitHistory', { logLevel: 'INFO' })
 
 /** Bounds the fetch so a stalled remote costs at most this much of the deployment, never all of it. */
 const FETCH_TIMEOUT_MS = 120_000
@@ -24,11 +27,40 @@ function git(args: string[], timeout?: number): string {
   }).trim()
 }
 
-if (process.env.VERCEL) {
+const {
+  VERCEL,
+  VERCEL_GIT_PROVIDER,
+  VERCEL_GIT_REPO_OWNER,
+  VERCEL_GIT_REPO_SLUG,
+  VERCEL_GIT_COMMIT_SHA,
+} = process.env
+
+if (VERCEL) {
   try {
     if (git(['rev-parse', '--is-shallow-repository']) === 'true') {
-      git(['fetch', '--unshallow', '--filter=blob:none', '--no-tags', 'origin'], FETCH_TIMEOUT_MS)
+      if (
+        VERCEL_GIT_PROVIDER !== 'github' ||
+        !VERCEL_GIT_REPO_OWNER ||
+        !VERCEL_GIT_REPO_SLUG ||
+        !VERCEL_GIT_COMMIT_SHA
+      ) {
+        throw new Error('Vercel did not provide the GitHub repository and commit for this build')
+      }
+      const repositoryUrl = `https://github.com/${VERCEL_GIT_REPO_OWNER}/${VERCEL_GIT_REPO_SLUG}.git`
+      git(
+        [
+          'fetch',
+          '--unshallow',
+          '--filter=blob:none',
+          '--no-tags',
+          repositoryUrl,
+          VERCEL_GIT_COMMIT_SHA,
+        ],
+        FETCH_TIMEOUT_MS
+      )
       logger.info('Fetched full commit history for docs last-modified dates')
+    } else {
+      logger.info('Clone already has full commit history')
     }
   } catch (error) {
     logger.warn(

apps/sim/app/(landing)/blog/[slug]/page.tsx

@@ -24,7 +24,7 @@ export async function generateMetadata({
   const { slug } = await params
   const post = await getPostBySlug(slug)
   if (!post || post.draft) return {}
-  return buildPostMetadata(post)
+  return buildPostMetadata(post, BLOG_SECTION)
 }
 
 export const revalidate = 86400

apps/sim/app/(landing)/changelog/[slug]/page.tsx

@@ -19,7 +19,7 @@ export async function generateStaticParams() {
 export async function generateMetadata({ params }: ChangelogEntryPageProps): Promise<Metadata> {
   const { slug } = await params
   const entry = await getEntryBySlug(slug)
-  return entry ? buildPostMetadata(entry) : {}
+  return entry ? buildPostMetadata(entry, CHANGELOG_SECTION) : {}
 }
 
 export default async function ChangelogEntryPage({ params }: ChangelogEntryPageProps) {

apps/sim/app/(landing)/components/content-post-page/content-post-page.tsx

@@ -40,7 +40,7 @@ export function ContentPostPage({
   const showUpdated = modifiedIso.slice(0, 10) !== post.date.slice(0, 10)
 
   return (
-    <article className='w-full bg-[var(--bg)]' itemScope itemType='https://schema.org/BlogPosting'>
+    <article className='w-full bg-[var(--bg)]'>
       <JsonLd data={graphJsonLd} />
       <header className='mx-auto w-full max-w-[1728px] px-10 pt-[112px] max-sm:pt-20 max-md:px-7 max-lg:px-8 max-xl:px-9'>
         <div className='mb-6'>
@@ -59,7 +59,6 @@ export function ContentPostPage({
                 sizes='(max-width: 1023px) calc(100vw - 64px), 450px'
                 priority
                 fetchPriority='high'
-                itemProp='image'
                 unoptimized
               />
             </div>
@@ -68,13 +67,13 @@ export function ContentPostPage({
             <div>
               <h1
                 className='text-balance text-[28px] text-[var(--text-primary)] leading-[110%] tracking-[-0.02em] sm:text-[36px] md:text-[44px] lg:text-[52px]'
-                itemProp='headline'
+                data-speakable='headline'
               >
                 {post.title}
               </h1>
               <p
                 className='mt-4 text-[var(--text-body)] text-base leading-[150%] tracking-[0.02em] sm:text-lg sm:leading-7'
-                itemProp='description'
+                data-speakable='description'
               >
                 {post.description}
               </p>
@@ -84,25 +83,21 @@ export function ContentPostPage({
                 <time
                   className='text-[var(--text-secondary)] text-xs uppercase tracking-[0.1em]'
                   dateTime={post.date}
-                  itemProp='datePublished'
                 >
                   {formatPostDate(post.date)}
                 </time>
-                {showUpdated ? (
+                {showUpdated && (
                   <>
                     <span aria-hidden='true' className='text-[var(--text-secondary)] text-xs'>
                       ·
                     </span>
                     <time
                       className='text-[var(--text-secondary)] text-xs uppercase tracking-[0.1em]'
                       dateTime={modifiedIso}
-                      itemProp='dateModified'
                     >
                       Updated {formatPostDate(modifiedIso)}
                     </time>
                   </>
-                ) : (
-                  <meta itemProp='dateModified' content={modifiedIso} />
                 )}
               </div>
               <div className='flex items-center gap-3'>
@@ -118,11 +113,8 @@ export function ContentPostPage({
                       href={`${basePath}/authors/${encodeURIComponent(a?.id ?? '')}`}
                       rel='author'
                       className='text-[var(--text-secondary)] text-xs uppercase tracking-[0.1em] hover:text-[var(--text-primary)]'
-                      itemProp='author'
-                      itemScope
-                      itemType='https://schema.org/Person'
                     >
-                      <span itemProp='name'>{a?.name}</span>
+                      {a?.name}
                     </Link>
                   </div>
                 ))}
@@ -139,7 +131,7 @@ export function ContentPostPage({
 
       <div className='mx-auto w-full max-w-[1728px] px-10 max-md:px-7 max-lg:px-8 max-xl:px-9'>
         <div className='border-[var(--border)] border-x'>
-          <div className='mx-auto max-w-[900px] px-6 py-16' itemProp='articleBody'>
+          <div className='mx-auto max-w-[900px] px-6 py-16'>
             <div
               className={cn(
                 'prose prose-lg max-w-none prose-headings:font-season prose-headings:tracking-[-0.02em]',
@@ -169,11 +161,6 @@ export function ContentPostPage({
       </div>
 
       <div className='-mt-px h-px w-full bg-[var(--border)]' />
-
-      <meta itemProp='publisher' content='Sim' />
-      <meta itemProp='inLanguage' content='en-US' />
-      <meta itemProp='keywords' content={post.tags.join(', ')} />
-      {post.wordCount && <meta itemProp='wordCount' content={String(post.wordCount)} />}
     </article>
   )
 }

apps/sim/app/(landing)/customers/[slug]/page.tsx

@@ -21,7 +21,7 @@ export async function generateMetadata({ params }: CustomerPageProps): Promise<M
   const { slug } = await params
   if (!CUSTOMER_STORIES.some((story) => story.slug === slug)) return {}
   const post = await getCustomerStoryBySlug(slug)
-  return post ? buildPostMetadata(post) : {}
+  return post ? buildPostMetadata(post, CUSTOMER_SECTION) : {}
 }
 
 export default async function Page({ params }: CustomerPageProps) {

apps/sim/app/(landing)/customers/components/customer-story-page/customer-story-page.tsx

@@ -28,7 +28,10 @@ export function CustomerStoryPage({ story, post, nextStory }: CustomerStoryPageP
       <article className={HOME_INSET}>
         <header className={cn('pb-12 max-sm:pb-8', LANDING_HERO_TOP_PADDING)}>
           <BackLink href='/customers' label='Customer stories' />
-          <h1 className='mt-7 max-w-[1100px] text-balance text-[64px] text-[var(--text-primary)] leading-[1.04] tracking-[-0.03em] max-sm:mt-5 max-sm:text-[36px] max-xl:text-[52px]'>
+          <h1
+            className='mt-7 max-w-[1100px] text-balance text-[64px] text-[var(--text-primary)] leading-[1.04] tracking-[-0.03em] max-sm:mt-5 max-sm:text-[36px] max-xl:text-[52px]'
+            data-speakable='headline'
+          >
             {post.title}
           </h1>
         </header>
@@ -72,7 +75,10 @@ export function CustomerStoryPage({ story, post, nextStory }: CustomerStoryPageP
           </aside>
 
           <div className='min-w-0 max-w-[740px]'>
-            <p className='mb-10 text-pretty text-[28px] text-[var(--text-primary)] leading-[1.3] tracking-[-0.015em] max-sm:text-[23px]'>
+            <p
+              className='mb-10 text-pretty text-[28px] text-[var(--text-primary)] leading-[1.3] tracking-[-0.015em] max-sm:text-[23px]'
+              data-speakable='description'
+            >
               {post.description}
             </p>
             <div className='[&_h2:first-child]:!mt-0 [&_:is(h1,h2,h3,h4,h5,h6,a,strong,b)]:font-normal [&_h2]:scroll-mt-28 [&_p]:text-pretty'>

apps/sim/app/library.md/[slug]/route.ts

@@ -0,0 +1,57 @@
+import type { NextRequest } from 'next/server'
+import type { ContentMeta } from '@/lib/content/schema'
+import { toSiteUrl } from '@/lib/core/utils/urls'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { getAllPostMeta, getPostSource } from '@/lib/library/registry'
+import { LIBRARY_SECTION } from '@/lib/library/seo'
+import { markdownResponse, section, toLlmsMarkdown } from '@/app/llms.txt/llms'
+
+type RouteContext = { params: Promise<{ slug: string }> }
+
+const NOT_FOUND_MARKDOWN = `# Article not found
+
+There is no library article at this address. Browse the [library](${toSiteUrl(LIBRARY_SECTION.basePath)}) or the [llms.txt](${toSiteUrl('/llms.txt')}) index.
+`
+
+function postMarkdownBlocks(post: ContentMeta, source: string): string[] {
+  const authors = (post.authors.length > 0 ? post.authors : [post.author]).map((a) => a.name)
+  const published = post.date.slice(0, 10)
+  const updated = (post.updated ?? post.date).slice(0, 10)
+  return [
+    `# ${post.title}`,
+    `> ${post.description}`,
+    [
+      `URL: ${post.canonical}`,
+      `By ${authors.join(', ')} · Published ${published}${updated !== published ? ` · Updated ${updated}` : ''}`,
+    ].join('\n'),
+    toLlmsMarkdown(source, 0),
+    section(
+      'FAQ',
+      (post.faq ?? []).map((item) => `### ${item.q}\n\n${item.a}`),
+      '\n\n'
+    ),
+  ]
+}
+
+/**
+ * A library article as Markdown, at `/library/<slug>.md` (and at `/library/<slug>` for a client
+ * whose `Accept` names `text/markdown`), via the rewrites in `next.config.ts`. Drafts and unknown
+ * slugs are a Markdown 404. Uncached, like the HTML post page: the same body answers at the post's
+ * own URL, so a cache that ignores `Vary` must never keep it.
+ */
+export const GET = withRouteHandler(async (_request: NextRequest, context: RouteContext) => {
+  const { slug } = await context.params
+  const [posts, source] = await Promise.all([getAllPostMeta(), getPostSource(slug)])
+  const post = posts.find((candidate) => candidate.slug === slug)
+  const response =
+    post && source !== null
+      ? markdownResponse(postMarkdownBlocks(post, source), 0)
+      : new Response(NOT_FOUND_MARKDOWN, {
+          status: 404,
+          headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
+        })
+  response.headers.set('Cache-Control', 'no-store')
+  response.headers.set('Vary', 'Accept')
+  if (post) response.headers.set('Link', `<${post.canonical}>; rel="canonical"`)
+  return response
+})

apps/sim/app/llms.txt/route.ts

@@ -79,6 +79,8 @@ export async function GET() {
         ),
       ]),
       section('Library', [
+        'Each article is also served as Markdown at its URL with `.md` appended.',
+        '',
         linkLine('Library', '/library', 'Comparisons, how-tos, and roundups on building AI agents'),
         ...libraryPosts.map((p) => linkLine(p.title, p.canonical, p.description)),
       ]),

apps/sim/content/authors/andrew.json

@@ -1,5 +1,6 @@
 {
   "id": "andrew",
   "name": "Andrew Caslow",
+  "url": "https://www.linkedin.com/in/andrewcaslow/",
   "avatarUrl": "/authors/andrew.jpg"
 }