From 804ad4afb68ca68cae60d4e96e8e4b79d4d0a66f Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 27 Aug 2026 10:33:36 +0200 Subject: [PATCH 1/8] Scaffold the draft PR for the llms.txt index (#660) From 71cc290493ad4d589d8e82892d26655ca6fb8252 Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 27 Aug 2026 10:37:39 +0200 Subject: [PATCH 2/8] Walk the docs website and derive public URLs Co-Authored-By: Claude Opus 5 (1M context) --- .fallout/build.schema.json | 1 + build/Build.Documentation.cs | 66 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 build/Build.Documentation.cs diff --git a/.fallout/build.schema.json b/.fallout/build.schema.json index 74c6b5567..c8c41563d 100644 --- a/.fallout/build.schema.json +++ b/.fallout/build.schema.json @@ -30,6 +30,7 @@ "CreateGitHubRelease", "DeletePackages", "DownloadLicenses", + "GenerateLlmsTxt", "GeneratePublicApi", "GenerateTools", "Install", diff --git a/build/Build.Documentation.cs b/build/Build.Documentation.cs new file mode 100644 index 000000000..66153b7b9 --- /dev/null +++ b/build/Build.Documentation.cs @@ -0,0 +1,66 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; +using Fallout.Common; +using Fallout.Common.IO; +using Fallout.Common.Utilities; +using Fallout.Common.Utilities.Collections; +using Serilog; + +partial class Build +{ + AbsolutePath DocsWebsiteDirectory => RootDirectory / "docs" / "website"; + + // The public site is built from docs/website by the separate Fallout-build/docs.fallout.build + // Docusaurus repository, which serves the pages under a /docs/ route prefix. Verified against + // https://docs.fallout.build/sitemap.xml, not against the README: the README's own links omit + // the prefix and 404 (see the follow-up on the PR). Hardcoded for the same reason as + // CanonicalRepositoryIdentifier: the generated file must be identical whichever fork + // regenerates it. + const string DocsBaseUrl = "https://docs.fallout.build/docs/"; + + // Docusaurus orders pages by a numeric prefix on the directory and file name, and strips that + // prefix from the served URL. So 01-getting-started/01-installation.md is served at + // /getting-started/installation, which is the URL the README already links. + static readonly Regex OrderPrefix = new(@"^(?\d+)-", RegexOptions.Compiled); + + static string StripOrderPrefix(string segment) => OrderPrefix.Replace(segment, string.Empty); + + static int GetOrder(string segment) + { + var match = OrderPrefix.Match(segment); + return match.Success ? int.Parse(match.Groups["order"].Value) : int.MaxValue; + } + + string ToPublicUrl(AbsolutePath page) + { + var relative = DocsWebsiteDirectory.GetUnixRelativePathTo(page).ToString(); + var slug = relative[..^".md".Length] + .Split('/') + .Select(StripOrderPrefix) + .JoinSlash(); + + return DocsBaseUrl + slug; + } + + // "01-getting-started" becomes "Getting Started". The directory slug is the only section name + // available: the docs tree has no per-directory metadata file. + static string ToSectionTitle(string directorySlug) + { + return StripOrderPrefix(directorySlug) + .Split('-') + .Select(x => char.ToUpperInvariant(x[0]) + x[1..]) + .JoinSpace(); + } + + Target GenerateLlmsTxt => _ => _ + .Executes(() => + { + DocsWebsiteDirectory.GlobFiles("**/*.md") + .OrderBy(x => x.ToString()) + .ForEach(x => Log.Information( + "{Relative} -> {Url}", + DocsWebsiteDirectory.GetUnixRelativePathTo(x), + ToPublicUrl(x))); + }); +} From 27dddd01f88856c2e7f48c633cec8f93abcb7428 Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 27 Aug 2026 10:43:59 +0200 Subject: [PATCH 3/8] Parse the docs frontmatter into page metadata Co-Authored-By: Claude Opus 5 (1M context) --- build/Build.Documentation.cs | 174 +++++++++++++++++++++++++++++++++-- 1 file changed, 164 insertions(+), 10 deletions(-) diff --git a/build/Build.Documentation.cs b/build/Build.Documentation.cs index 66153b7b9..470c08948 100644 --- a/build/Build.Documentation.cs +++ b/build/Build.Documentation.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using Fallout.Common; @@ -21,7 +22,7 @@ partial class Build // Docusaurus orders pages by a numeric prefix on the directory and file name, and strips that // prefix from the served URL. So 01-getting-started/01-installation.md is served at - // /getting-started/installation, which is the URL the README already links. + // /docs/getting-started/installation. static readonly Regex OrderPrefix = new(@"^(?\d+)-", RegexOptions.Compiled); static string StripOrderPrefix(string segment) => OrderPrefix.Replace(segment, string.Empty); @@ -43,24 +44,177 @@ string ToPublicUrl(AbsolutePath page) return DocsBaseUrl + slug; } - // "01-getting-started" becomes "Getting Started". The directory slug is the only section name - // available: the docs tree has no per-directory metadata file. - static string ToSectionTitle(string directorySlug) + // Each section directory carries a Docusaurus _category_.json whose "label" is what the site's + // sidebar shows, so that is the authoritative section name. It matters: title-casing the slug + // instead would give "Cicd" and "Ide" where the site says "CI/CD Support" and "IDE Support", + // and "Common" where it says "Common Tasks". The slug is only a fallback for a directory that + // has no _category_.json. + string ToSectionTitle(string directorySlug) { + var category = DocsWebsiteDirectory / directorySlug / "_category_.json"; + if (category.FileExists()) + { + var label = category.ReadJsonObject().GetPropertyValue("label"); + if (!label.IsNullOrWhiteSpace()) + return label; + } + return StripOrderPrefix(directorySlug) .Split('-') .Select(x => char.ToUpperInvariant(x[0]) + x[1..]) .JoinSpace(); } + sealed record DocPage(string Title, string Description, string Url, string Section, int SectionOrder, int Order); + + const int MaxDescriptionLength = 200; + + // Inline markdown links render as "[text](url)". Only the text belongs in a one-line summary. + static readonly Regex InlineLink = new(@"\[(?[^\]]+)\]\([^)]+\)", RegexOptions.Compiled); + + static readonly Regex FrontmatterEntry = new(@"^(?[a-zA-Z]+):\s*(?.*)$", RegexOptions.Compiled); + + DocPage ReadPage(AbsolutePath file) + { + var lines = file.ReadAllLines(); + var frontmatterEnd = GetFrontmatterEnd(lines); + var frontmatter = ReadFrontmatter(lines, frontmatterEnd); + + // Docusaurus falls back to the first H1 when a page declares no 'title', and docs/website + // has a page that relies on it: badge.md carries no frontmatter at all and is served as + // "Badge". Rejecting it would refuse a page the site renders correctly, so the fallback + // matches Docusaurus. A page with neither still fails, because that leaves no link text. + var title = frontmatter.GetValueOrDefault("title") ?? GetFirstHeading(lines); + Assert.NotNullOrWhiteSpace( + title, + $"{DocsWebsiteDirectory.GetUnixRelativePathTo(file)} has neither a 'title' in its " + + "frontmatter nor a top-level heading. One of the two is needed: it is the link text " + + "in docs/llms.txt."); + + var description = frontmatter.GetValueOrDefault("description") + ?? GetFirstProseParagraph(lines, frontmatterEnd); + + var relative = DocsWebsiteDirectory.GetUnixRelativePathTo(file).ToString(); + var segments = relative.Split('/'); + var isNested = segments.Length > 1; + + return new DocPage( + Title: title, + Description: Summarize(description), + Url: ToPublicUrl(file), + // Root-level pages have no section. Task 3 collects them under "Optional", which is the + // part of the llmstxt.org format meant for lower-priority links. + Section: isNested ? ToSectionTitle(segments[0]) : null, + SectionOrder: isNested ? GetOrder(segments[0]) : int.MaxValue, + Order: GetOrder(segments[^1])); + } + + static string GetFirstHeading(string[] lines) + { + return lines + .Select(x => x.Trim()) + .FirstOrDefault(x => x.StartsWith("# ")) + ?[2..].Trim(); + } + + static int GetFrontmatterEnd(string[] lines) + { + if (lines.Length == 0 || lines[0].Trim() != "---") + return 0; + + var end = Array.FindIndex(lines, startIndex: 1, x => x.Trim() == "---"); + return end < 0 ? 0 : end + 1; + } + + static Dictionary ReadFrontmatter(string[] lines, int frontmatterEnd) + { + var entries = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 1; i < Math.Max(frontmatterEnd - 1, 1); i++) + { + var match = FrontmatterEntry.Match(lines[i]); + if (!match.Success) + continue; + + var value = match.Groups["value"].Value.Trim().TrimMatchingDoubleQuotes().Trim('\''); + if (!value.IsNullOrWhiteSpace()) + entries[match.Groups["key"].Value] = value; + } + + return entries; + } + + // Only introduction.md declares a 'description', so for the other 36 pages the summary falls + // back to the page's own opening paragraph. Several open with a Docusaurus import or an MDX + // component instead, and those are not prose, so they are skipped along with headings, + // admonitions, tables, images and code fences. + // + // The whole paragraph is collected, not just its first line: docs/website hard-wraps prose, so + // stopping at the first newline would cut a sentence mid-way ("... help other" on badge.md). + static string GetFirstProseParagraph(string[] lines, int frontmatterEnd) + { + var paragraph = new List(); + var insideFence = false; + + foreach (var line in lines.Skip(frontmatterEnd)) + { + var trimmed = line.Trim(); + + if (trimmed.StartsWith("```")) + { + insideFence = !insideFence; + continue; + } + + var isProse = !insideFence && + !trimmed.IsNullOrWhiteSpace() && + !trimmed.StartsWith("import ") && + !trimmed.StartsWith('<') && + !trimmed.StartsWith(":::") && + !trimmed.StartsWith('#') && + !trimmed.StartsWith('|') && + !trimmed.StartsWith('!'); + + if (isProse) + paragraph.Add(trimmed); + else if (paragraph.Count > 0) + break; + } + + return paragraph.Count > 0 ? paragraph.JoinSpace() : null; + } + + static string Summarize(string text) + { + if (text.IsNullOrWhiteSpace()) + return null; + + var flattened = InlineLink.Replace(text, "${text}").Trim(); + if (flattened.Length <= MaxDescriptionLength) + return flattened; + + // Cut on a word boundary so the summary never ends mid-word. + var cut = flattened.LastIndexOf(' ', MaxDescriptionLength); + return flattened[..(cut > 0 ? cut : MaxDescriptionLength)].TrimEnd(',', ';', ':', '.') + "..."; + } + + IReadOnlyList ReadDocPages() + { + return DocsWebsiteDirectory.GlobFiles("**/*.md") + .Select(ReadPage) + .OrderBy(x => x.SectionOrder) + .ThenBy(x => x.Order) + .ThenBy(x => x.Title) + .ToList(); + } + Target GenerateLlmsTxt => _ => _ .Executes(() => { - DocsWebsiteDirectory.GlobFiles("**/*.md") - .OrderBy(x => x.ToString()) - .ForEach(x => Log.Information( - "{Relative} -> {Url}", - DocsWebsiteDirectory.GetUnixRelativePathTo(x), - ToPublicUrl(x))); + ReadDocPages().ForEach(x => Log.Information( + "[{Section}] {Title} -> {Url} :: {Description}", + x.Section ?? "(root)", + x.Title, + x.Url, + x.Description ?? "(none)")); }); } From de39ca60e540d0c80825a21e9e332d98311499ab Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 27 Aug 2026 10:47:45 +0200 Subject: [PATCH 4/8] Generate an llms.txt index from the docs frontmatter Co-Authored-By: Claude Opus 5 (1M context) --- build/Build.Documentation.cs | 88 +++++++++++++++++++++++++++++++----- docs/llms.txt | 67 +++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 docs/llms.txt diff --git a/build/Build.Documentation.cs b/build/Build.Documentation.cs index 470c08948..e1b9de466 100644 --- a/build/Build.Documentation.cs +++ b/build/Build.Documentation.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Text.RegularExpressions; using Fallout.Common; using Fallout.Common.IO; @@ -65,14 +66,27 @@ string ToSectionTitle(string directorySlug) .JoinSpace(); } - sealed record DocPage(string Title, string Description, string Url, string Section, int SectionOrder, int Order); + /// + /// Whether the site places this page in its sidebar. Only meaningful for the pages at the root + /// of docs/website, which have no section to sort them: introduction.md declares a + /// 'sidebar_position' and badge.md does not, and that is exactly the split between a link that + /// belongs in the main body and one that belongs under "Optional". + /// + sealed record DocPage( + string Title, + string Description, + string Url, + string Section, + int SectionOrder, + int Order, + bool IsPrimary); const int MaxDescriptionLength = 200; // Inline markdown links render as "[text](url)". Only the text belongs in a one-line summary. static readonly Regex InlineLink = new(@"\[(?[^\]]+)\]\([^)]+\)", RegexOptions.Compiled); - static readonly Regex FrontmatterEntry = new(@"^(?[a-zA-Z]+):\s*(?.*)$", RegexOptions.Compiled); + static readonly Regex FrontmatterEntry = new(@"^(?[a-zA-Z_]+):\s*(?.*)$", RegexOptions.Compiled); DocPage ReadPage(AbsolutePath file) { @@ -102,11 +116,12 @@ DocPage ReadPage(AbsolutePath file) Title: title, Description: Summarize(description), Url: ToPublicUrl(file), - // Root-level pages have no section. Task 3 collects them under "Optional", which is the - // part of the llmstxt.org format meant for lower-priority links. + // Root-level pages have no section, so they are rendered either above the first one or + // under "Optional", depending on IsPrimary. Section: isNested ? ToSectionTitle(segments[0]) : null, SectionOrder: isNested ? GetOrder(segments[0]) : int.MaxValue, - Order: GetOrder(segments[^1])); + Order: GetOrder(segments[^1]), + IsPrimary: isNested || frontmatter.ContainsKey("sidebar_position")); } static string GetFirstHeading(string[] lines) @@ -207,14 +222,65 @@ IReadOnlyList ReadDocPages() .ToList(); } + AbsolutePath LlmsTxtFile => RootDirectory / "docs" / "llms.txt"; + + // https://llmstxt.org: an H1, an optional blockquote summary, then H2 sections of link lines. + // A list may also sit between the blockquote and the first H2, which is where the pages that + // live at the root of docs/website go when the site gives them a sidebar position. + string RenderLlmsTxt(IReadOnlyList pages) + { + var builder = new StringBuilder(); + builder.AppendLine("# Fallout"); + builder.AppendLine(); + + // Single source for the summary: introduction.md's own 'description', which is what the + // site serves as its meta description. Generating it from anywhere else would let the two + // drift apart. + var summary = pages.Single(x => x.Url == DocsBaseUrl + "introduction").Description; + builder.AppendLine($"> {summary}"); + builder.AppendLine(); + builder.AppendLine( + "Generated from the documentation sources by './build.ps1 GenerateLlmsTxt'. Do not edit by hand."); + builder.AppendLine(); + + foreach (var page in pages.Where(x => x.Section == null && x.IsPrimary)) + builder.AppendLine(RenderEntry(page)); + + foreach (var section in pages.Where(x => x.Section != null).GroupBy(x => x.Section)) + { + builder.AppendLine(); + builder.AppendLine($"## {section.Key}"); + builder.AppendLine(); + section.ForEach(x => builder.AppendLine(RenderEntry(x))); + } + + // llms.txt reserves "Optional" for links a consumer may skip when it needs a shorter + // context. Root pages the site does not place in the sidebar belong there. + var optional = pages.Where(x => x.Section == null && !x.IsPrimary).ToList(); + if (optional.Count > 0) + { + builder.AppendLine(); + builder.AppendLine("## Optional"); + builder.AppendLine(); + optional.ForEach(x => builder.AppendLine(RenderEntry(x))); + } + + return builder.ToString(); + } + + static string RenderEntry(DocPage page) + { + return page.Description.IsNullOrWhiteSpace() + ? $"- [{page.Title}]({page.Url})" + : $"- [{page.Title}]({page.Url}): {page.Description}"; + } + Target GenerateLlmsTxt => _ => _ .Executes(() => { - ReadDocPages().ForEach(x => Log.Information( - "[{Section}] {Title} -> {Url} :: {Description}", - x.Section ?? "(root)", - x.Title, - x.Url, - x.Description ?? "(none)")); + var pages = ReadDocPages(); + LlmsTxtFile.WriteAllText(RenderLlmsTxt(pages)); + + Log.Information("Wrote {File} with {Count} pages", RootDirectory.GetUnixRelativePathTo(LlmsTxtFile), pages.Count); }); } diff --git a/docs/llms.txt b/docs/llms.txt new file mode 100644 index 000000000..0b95d0221 --- /dev/null +++ b/docs/llms.txt @@ -0,0 +1,67 @@ +# Fallout + +> Fallout is a C#-first build automation framework for .NET — the hard-fork successor to NUKE. Write your CI/CD pipelines in plain C#, debug them locally, and share build steps across repositories. + +Generated from the documentation sources by './build.ps1 GenerateLlmsTxt'. Do not edit by hand. + +- [Introduction](https://docs.fallout.build/docs/introduction): Fallout is a C#-first build automation framework for .NET — the hard-fork successor to NUKE. Write your CI/CD pipelines in plain C#, debug them locally, and share build steps across repositories. + +## Getting Started + +- [Installation](https://docs.fallout.build/docs/getting-started/installation): Before you can set up a build project, you need to install Fallout's dedicated .NET global tool: +- [Build Setup](https://docs.fallout.build/docs/getting-started/setup): After installing the Fallout global tool, you can call it from anywhere on your machine to set up a new build: +- [Build Execution](https://docs.fallout.build/docs/getting-started/execution): After you've set up a build you can run it either through the global tool or one of the installed bootstrapping scripts: + +## Fundamentals + +- [Build Anatomy](https://docs.fallout.build/docs/fundamentals/builds): A build project is a regular .NET console application. However, unlike regular console applications, Fallout chooses to name the main class `Build` instead of `Program`. This establishes a convention... +- [Target Definitions](https://docs.fallout.build/docs/fundamentals/targets): Inside a `Build` class, you can define your build steps as `Target` properties. The implementation for a build step is provided as a lambda function through the `Executes` method: +- [Parameters](https://docs.fallout.build/docs/fundamentals/parameters): Another important aspect of build automation is the ability of passing input values to your build. These input values can be anything from generic texts, numeric and enum values, file and directory... +- [Logging](https://docs.fallout.build/docs/fundamentals/logging): As with any other application, good logging greatly reduces the time to detect the source of errors and fix them quickly. Fallout integrates with Serilog and prepares a console and file logger for... +- [Assertions](https://docs.fallout.build/docs/fundamentals/assertions): As in any other codebase, it is good practice to assert assumptions before continuing with more heavy procedures in your build automation. When an assertion is violated, it usually entails that the... + +## Common Tasks + +- [Constructing Paths](https://docs.fallout.build/docs/common/paths): Referencing files and directories seems like a trivial task. Nevertheless, developers often run into problems where relative paths no longer match the current working directory, or find themselves... +- [Repository Insights](https://docs.fallout.build/docs/common/repository): Having knowledge about the current branch, applied tags, and the repository origin is eminently important in various scenarios. For instance, the deployment destination for an application is different... +- [Data Serialization](https://docs.fallout.build/docs/common/serialization): Structured data plays an essential role in build automation. You may want to read a list of repositories to be checked out, write data that's consumed by another tool, or update version numbers of... +- [Versioning Artifacts](https://docs.fallout.build/docs/common/versioning): Whenever a build produces artifacts, those should be identifiable with a unique version number. This avoids making wrong expectations about available features or fixed bugs, and allows for clear... +- [Solution & Project Model](https://docs.fallout.build/docs/common/solution-project-model): Particularly when building .NET applications, your build may require information related to solution or project files. Such information is often duplicated with string literals and quickly becomes... +- [Executing CLI Tools](https://docs.fallout.build/docs/common/cli-tools): Interacting with third-party command-line interface tools (CLIs) is an essential task in build automation. This includes a wide range of aspects, such as resolution of the tool path, construction of... +- [Archive Compression](https://docs.fallout.build/docs/common/compression): In many situations you have to deal with compressed archives. Good examples are when you want to provide additional assets for your GitHub releases, or when you depend on other project's release... +- [Chats & Social Media](https://docs.fallout.build/docs/common/chats): As a final step of your build automation process, you may want to report errors or announce a new version through different chats and social media channels. Fallout comes with basic support for the... + +## Build Sharing + +- [Global Builds](https://docs.fallout.build/docs/sharing/global-builds): Instead of adding and maintaining build projects in all your repositories, you can also build them by convention using a global build. Global builds are based on the concept of .NET global tools and... +- [Build Components](https://docs.fallout.build/docs/sharing/build-components): With build components you can implement your build infrastructure once, and compose individual builds across different repositories. Central to the idea of build components are interface default... + +## CI/CD Support + +- [AppVeyor](https://docs.fallout.build/docs/cicd/appveyor): Running on AppVeyor will automatically enable custom theming for your build log output: +- [Azure Pipelines](https://docs.fallout.build/docs/cicd/azure-pipelines): Running on Azure Pipelines will automatically enable custom theming for your build log output including collapsible sections for better structuring: +- [Bitbucket](https://docs.fallout.build/docs/cicd/bitbucket): Running on Bitbucket will use the standard theming for your build log output. +- [GitHub Actions](https://docs.fallout.build/docs/cicd/github-actions): Running on GitHub Actions will automatically enable custom theming for your build log output including collapsible groups for better structuring: +- [GitLab](https://docs.fallout.build/docs/cicd/gitlab): Running on GitLab will automatically enable custom theming for your build log output including collapsible sections for better structuring: +- [Jenkins](https://docs.fallout.build/docs/cicd/jenkins): Running on Jenkins will use the standard theming for your build log output. +- [Space Automation](https://docs.fallout.build/docs/cicd/space-automation): Running on JetBrains Space will use the standard theming for your build log output: +- [TeamCity](https://docs.fallout.build/docs/cicd/teamcity): Running on TeamCity will automatically enable custom theming for your build log output including collapsible blocks for better structuring: + +## Global Tool + +- [Shell Completion](https://docs.fallout.build/docs/global-tool/shell-completion): Typing long target names or parameters can be tedious and error-prone. The global tool helps you to invoke commands more quickly and without any typos, similar to tab completion for the .NET CLI. +- [Adding NuGet Packages](https://docs.fallout.build/docs/global-tool/packages): In many cases, build automation relies on third-party tools. Fallout provides you with a great API for working with CLI tools, however, it is the responsibility of a build project to reference these... +- [Managing Secrets](https://docs.fallout.build/docs/global-tool/secrets): Historically, secret values like passwords or auth-tokens are often saved as environment variables on local machines or CI/CD servers. This imposes both, security issues because other processes can... +- [Navigation](https://docs.fallout.build/docs/global-tool/navigation): Over time, you might accumulate more and more projects that are built using Fallout. Some of these might even form a hierarchical structure, where one root directory contains several other root... +- [Converting from Cake](https://docs.fallout.build/docs/global-tool/cake): Over the years, the .NET community has come up with a lot of great build automation tools, including FAKE, Cake, FlubuCore, and BullsEye. When coming from Cake Scripting, the time for converting build... + +## IDE Support + +- [JetBrains Rider](https://docs.fallout.build/docs/ide/rider): In JetBrains Rider you can install the _NUKE Support plugin_ to be more productive in writing, running, and debugging your builds. +- [ReSharper](https://docs.fallout.build/docs/ide/resharper): In ReSharper you can install the _NUKE Support extension_ to be more productive in writing, running, and debugging your builds. +- [Visual Studio](https://docs.fallout.build/docs/ide/visual-studio): In Visual Studio you can install the _NUKE Support extension_ to be more productive in writing, running, and debugging your builds. +- [Visual Studio Code](https://docs.fallout.build/docs/ide/vscode): In Visual Studio Code you can install the _NUKE Support extension_ to be more productive in writing, running, and debugging your builds. + +## Optional + +- [Badge](https://docs.fallout.build/docs/badge): If you build with Fallout, link back with the badge. It is the quickest way to help other people find the project. From 86d510972cf444a95acd80e628432eb4f92c64a4 Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 27 Aug 2026 10:51:41 +0200 Subject: [PATCH 5/8] Fail the build when llms.txt is out of sync with the docs Wired into both workflows. build.yml ignores docs/**, so on its own the gate would never fire on the change that invalidates the file. build-skip.yml is the workflow that handles docs-only PRs, so it runs the target instead of echoing. Co-Authored-By: Claude Opus 5 (1M context) --- .fallout/build.schema.json | 3 ++- .github/workflows/build-skip.yml | 35 ++++++++++++++++++++++++++------ .github/workflows/build.yml | 4 ++-- build/Build.CI.GitHubActions.cs | 2 +- build/Build.Documentation.cs | 21 +++++++++++++++++++ 5 files changed, 55 insertions(+), 10 deletions(-) diff --git a/.fallout/build.schema.json b/.fallout/build.schema.json index c8c41563d..d1c9339d7 100644 --- a/.fallout/build.schema.json +++ b/.fallout/build.schema.json @@ -43,7 +43,8 @@ "Test", "UpdateContributors", "UpdateStargazers", - "VerifyGeneratedTools" + "VerifyGeneratedTools", + "VerifyLlmsTxt" ] }, "Verbosity": { diff --git a/.github/workflows/build-skip.yml b/.github/workflows/build-skip.yml index 58c333065..8babfd70e 100644 --- a/.github/workflows/build-skip.yml +++ b/.github/workflows/build-skip.yml @@ -9,10 +9,15 @@ # - Without a substitute, docs-only PRs sit BLOCKED waiting for a check that # never reports. # -# This workflow fires on the inverse path set (docs-only changes), runs nothing -# of substance, and reports success under the same `ubuntu-latest` status-check -# context — satisfying the protection rule without spending CI minutes on a real -# build. +# This workflow fires on the inverse path set (docs-only changes) and reports +# success under the same `ubuntu-latest` status-check context, satisfying the +# protection rule without spending CI minutes on a full build/test/pack. +# +# It is not a no-op any more. docs/llms.txt is generated from docs/website by +# `GenerateLlmsTxt`, so a docs-only PR is exactly the change that can leave it +# stale — and it is exactly the change build.yml ignores. VerifyLlmsTxt therefore +# runs here, which is the only workflow that sees these PRs. It builds the build +# project and regenerates one file; it does not run the test or pack targets. # # Keep the job name `ubuntu-latest` aligned with build.yml so both files produce # a status check named `ubuntu-latest`; the workflow `name:` mirrors build.yml's @@ -37,5 +42,23 @@ jobs: name: ubuntu-latest runs-on: ubuntu-latest steps: - - name: 'Skip: docs-only PR, no build needed' - run: echo "Docs-only change — ubuntu-latest validation skipped via .github/workflows/build-skip.yml." + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.head_ref }} + - name: 'Cache: .fallout/temp, ~/.nuget/packages' + uses: actions/cache@v6 + with: + path: | + .fallout/temp + ~/.nuget/packages + key: ${{ runner.os }}-${{ hashFiles('**/global.json', '**/*.csproj', '**/Directory.Packages.props') }} + - name: 'Setup: .NET SDK' + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + - name: 'Restore: dotnet tools' + run: dotnet tool restore + - name: 'Run: VerifyLlmsTxt' + run: dotnet fallout VerifyLlmsTxt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0463d22b0..a065d3899 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,5 +55,5 @@ jobs: global-json-file: global.json - name: 'Restore: dotnet tools' run: dotnet tool restore - - name: 'Run: VerifyGeneratedTools, Test, Pack' - run: dotnet fallout VerifyGeneratedTools Test Pack + - name: 'Run: VerifyGeneratedTools, VerifyLlmsTxt, Test, Pack' + run: dotnet fallout VerifyGeneratedTools VerifyLlmsTxt Test Pack diff --git a/build/Build.CI.GitHubActions.cs b/build/Build.CI.GitHubActions.cs index 4fb503a1e..42d40fbb3 100644 --- a/build/Build.CI.GitHubActions.cs +++ b/build/Build.CI.GitHubActions.cs @@ -44,7 +44,7 @@ // long-lived and protected; all require the ubuntu-latest check. OnPullRequestBranches = new[] { DevelopBranch, MainBranch, ReleaseBranchPattern, SupportBranchPattern }, OnPullRequestExcludePaths = new[] { "docs/**", ".assets/**", "**/*.md" }, - InvokedTargets = new[] { nameof(VerifyGeneratedTools), nameof(ITest.Test), nameof(IPack.Pack) }, + InvokedTargets = new[] { nameof(VerifyGeneratedTools), nameof(VerifyLlmsTxt), nameof(ITest.Test), nameof(IPack.Pack) }, PublishArtifacts = false)] [GitHubActions( "build-cross-platform", diff --git a/build/Build.Documentation.cs b/build/Build.Documentation.cs index e1b9de466..cd7d29b8e 100644 --- a/build/Build.Documentation.cs +++ b/build/Build.Documentation.cs @@ -8,6 +8,7 @@ using Fallout.Common.Utilities; using Fallout.Common.Utilities.Collections; using Serilog; +using static Fallout.Common.Tools.Git.GitTasks; partial class Build { @@ -283,4 +284,24 @@ static string RenderEntry(DocPage page) Log.Information("Wrote {File} with {Count} pages", RootDirectory.GetUnixRelativePathTo(LlmsTxtFile), pages.Count); }); + + // CI gate, in the shape of VerifyGeneratedTools: GenerateLlmsTxt only runs when a contributor + // remembers to invoke it, so a page added under docs/website without regenerating would merge + // with docs/llms.txt silently missing it. `Requires` is asserted for the whole scheduled plan + // before any target runs, so the "start clean" check below still fires before GenerateLlmsTxt + // regenerates anything; the explicit re-check afterward catches drift with a message pointing + // at the fix. + // + // Wired into BOTH workflows on purpose. build.yml ignores docs/**, so on its own it would never + // fire on the change that actually invalidates the file; build-skip.yml is the workflow that + // handles those PRs, and it runs this target for exactly that reason. + Target VerifyLlmsTxt => _ => _ + .Requires(() => GitHasCleanWorkingCopy()) + .DependsOn(GenerateLlmsTxt) + .Executes(() => + { + Assert.True( + GitHasCleanWorkingCopy(), + "docs/llms.txt is out of sync with docs/website. Run './build.ps1 GenerateLlmsTxt' locally and commit the result."); + }); } From 7383c59e9cf458911131e43aa9d4e852d50c79f8 Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 27 Aug 2026 11:10:37 +0200 Subject: [PATCH 6/8] Harden the llms.txt generator against docs it does not yet contain Code-review follow-ups. None changes the generated file today; each closes a way it could go wrong or crash on a page the docs do not have yet. - _category_.json without a 'label' crashed instead of falling back to the slug. GetPropertyValue throws when the property is absent, so the guard and the fallback below it were dead code. - Files and directories whose name starts with an underscore are skipped. Docusaurus excludes them from routing and docs/website/_snippets already exists, so indexing one emitted a URL the site never serves. - .mdx pages are indexed too. Docusaurus routes them, and the gate could not have caught their absence: it only compares the generator to itself. - 'sidebar_position' now orders pages inside a section, as it does on the site. 07-ide has no numeric prefixes and rider.md relies on it. - Titles are read after the frontmatter and outside code fences, so a page opening with a '# terminal-command' marker cannot take it as its link text. - Unclosed frontmatter fails instead of being shipped as a description, and a YAML block scalar no longer renders as a bare '>'. - A missing introduction.md now says so instead of throwing a bare InvalidOperationException from Single(). - Titles sort with StringComparer.Ordinal. The file is verified byte for byte, so culture-sensitive ordering could fail the gate with no real drift. - build-skip.yml gets build.yml's concurrency group now that it runs a build. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-skip.yml | 7 ++++ build/Build.Documentation.cs | 72 ++++++++++++++++++++++++++------ 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-skip.yml b/.github/workflows/build-skip.yml index 8babfd70e..ffd33ca48 100644 --- a/.github/workflows/build-skip.yml +++ b/.github/workflows/build-skip.yml @@ -25,6 +25,13 @@ name: build +# Mirrors build.yml. This job does real work now, so rapid pushes to a docs PR +# would otherwise stack concurrent builds, which is the cost this file exists to +# avoid in the first place. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: pull_request: branches: diff --git a/build/Build.Documentation.cs b/build/Build.Documentation.cs index cd7d29b8e..ee0de38a8 100644 --- a/build/Build.Documentation.cs +++ b/build/Build.Documentation.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; @@ -38,7 +39,7 @@ static int GetOrder(string segment) string ToPublicUrl(AbsolutePath page) { var relative = DocsWebsiteDirectory.GetUnixRelativePathTo(page).ToString(); - var slug = relative[..^".md".Length] + var slug = relative[..^Path.GetExtension(relative).Length] .Split('/') .Select(StripOrderPrefix) .JoinSlash(); @@ -56,13 +57,17 @@ string ToSectionTitle(string directorySlug) var category = DocsWebsiteDirectory / directorySlug / "_category_.json"; if (category.FileExists()) { - var label = category.ReadJsonObject().GetPropertyValue("label"); + // OrNull, not GetPropertyValue: that one throws when the property is absent, which + // would make the fallback below unreachable. A _category_.json may legitimately carry + // only "position" or "collapsed". + var label = category.ReadJsonObject().GetPropertyValueOrNull("label"); if (!label.IsNullOrWhiteSpace()) return label; } return StripOrderPrefix(directorySlug) .Split('-') + .Where(x => x.Length > 0) .Select(x => char.ToUpperInvariant(x[0]) + x[1..]) .JoinSpace(); } @@ -99,7 +104,7 @@ DocPage ReadPage(AbsolutePath file) // has a page that relies on it: badge.md carries no frontmatter at all and is served as // "Badge". Rejecting it would refuse a page the site renders correctly, so the fallback // matches Docusaurus. A page with neither still fails, because that leaves no link text. - var title = frontmatter.GetValueOrDefault("title") ?? GetFirstHeading(lines); + var title = frontmatter.GetValueOrDefault("title") ?? GetFirstHeading(lines, frontmatterEnd); Assert.NotNullOrWhiteSpace( title, $"{DocsWebsiteDirectory.GetUnixRelativePathTo(file)} has neither a 'title' in its " @@ -121,16 +126,30 @@ DocPage ReadPage(AbsolutePath file) // under "Optional", depending on IsPrimary. Section: isNested ? ToSectionTitle(segments[0]) : null, SectionOrder: isNested ? GetOrder(segments[0]) : int.MaxValue, - Order: GetOrder(segments[^1]), + // Docusaurus lets a page's own 'sidebar_position' override the numeric filename prefix, + // and docs/website uses it: 07-ide has no prefixes, and rider.md declares position 1 to + // sort first. Reading only the prefix would order that section by title instead. + Order: frontmatter.TryGetValue("sidebar_position", out var position) && int.TryParse(position, out var parsed) + ? parsed + : GetOrder(segments[^1]), IsPrimary: isNested || frontmatter.ContainsKey("sidebar_position")); } - static string GetFirstHeading(string[] lines) + // Starts after the frontmatter and ignores fenced code, because "# terminal-command" is used as + // a marker throughout this doc set and would otherwise become a page's link text. + static string GetFirstHeading(string[] lines, int frontmatterEnd) { - return lines - .Select(x => x.Trim()) - .FirstOrDefault(x => x.StartsWith("# ")) - ?[2..].Trim(); + var insideFence = false; + foreach (var line in lines.Skip(frontmatterEnd)) + { + var trimmed = line.Trim(); + if (trimmed.StartsWith("```")) + insideFence = !insideFence; + else if (!insideFence && trimmed.StartsWith("# ")) + return trimmed[2..].Trim(); + } + + return null; } static int GetFrontmatterEnd(string[] lines) @@ -139,7 +158,11 @@ static int GetFrontmatterEnd(string[] lines) return 0; var end = Array.FindIndex(lines, startIndex: 1, x => x.Trim() == "---"); - return end < 0 ? 0 : end + 1; + // An opened but unclosed block is malformed. Returning 0 would hand the delimiter and the + // key/value lines to the prose reader and ship them as a description, so fail instead: a + // wrong entry in a generated index is worse than a build that says what is wrong. + Assert.True(end >= 0, "Frontmatter is opened with '---' but never closed."); + return end + 1; } static Dictionary ReadFrontmatter(string[] lines, int frontmatterEnd) @@ -152,6 +175,13 @@ static Dictionary ReadFrontmatter(string[] lines, int frontmatte continue; var value = match.Groups["value"].Value.Trim().TrimMatchingDoubleQuotes().Trim('\''); + + // ">" and "|" open a YAML block scalar whose text sits on the following lines. This + // parser is line-based, so it would store the indicator itself and render + // "- [Title](url): >". Treat the key as absent and let the prose fallback handle it. + if (value is ">" or "|" or ">-" or "|-") + continue; + if (!value.IsNullOrWhiteSpace()) entries[match.Groups["key"].Value] = value; } @@ -213,13 +243,22 @@ static string Summarize(string text) return flattened[..(cut > 0 ? cut : MaxDescriptionLength)].TrimEnd(',', ';', ':', '.') + "..."; } + // Docusaurus routes both .md and .mdx, and excludes anything whose file or directory name + // starts with an underscore (**/_*.md, **/_*/**). docs/website/_snippets/ exists for exactly + // that reason, so indexing it would emit URLs the site never serves. IReadOnlyList ReadDocPages() { - return DocsWebsiteDirectory.GlobFiles("**/*.md") + return DocsWebsiteDirectory.GlobFiles("**/*.md", "**/*.mdx") + .Where(x => !DocsWebsiteDirectory.GetUnixRelativePathTo(x).ToString() + .Split('/') + .Any(segment => segment.StartsWith('_'))) .Select(ReadPage) .OrderBy(x => x.SectionOrder) .ThenBy(x => x.Order) - .ThenBy(x => x.Title) + // Ordinal, not the culture-sensitive default: docs/llms.txt is verified byte for byte, + // so a contributor on another culture must not regenerate a differently ordered file + // and trip VerifyLlmsTxt with no real drift. + .ThenBy(x => x.Title, StringComparer.Ordinal) .ToList(); } @@ -237,7 +276,14 @@ string RenderLlmsTxt(IReadOnlyList pages) // Single source for the summary: introduction.md's own 'description', which is what the // site serves as its meta description. Generating it from anywhere else would let the two // drift apart. - var summary = pages.Single(x => x.Url == DocsBaseUrl + "introduction").Description; + var introduction = pages.SingleOrDefault(x => x.Url == DocsBaseUrl + "introduction"); + Assert.NotNull( + introduction, + "No page resolves to " + DocsBaseUrl + "introduction, which is where the llms.txt summary " + + "comes from. If introduction.md was renamed, moved into a section or given a 'slug', " + + "point this lookup at its new location."); + + var summary = introduction.Description; builder.AppendLine($"> {summary}"); builder.AppendLine(); builder.AppendLine( From 79b65f5c72397219f4a95964199581685a87705e Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 3 Sep 2026 12:23:35 +0200 Subject: [PATCH 7/8] Give the docs-only build its own concurrency group Both this workflow and build.yml are `name: build`, and `${{ github.workflow }}` expands to the workflow name, so the two shared one concurrency group. A PR that touches code and a doc fires both, and `cancel-in-progress` then let whichever started second cancel the other. Use a literal group instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VyD69qYha8hBMuja7opyj2 --- .github/workflows/build-skip.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-skip.yml b/.github/workflows/build-skip.yml index ffd33ca48..371a570f1 100644 --- a/.github/workflows/build-skip.yml +++ b/.github/workflows/build-skip.yml @@ -28,8 +28,13 @@ name: build # Mirrors build.yml. This job does real work now, so rapid pushes to a docs PR # would otherwise stack concurrent builds, which is the cost this file exists to # avoid in the first place. +# +# The group is a literal, NOT ${{ github.workflow }}: that expands to the workflow +# `name:`, which is `build` in both this file and build.yml on purpose. A PR that +# touches code AND a doc fires both workflows, and a shared group plus +# cancel-in-progress would let whichever starts second cancel the other. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: build-skip-${{ github.ref }} cancel-in-progress: true on: From 4929f86a227408d40b95cd99821a32fcf8f2b05e Mon Sep 17 00:00:00 2001 From: Philippe Matray Date: Thu, 3 Sep 2026 12:23:42 +0200 Subject: [PATCH 8/8] Apply the review pass to the llms.txt generator - Deserialize the frontmatter with YamlExtensions.GetYaml instead of a hand-rolled line parser, so quoting, escaping and block scalars follow the YAML spec. Unmatched Docusaurus keys are ignored, so a page adding one of the many others it accepts (tags, slug, keywords) does not fail the build. - Skip list items and blockquotes when falling back to a page's lead paragraph. A page opening with either would otherwise be indexed as its description. - Drop the Math.Max clamp in the frontmatter loop. GetFrontmatterEnd returns 0 or at least 2, so it never changed the bound. - camelCase the private static readonly regexes, which .editorconfig asks for and the rest of the repo already does. - Wrap the one line over the 130-character limit. - Group the fields and put the targets ahead of their helpers, matching Build.CodeGeneration.cs. docs/llms.txt regenerates byte for byte identical, so none of this moves the generated index. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VyD69qYha8hBMuja7opyj2 --- build/Build.Documentation.cs | 278 ++++++++++++++++++----------------- 1 file changed, 146 insertions(+), 132 deletions(-) diff --git a/build/Build.Documentation.cs b/build/Build.Documentation.cs index ee0de38a8..07c2f658a 100644 --- a/build/Build.Documentation.cs +++ b/build/Build.Documentation.cs @@ -8,13 +8,18 @@ using Fallout.Common.IO; using Fallout.Common.Utilities; using Fallout.Common.Utilities.Collections; +using Fallout.Utilities.Text.Yaml; using Serilog; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; using static Fallout.Common.Tools.Git.GitTasks; partial class Build { AbsolutePath DocsWebsiteDirectory => RootDirectory / "docs" / "website"; + AbsolutePath LlmsTxtFile => RootDirectory / "docs" / "llms.txt"; + // The public site is built from docs/website by the separate Fallout-build/docs.fallout.build // Docusaurus repository, which serves the pages under a /docs/ route prefix. Verified against // https://docs.fallout.build/sitemap.xml, not against the README: the README's own links omit @@ -23,54 +28,22 @@ partial class Build // regenerates it. const string DocsBaseUrl = "https://docs.fallout.build/docs/"; + const int MaxDescriptionLength = 200; + // Docusaurus orders pages by a numeric prefix on the directory and file name, and strips that // prefix from the served URL. So 01-getting-started/01-installation.md is served at // /docs/getting-started/installation. - static readonly Regex OrderPrefix = new(@"^(?\d+)-", RegexOptions.Compiled); - - static string StripOrderPrefix(string segment) => OrderPrefix.Replace(segment, string.Empty); - - static int GetOrder(string segment) - { - var match = OrderPrefix.Match(segment); - return match.Success ? int.Parse(match.Groups["order"].Value) : int.MaxValue; - } - - string ToPublicUrl(AbsolutePath page) - { - var relative = DocsWebsiteDirectory.GetUnixRelativePathTo(page).ToString(); - var slug = relative[..^Path.GetExtension(relative).Length] - .Split('/') - .Select(StripOrderPrefix) - .JoinSlash(); + static readonly Regex orderPrefix = new(@"^(?\d+)-", RegexOptions.Compiled); - return DocsBaseUrl + slug; - } - - // Each section directory carries a Docusaurus _category_.json whose "label" is what the site's - // sidebar shows, so that is the authoritative section name. It matters: title-casing the slug - // instead would give "Cicd" and "Ide" where the site says "CI/CD Support" and "IDE Support", - // and "Common" where it says "Common Tasks". The slug is only a fallback for a directory that - // has no _category_.json. - string ToSectionTitle(string directorySlug) - { - var category = DocsWebsiteDirectory / directorySlug / "_category_.json"; - if (category.FileExists()) - { - // OrNull, not GetPropertyValue: that one throws when the property is absent, which - // would make the fallback below unreachable. A _category_.json may legitimately carry - // only "position" or "collapsed". - var label = category.ReadJsonObject().GetPropertyValueOrNull("label"); - if (!label.IsNullOrWhiteSpace()) - return label; - } + // Inline markdown links render as "[text](url)". Only the text belongs in a one-line summary. + static readonly Regex inlineLink = new(@"\[(?[^\]]+)\]\([^)]+\)", RegexOptions.Compiled); - return StripOrderPrefix(directorySlug) - .Split('-') - .Where(x => x.Length > 0) - .Select(x => char.ToUpperInvariant(x[0]) + x[1..]) - .JoinSpace(); - } + // Deserialized rather than hand-parsed, so quoting, escaping and block scalars follow the YAML + // spec instead of a line regex. Underscored, not the repo-default camelCase builder, because + // Docusaurus spells its keys 'sidebar_position'. + static readonly DeserializerBuilder frontmatterDeserializer = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties(); /// /// Whether the site places this page in its sidebar. Only meaningful for the pages at the root @@ -87,12 +60,65 @@ sealed record DocPage( int Order, bool IsPrimary); - const int MaxDescriptionLength = 200; + // Only the three keys this index reads. Docusaurus accepts many more (tags, slug, keywords, + // hide_title...), so unmatched ones are ignored rather than failing a page that adds one. + sealed record Frontmatter + { + public string Title { get; init; } - // Inline markdown links render as "[text](url)". Only the text belongs in a one-line summary. - static readonly Regex InlineLink = new(@"\[(?[^\]]+)\]\([^)]+\)", RegexOptions.Compiled); + public string Description { get; init; } - static readonly Regex FrontmatterEntry = new(@"^(?[a-zA-Z_]+):\s*(?.*)$", RegexOptions.Compiled); + public int? SidebarPosition { get; init; } + } + + Target GenerateLlmsTxt => _ => _ + .Executes(() => + { + var pages = ReadDocPages(); + LlmsTxtFile.WriteAllText(RenderLlmsTxt(pages)); + + Log.Information("Wrote {File} with {Count} pages", RootDirectory.GetUnixRelativePathTo(LlmsTxtFile), pages.Count); + }); + + // CI gate, in the shape of VerifyGeneratedTools: GenerateLlmsTxt only runs when a contributor + // remembers to invoke it, so a page added under docs/website without regenerating would merge + // with docs/llms.txt silently missing it. `Requires` is asserted for the whole scheduled plan + // before any target runs, so the "start clean" check below still fires before GenerateLlmsTxt + // regenerates anything; the explicit re-check afterward catches drift with a message pointing + // at the fix. + // + // Wired into BOTH workflows on purpose. build.yml ignores docs/**, so on its own it would never + // fire on the change that actually invalidates the file; build-skip.yml is the workflow that + // handles those PRs, and it runs this target for exactly that reason. + Target VerifyLlmsTxt => _ => _ + .Requires(() => GitHasCleanWorkingCopy()) + .DependsOn(GenerateLlmsTxt) + .Executes(() => + { + Assert.True( + GitHasCleanWorkingCopy(), + "docs/llms.txt is out of sync with docs/website. Run './build.ps1 GenerateLlmsTxt' " + + "locally and commit the result."); + }); + + // Docusaurus routes both .md and .mdx, and excludes anything whose file or directory name + // starts with an underscore (**/_*.md, **/_*/**). docs/website/_snippets/ exists for exactly + // that reason, so indexing it would emit URLs the site never serves. + IReadOnlyList ReadDocPages() + { + return DocsWebsiteDirectory.GlobFiles("**/*.md", "**/*.mdx") + .Where(x => !DocsWebsiteDirectory.GetUnixRelativePathTo(x).ToString() + .Split('/') + .Any(segment => segment.StartsWith('_'))) + .Select(ReadPage) + .OrderBy(x => x.SectionOrder) + .ThenBy(x => x.Order) + // Ordinal, not the culture-sensitive default: docs/llms.txt is verified byte for byte, + // so a contributor on another culture must not regenerate a differently ordered file + // and trip VerifyLlmsTxt with no real drift. + .ThenBy(x => x.Title, StringComparer.Ordinal) + .ToList(); + } DocPage ReadPage(AbsolutePath file) { @@ -104,15 +130,18 @@ DocPage ReadPage(AbsolutePath file) // has a page that relies on it: badge.md carries no frontmatter at all and is served as // "Badge". Rejecting it would refuse a page the site renders correctly, so the fallback // matches Docusaurus. A page with neither still fails, because that leaves no link text. - var title = frontmatter.GetValueOrDefault("title") ?? GetFirstHeading(lines, frontmatterEnd); + var title = frontmatter.Title.IsNullOrWhiteSpace() + ? GetFirstHeading(lines, frontmatterEnd) + : frontmatter.Title; Assert.NotNullOrWhiteSpace( title, $"{DocsWebsiteDirectory.GetUnixRelativePathTo(file)} has neither a 'title' in its " + "frontmatter nor a top-level heading. One of the two is needed: it is the link text " + "in docs/llms.txt."); - var description = frontmatter.GetValueOrDefault("description") - ?? GetFirstProseParagraph(lines, frontmatterEnd); + var description = frontmatter.Description.IsNullOrWhiteSpace() + ? GetFirstProseParagraph(lines, frontmatterEnd) + : frontmatter.Description; var relative = DocsWebsiteDirectory.GetUnixRelativePathTo(file).ToString(); var segments = relative.Split('/'); @@ -129,27 +158,44 @@ DocPage ReadPage(AbsolutePath file) // Docusaurus lets a page's own 'sidebar_position' override the numeric filename prefix, // and docs/website uses it: 07-ide has no prefixes, and rider.md declares position 1 to // sort first. Reading only the prefix would order that section by title instead. - Order: frontmatter.TryGetValue("sidebar_position", out var position) && int.TryParse(position, out var parsed) - ? parsed - : GetOrder(segments[^1]), - IsPrimary: isNested || frontmatter.ContainsKey("sidebar_position")); + Order: frontmatter.SidebarPosition ?? GetOrder(segments[^1]), + IsPrimary: isNested || frontmatter.SidebarPosition.HasValue); } - // Starts after the frontmatter and ignores fenced code, because "# terminal-command" is used as - // a marker throughout this doc set and would otherwise become a page's link text. - static string GetFirstHeading(string[] lines, int frontmatterEnd) + string ToPublicUrl(AbsolutePath page) { - var insideFence = false; - foreach (var line in lines.Skip(frontmatterEnd)) + var relative = DocsWebsiteDirectory.GetUnixRelativePathTo(page).ToString(); + var slug = relative[..^Path.GetExtension(relative).Length] + .Split('/') + .Select(StripOrderPrefix) + .JoinSlash(); + + return DocsBaseUrl + slug; + } + + // Each section directory carries a Docusaurus _category_.json whose "label" is what the site's + // sidebar shows, so that is the authoritative section name. It matters: title-casing the slug + // instead would give "Cicd" and "Ide" where the site says "CI/CD Support" and "IDE Support", + // and "Common" where it says "Common Tasks". The slug is only a fallback for a directory that + // has no _category_.json. + string ToSectionTitle(string directorySlug) + { + var category = DocsWebsiteDirectory / directorySlug / "_category_.json"; + if (category.FileExists()) { - var trimmed = line.Trim(); - if (trimmed.StartsWith("```")) - insideFence = !insideFence; - else if (!insideFence && trimmed.StartsWith("# ")) - return trimmed[2..].Trim(); + // OrNull, not GetPropertyValue: that one throws when the property is absent, which + // would make the fallback below unreachable. A _category_.json may legitimately carry + // only "position" or "collapsed". + var label = category.ReadJsonObject().GetPropertyValueOrNull("label"); + if (!label.IsNullOrWhiteSpace()) + return label; } - return null; + return StripOrderPrefix(directorySlug) + .Split('-') + .Where(x => x.Length > 0) + .Select(x => char.ToUpperInvariant(x[0]) + x[1..]) + .JoinSpace(); } static int GetFrontmatterEnd(string[] lines) @@ -165,28 +211,32 @@ static int GetFrontmatterEnd(string[] lines) return end + 1; } - static Dictionary ReadFrontmatter(string[] lines, int frontmatterEnd) + static Frontmatter ReadFrontmatter(string[] lines, int frontmatterEnd) { - var entries = new Dictionary(StringComparer.OrdinalIgnoreCase); - for (var i = 1; i < Math.Max(frontmatterEnd - 1, 1); i++) - { - var match = FrontmatterEntry.Match(lines[i]); - if (!match.Success) - continue; + // frontmatterEnd is one past the closing '---', so the block itself is lines 1..end-2. + // 0 means the page opens with no frontmatter at all; badge.md is one such page. + if (frontmatterEnd == 0) + return new Frontmatter(); - var value = match.Groups["value"].Value.Trim().TrimMatchingDoubleQuotes().Trim('\''); - - // ">" and "|" open a YAML block scalar whose text sits on the following lines. This - // parser is line-based, so it would store the indicator itself and render - // "- [Title](url): >". Treat the key as absent and let the prose fallback handle it. - if (value is ">" or "|" or ">-" or "|-") - continue; + var yaml = lines[1..(frontmatterEnd - 1)].JoinNewLine(); + return yaml.GetYaml(frontmatterDeserializer) ?? new Frontmatter(); + } - if (!value.IsNullOrWhiteSpace()) - entries[match.Groups["key"].Value] = value; + // Starts after the frontmatter and ignores fenced code, because "# terminal-command" is used as + // a marker throughout this doc set and would otherwise become a page's link text. + static string GetFirstHeading(string[] lines, int frontmatterEnd) + { + var insideFence = false; + foreach (var line in lines.Skip(frontmatterEnd)) + { + var trimmed = line.Trim(); + if (trimmed.StartsWith("```")) + insideFence = !insideFence; + else if (!insideFence && trimmed.StartsWith("# ")) + return trimmed[2..].Trim(); } - return entries; + return null; } // Only introduction.md declares a 'description', so for the other 36 pages the summary falls @@ -218,7 +268,13 @@ static string GetFirstProseParagraph(string[] lines, int frontmatterEnd) !trimmed.StartsWith(":::") && !trimmed.StartsWith('#') && !trimmed.StartsWith('|') && - !trimmed.StartsWith('!'); + !trimmed.StartsWith('!') && + // A page opening with a list or a quote has no lead paragraph to take. + // "- "/"* " and not '-'/'*', so a '---' rule or a '**bold**' lead-in + // is still read as the prose it is. + !trimmed.StartsWith("- ") && + !trimmed.StartsWith("* ") && + !trimmed.StartsWith('>'); if (isProse) paragraph.Add(trimmed); @@ -234,7 +290,7 @@ static string Summarize(string text) if (text.IsNullOrWhiteSpace()) return null; - var flattened = InlineLink.Replace(text, "${text}").Trim(); + var flattened = inlineLink.Replace(text, "${text}").Trim(); if (flattened.Length <= MaxDescriptionLength) return flattened; @@ -243,27 +299,6 @@ static string Summarize(string text) return flattened[..(cut > 0 ? cut : MaxDescriptionLength)].TrimEnd(',', ';', ':', '.') + "..."; } - // Docusaurus routes both .md and .mdx, and excludes anything whose file or directory name - // starts with an underscore (**/_*.md, **/_*/**). docs/website/_snippets/ exists for exactly - // that reason, so indexing it would emit URLs the site never serves. - IReadOnlyList ReadDocPages() - { - return DocsWebsiteDirectory.GlobFiles("**/*.md", "**/*.mdx") - .Where(x => !DocsWebsiteDirectory.GetUnixRelativePathTo(x).ToString() - .Split('/') - .Any(segment => segment.StartsWith('_'))) - .Select(ReadPage) - .OrderBy(x => x.SectionOrder) - .ThenBy(x => x.Order) - // Ordinal, not the culture-sensitive default: docs/llms.txt is verified byte for byte, - // so a contributor on another culture must not regenerate a differently ordered file - // and trip VerifyLlmsTxt with no real drift. - .ThenBy(x => x.Title, StringComparer.Ordinal) - .ToList(); - } - - AbsolutePath LlmsTxtFile => RootDirectory / "docs" / "llms.txt"; - // https://llmstxt.org: an H1, an optional blockquote summary, then H2 sections of link lines. // A list may also sit between the blockquote and the first H2, which is where the pages that // live at the root of docs/website go when the site gives them a sidebar position. @@ -322,32 +357,11 @@ static string RenderEntry(DocPage page) : $"- [{page.Title}]({page.Url}): {page.Description}"; } - Target GenerateLlmsTxt => _ => _ - .Executes(() => - { - var pages = ReadDocPages(); - LlmsTxtFile.WriteAllText(RenderLlmsTxt(pages)); + static string StripOrderPrefix(string segment) => orderPrefix.Replace(segment, string.Empty); - Log.Information("Wrote {File} with {Count} pages", RootDirectory.GetUnixRelativePathTo(LlmsTxtFile), pages.Count); - }); - - // CI gate, in the shape of VerifyGeneratedTools: GenerateLlmsTxt only runs when a contributor - // remembers to invoke it, so a page added under docs/website without regenerating would merge - // with docs/llms.txt silently missing it. `Requires` is asserted for the whole scheduled plan - // before any target runs, so the "start clean" check below still fires before GenerateLlmsTxt - // regenerates anything; the explicit re-check afterward catches drift with a message pointing - // at the fix. - // - // Wired into BOTH workflows on purpose. build.yml ignores docs/**, so on its own it would never - // fire on the change that actually invalidates the file; build-skip.yml is the workflow that - // handles those PRs, and it runs this target for exactly that reason. - Target VerifyLlmsTxt => _ => _ - .Requires(() => GitHasCleanWorkingCopy()) - .DependsOn(GenerateLlmsTxt) - .Executes(() => - { - Assert.True( - GitHasCleanWorkingCopy(), - "docs/llms.txt is out of sync with docs/website. Run './build.ps1 GenerateLlmsTxt' locally and commit the result."); - }); + static int GetOrder(string segment) + { + var match = orderPrefix.Match(segment); + return match.Success ? int.Parse(match.Groups["order"].Value) : int.MaxValue; + } }