From 3ae6798854e8b27237ea6febdda04458c3961b14 Mon Sep 17 00:00:00 2001 From: Dag Brattli Date: Sun, 21 Dec 2025 10:33:56 +0100 Subject: [PATCH] feat: add Fable.Literate test infrastructure and fix boilerplate filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add test project with Expecto (.NET) and Fable.Pyxpecto (Python) support - Fix isBoilerplate to keep modules with bodies (module Foo = ...) - Add trimCode utility to preserve indentation while removing empty lines - Add test commands to justfile (test, build-tests, test-python, test-all) - Regenerate docs with improved code visibility The boilerplate filter was too aggressive, removing all module declarations including those with bodies. Now it only filters standalone declarations (module Foo, namespace Bar) while keeping module definitions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .vscode/settings.json | 1 + .../Fable.Literate.Tests.fsproj | 27 +++ Fable.Literate.Tests/Program.fs | 26 +++ Fable.Literate.Tests/Tests.fs | 155 +++++++++++++++ Fable.Literate/App.fs | 22 ++- Fable.Literate/Python.fs | 3 + docs/async-programming.md | 32 +++- docs/fable-literate.md | 179 ++++++++++++++++++ docs/pydantic.md | 26 +++ justfile | 15 ++ 10 files changed, 482 insertions(+), 4 deletions(-) create mode 100644 Fable.Literate.Tests/Fable.Literate.Tests.fsproj create mode 100644 Fable.Literate.Tests/Program.fs create mode 100644 Fable.Literate.Tests/Tests.fs diff --git a/.vscode/settings.json b/.vscode/settings.json index eada51a..3ce5c40 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,6 +12,7 @@ "Hashnode", "pathlib", "pyname", + "Pyxpecto", "stroustrup" ] } \ No newline at end of file diff --git a/Fable.Literate.Tests/Fable.Literate.Tests.fsproj b/Fable.Literate.Tests/Fable.Literate.Tests.fsproj new file mode 100644 index 0000000..f3925d2 --- /dev/null +++ b/Fable.Literate.Tests/Fable.Literate.Tests.fsproj @@ -0,0 +1,27 @@ + + + + Exe + net8.0 + preview + false + $(DefineConstants);TESTING + + + + + + + + + + + + + + + + + + + diff --git a/Fable.Literate.Tests/Program.fs b/Fable.Literate.Tests/Program.fs new file mode 100644 index 0000000..fec2cd2 --- /dev/null +++ b/Fable.Literate.Tests/Program.fs @@ -0,0 +1,26 @@ +module Fable.Literate.Tests.Program + +#if FABLE_COMPILER +open Fable.Pyxpecto +#else +open Expecto +#endif + +open Fable.Literate.Tests.AppTests + +let allTests = + testList "All" [ + parserTests + transformTests + utilsTests + markdownPrinterTests + pipelineTests + ] + +[] +let main args = +#if FABLE_COMPILER + Pyxpecto.runTests [||] allTests +#else + runTestsWithCLIArgs [] args allTests +#endif diff --git a/Fable.Literate.Tests/Tests.fs b/Fable.Literate.Tests/Tests.fs new file mode 100644 index 0000000..09c3972 --- /dev/null +++ b/Fable.Literate.Tests/Tests.fs @@ -0,0 +1,155 @@ +module Fable.Literate.Tests.AppTests + +#if FABLE_COMPILER +open Fable.Pyxpecto +#else +open Expecto +#endif +open Fable.Literate.App + +/// Helper to parse a string into lines +let lines (s: string) = s.Split('\n') |> Array.toSeq + +let parserTests = + testList "Parser" [ + testCase "parses simple markdown block" <| fun _ -> + let input = lines """(** +Hello world +*)""" + let result = Parser.parse input + Expect.equal result [Markdown "Hello world"] "Should parse markdown block" + + testCase "parses single-line markdown" <| fun _ -> + let input = lines "(** Hello *)" + let result = Parser.parse input + Expect.equal result [Markdown "Hello"] "Should parse single-line markdown" + + testCase "parses F# code block" <| fun _ -> + let input = lines "let x = 42" + let result = Parser.parse input + Expect.equal result [FSharpCode ["let x = 42"]] "Should parse code block" + + testCase "parses hide directive" <| fun _ -> + let input = lines """(*** hide ***) +let secret = 42 +(** +Visible +*)""" + let result = Parser.parse input + Expect.equal result [ + Hidden ["let secret = 42"] + Markdown "Visible" + ] "Should parse hidden block followed by markdown" + + testCase "parses include-python directive" <| fun _ -> + let input = lines "(*** include-python: foo, bar ***)" + let result = Parser.parse input + Expect.equal result [IncludePython ["foo"; "bar"]] "Should parse include-python directive" + + testCase "parses module with body" <| fun _ -> + let input = lines """module Foo = + let x = 1""" + let result = Parser.parse input + Expect.equal result [FSharpCode ["module Foo ="; " let x = 1"]] "Should parse module with body" + ] + +let transformTests = + testList "Transform" [ + testCase "filterHidden removes Hidden blocks" <| fun _ -> + let doc = [ + Markdown "Before" + Hidden ["secret"] + Markdown "After" + ] + let result = Transform.filterHidden doc + Expect.equal result [Markdown "Before"; Markdown "After"] "Should remove hidden blocks" + + testCase "filterBoilerplate removes standalone module declaration" <| fun _ -> + let doc = [FSharpCode ["module Foo"]] + let result = Transform.filterBoilerplate doc + Expect.equal result [] "Should filter standalone module" + + testCase "filterBoilerplate keeps module with body" <| fun _ -> + let doc = [FSharpCode ["module Foo ="; " let x = 1"]] + let result = Transform.filterBoilerplate doc + Expect.equal result [FSharpCode ["module Foo ="; " let x = 1"]] "Should keep module with body" + + testCase "filterBoilerplate removes namespace declaration" <| fun _ -> + let doc = [FSharpCode ["namespace MyLib"]] + let result = Transform.filterBoilerplate doc + Expect.equal result [] "Should filter namespace" + + testCase "filterBoilerplate removes empty code blocks" <| fun _ -> + let doc = [FSharpCode [""]; FSharpCode [" "]] + let result = Transform.filterBoilerplate doc + Expect.equal result [] "Should filter empty code blocks" + + testCase "filterBoilerplate keeps regular code" <| fun _ -> + let doc = [FSharpCode ["let x = 42"]] + let result = Transform.filterBoilerplate doc + Expect.equal result [FSharpCode ["let x = 42"]] "Should keep regular code" + ] + +let utilsTests = + testList "Utils" [ + testCase "trimCode preserves indentation" <| fun _ -> + let input = " let x = 1\n let y = 2" + let result = Utils.trimCode input + Expect.equal result " let x = 1\n let y = 2" "Should preserve indentation" + + testCase "trimCode removes leading empty lines" <| fun _ -> + let input = "\n\n let x = 1" + let result = Utils.trimCode input + Expect.equal result " let x = 1" "Should remove leading empty lines" + + testCase "trimCode removes trailing whitespace" <| fun _ -> + let input = "let x = 1\n " + let result = Utils.trimCode input + Expect.equal result "let x = 1" "Should remove trailing whitespace" + ] + +let markdownPrinterTests = + testList "MarkdownPrinter" [ + testCase "printMarkdown renders F# code block" <| fun _ -> + let doc = [FSharpCode ["let x = 42"]] + let result = MarkdownPrinter.printMarkdown doc + Expect.stringContains result "```fsharp" "Should have fsharp fence" + Expect.stringContains result "let x = 42" "Should contain code" + ] + +let pipelineTests = + testList "Pipeline" [ + testCase "full pipeline processes simple document" <| fun _ -> + let input = lines """(** +# Hello + +Some text +*) + +let greeting = "world" + +(*** hide ***) +let secret = 42 + +(** +More text +*)""" + let result = Pipeline.standard None input + Expect.stringContains result "# Hello" "Should contain heading" + Expect.stringContains result "Some text" "Should contain text" + Expect.stringContains result "let greeting" "Should contain visible code" + Expect.isFalse (result.Contains "secret") "Should not contain hidden code" + Expect.stringContains result "More text" "Should contain text after hidden section" + + testCase "pipeline preserves indentation in code blocks" <| fun _ -> + let input = lines """(** +Text +*) + +module Foo = + let x = 1 + let y = 2""" + let result = Pipeline.standard None input + Expect.stringContains result " let x = 1" "Should preserve 4-space indent" + Expect.stringContains result " let y = 2" "Should preserve 4-space indent" + ] diff --git a/Fable.Literate/App.fs b/Fable.Literate/App.fs index 40b8bea..d65b882 100644 --- a/Fable.Literate/App.fs +++ b/Fable.Literate/App.fs @@ -39,6 +39,8 @@ The input syntax: *) (*** hide ***) +module Fable.Literate.App + open System open Fable.Core open Fable.Literate.Python @@ -147,6 +149,12 @@ module Utils = | "*)" -> MarkdownClose | _ -> Content + /// Trim empty lines from front, whitespace from end (preserving indentation). + let trimCode (code: string) : string = + code.TrimEnd().Split '\n' + |> Array.skipWhile String.IsNullOrWhiteSpace + |> String.concat "\n" + open Utils (** @@ -169,6 +177,7 @@ module Parser = Blocks: Block list // Accumulated blocks (in reverse) } + (*** hide ***) /// Flush current state to a block if non-empty. let private flushState (ctx: ParseContext) : ParseContext = match ctx.State with @@ -234,6 +243,8 @@ module Parser = | CollectingMarkdown _, (MarkdownOpen _ | MarkdownSingle _ | IncludePythonCmd _) -> ctx | (CollectingCode _ | Ready), MarkdownClose -> ctx + (** Parse lines into a document AST *) + /// Parse lines into a document AST. let parse (lines: string seq) : Document = let initial = { @@ -264,11 +275,14 @@ module Transform = | _ -> true) /// Check if code lines are empty or boilerplate-only. + /// Filters standalone module/namespace declarations (e.g., "module Foo" or "namespace Bar") + /// but keeps module definitions with bodies (e.g., "module Foo ="). let private isBoilerplate (lines: string list) : bool = let code = lines |> String.concat "\n" |> (fun s -> s.Trim()) String.IsNullOrWhiteSpace code - || boilerplatePrefixes |> List.exists (fun prefix -> code.Trim().StartsWith prefix) + || code.StartsWith "namespace " + || code.StartsWith "module " && not (code.Contains "=") /// Remove empty or boilerplate-only code blocks. let filterBoilerplate (doc: Document) : Document = @@ -277,6 +291,7 @@ module Transform = | FSharpCode lines when isBoilerplate lines -> false | _ -> true) + (*** hide ***) /// Resolve IncludePython blocks to actual Python code blocks. let resolvePythonIncludes (pythonContent: string option) (doc: Document) : Document = doc @@ -307,7 +322,7 @@ module MarkdownPrinter = match block with | Markdown content -> content + "\n" | FSharpCode lines -> - let code = lines |> String.concat "\n" |> (fun s -> s.Trim()) + let code = lines |> String.concat "\n" |> trimCode "\n```fsharp\n" + code + "\n```\n\n" | PythonCode content -> "\n```python\n" + content + "\n```\n\n" | IncludePython symbols -> @@ -320,6 +335,7 @@ module MarkdownPrinter = let printMarkdown (doc: Document) : string = doc |> List.map printBlock |> String.concat "" + (*** hide ***) /// Increases all markdown header levels by one (# becomes ##, etc.). /// Preserves headers inside fenced code blocks. let adjustHeaderLevels (markdown: string) : string = @@ -404,7 +420,9 @@ let getPositionalArgs (args: string[]) : string[] = /// Main entry point. Converts a literate F# file to Markdown. /// Use --increase-headers flag to bump all header levels by one. /// Use --python-file to enable include-python directives. +#if !TESTING [] +#endif let main (args: string[]) = let hasFlag flag = args |> Array.contains flag let pythonFilePath = getFlagValue "--python-file" args diff --git a/Fable.Literate/Python.fs b/Fable.Literate/Python.fs index 3448ecf..4bb3a0a 100644 --- a/Fable.Literate/Python.fs +++ b/Fable.Literate/Python.fs @@ -52,6 +52,8 @@ let symbolPatterns (symbol: string) = [ symbol + ": " "def " + symbol + "(" "def " + symbol + "[" + "async def " + symbol + "(" + "async def " + symbol + "[" "class " + symbol + "(" "class " + symbol + ":" "class " + symbol + "[" @@ -79,6 +81,7 @@ let isMultilineDefinition (line: string) : bool = trimmed.StartsWith "class " || trimmed.StartsWith "def " + || trimmed.StartsWith "async def " || trimmed.EndsWith "(" || trimmed.EndsWith "[" || trimmed.EndsWith "{" diff --git a/docs/async-programming.md b/docs/async-programming.md index f52774b..a8f1d3e 100644 --- a/docs/async-programming.md +++ b/docs/async-programming.md @@ -138,7 +138,22 @@ let processItemTask (item: string) = ``` This generates: - + +```python +async def process_item_task(item: str) -> str: + builder_0040: Any = task() + + def _arrow49( + __unit: None = None, item: Any = item + ) -> Callable[[FSharpRef[Any]], bool]: + def _arrow48(__unit: None = None) -> Callable[[FSharpRef[Any]], bool]: + return builder_0040.Return(item.upper()) + + return builder_0040.Bind(delay(int32(100)), _arrow48) + + return await builder_0040.Run(builder_0040.Delay(_arrow49)) +``` + Now frameworks like FastAPI can detect and handle these as proper async endpoints. ### Task vs Async: Key Differences @@ -231,7 +246,20 @@ let simpleTask () = ``` In Python, this generates: - + +```python +async def simple_task(__unit: None = None) -> int32: + builder_0040: Any = task() + + def _arrow60(__unit: None = None) -> Callable[[FSharpRef[Any]], bool]: + def _arrow59(__unit: None = None) -> Callable[[FSharpRef[Any]], bool]: + return builder_0040.Return(int32(42)) + + return builder_0040.Bind(delay(int32(500)), _arrow59) + + return await builder_0040.Run(builder_0040.Delay(_arrow60)) +``` + ### Running Tasks from F`#` To run a task and get its result in F#: diff --git a/docs/fable-literate.md b/docs/fable-literate.md index 7a3476f..69c472a 100644 --- a/docs/fable-literate.md +++ b/docs/fable-literate.md @@ -63,22 +63,201 @@ type Document = Block list Utility functions for naming conversion and line classification: +```fsharp +module Utils = + /// List of contributors to thank (Fable-style). + let contributors = [| + "@dbrattli" + "@alfonsogarciacaro" + "@ncave" + "@MangelMaxime" + "@claude-code 🤖" + |] + + /// Returns a random contributor from the list. + let randomContributor () : string = + let rnd = Random() + contributors.[rnd.Next(contributors.Length)] + + /// Converts camelCase to snake_case for the function part. + let private toSnakeCase (name: string) : string = + if name.Length > 0 && Char.IsLower(name.[0]) then + System.Text.RegularExpressions.Regex.Replace( + name, + "[a-z]?[A-Z]", + fun m -> + if m.Value.Length = 1 then + m.Value.ToLowerInvariant() + else + m.Value.Substring(0, 1) + "_" + m.Value.Substring(1, 1).ToLowerInvariant() + ) + else + name + + /// Converts F# symbol reference to Python naming. + /// - "Module.func" -> "Module_func" (Fable keeps camelCase for module functions) + /// - "func" -> "func" (with snake_case conversion for top-level) + let toPythonNaming (name: string) : string = + match name.Split('.') with + | [| moduleName; funcName |] -> moduleName + "_" + funcName // Module functions stay camelCase + | _ -> toSnakeCase name // Top-level functions get snake_case + + /// Parses a comma-separated list of symbols from an include-python directive. + let parseSymbolList (directive: string) : string list = + // Extract content between "(*** include-python:" and "***)" + let start = "(*** include-python:".Length + let endPos = directive.LastIndexOf("***)") + + if endPos > start then + directive.Substring(start, endPos - start).Trim() + |> fun s -> s.Split(',') + |> Array.map (fun s -> s.Trim()) + |> Array.filter (fun s -> s.Length > 0) + |> Array.toList + else + [] + + /// Active pattern for classifying source lines. + /// - `HideCmd`: The (*** hide ***) directive + /// - `IncludePythonCmd symbols`: The (*** include-python: sym1, sym2 ***) directive + /// - `MarkdownSingle content`: Single-line markdown (** content *) + /// - `MarkdownOpen content`: Start of markdown block, possibly with content + /// - `MarkdownClose`: End of markdown block *) + /// - `Content`: Any other line + let (|HideCmd|IncludePythonCmd|MarkdownSingle|MarkdownOpen|MarkdownClose|Content|) (line: string) = + let trimmed = line.Trim() + + match trimmed with + | "(*** hide ***)" -> HideCmd + | s when s.StartsWith("(*** include-python:") && s.EndsWith("***)") -> IncludePythonCmd(parseSymbolList s) + | s when s.StartsWith("(**") && s.EndsWith("*)") && s.Length > 5 -> + MarkdownSingle(s.Substring(3, s.Length - 5).Trim()) + | s when s.StartsWith("(**") -> + let content = if s.Length > 3 then s.Substring(3).Trim() else "" + MarkdownOpen content + | "*)" -> MarkdownClose + | _ -> Content + +open Utils +``` + ## Parser Module The parser converts source lines into a Block AST using a fold: +```fsharp +module Parser = + /// Internal state for block accumulation during parsing. + type private ParserState = + | CollectingMarkdown of lines: string list + | CollectingCode of lines: string list + | CollectingHidden of lines: string list + | Ready + + /// Parse context threaded through the fold. + type private ParseContext = { + State: ParserState + Blocks: Block list // Accumulated blocks (in reverse) + } +``` + +Parse lines into a document AST + +```fsharp + /// Parse lines into a document AST. + let parse (lines: string seq) : Document = + let initial = { + State = Ready + Blocks = [] + } + + lines + |> Seq.fold parseLine initial + |> flushState + |> fun ctx -> List.rev ctx.Blocks +``` + ## Transform Module Pure transformations on the document AST: +```fsharp +module Transform = + /// Boilerplate prefixes that should be excluded from code blocks. + let boilerplatePrefixes = [ "module "; "namespace " ] + + /// Remove Hidden blocks from the document. + let filterHidden (doc: Document) : Document = + doc + |> List.filter (function + | Hidden _ -> false + | _ -> true) + + /// Check if code lines are empty or boilerplate-only. + /// Filters standalone module/namespace declarations (e.g., "module Foo" or "namespace Bar") + /// but keeps module definitions with bodies (e.g., "module Foo ="). + let private isBoilerplate (lines: string list) : bool = + let code = lines |> String.concat "\n" |> (fun s -> s.Trim()) + + String.IsNullOrWhiteSpace code + || code.StartsWith "namespace " + || code.StartsWith "module " && not (code.Contains "=") + + /// Remove empty or boilerplate-only code blocks. + let filterBoilerplate (doc: Document) : Document = + doc + |> List.filter (function + | FSharpCode lines when isBoilerplate lines -> false + | _ -> true) +``` + ## MarkdownPrinter Module Renders the document AST to markdown: +```fsharp +module MarkdownPrinter = + /// Trim empty lines from front, whitespace from end (preserving indentation). + let private trimCode (code: string) : string = + code.TrimEnd().Split '\n' + |> Array.skipWhile String.IsNullOrWhiteSpace + |> String.concat "\n" + + /// Render a single block to markdown. + let private printBlock (block: Block) : string = + match block with + | Markdown content -> content + "\n" + | FSharpCode lines -> + let code = lines |> String.concat "\n" |> trimCode + "\n```fsharp\n" + code + "\n```\n\n" + | PythonCode content -> "\n```python\n" + content + "\n```\n\n" + | IncludePython symbols -> + // Unresolved - should have been transformed + let symbolList = String.concat ", " symbols + "\n\n" + | Hidden _ -> "" // Should have been filtered + + /// Render a document to markdown string. + let printMarkdown (doc: Document) : string = + doc |> List.map printBlock |> String.concat "" +``` + ## Pipeline Module Composes the phases into a complete pipeline: +```fsharp +module Pipeline = + /// Standard processing pipeline. + let standard (pythonContent: string option) (lines: string seq) : string = + lines + |> Parser.parse + |> Transform.filterHidden + |> Transform.filterBoilerplate + |> Transform.resolvePythonIncludes pythonContent + |> MarkdownPrinter.printMarkdown +``` + ## Including Generated Python Code One of Fable.Literate's unique features is the ability to show the generated Python diff --git a/docs/pydantic.md b/docs/pydantic.md index 5707d65..624cc18 100644 --- a/docs/pydantic.md +++ b/docs/pydantic.md @@ -271,6 +271,32 @@ type UserDTO() = Explicit transformation between domain and DTO: +```fsharp +module UserMapping = + let toDTO (user: DomainUser) : UserDTO = + let dto = UserDTO() + dto.Id <- match user.Id with UserId guid -> string guid + dto.Name <- user.Name + dto.Age <- int user.Age + dto.BalanceAmount <- float user.Balance.Amount + dto.BalanceCurrency <- user.Balance.Currency + dto + + let fromDTO (dto: UserDTO) : Result = + try + Ok { + Id = UserId (System.Guid.Parse dto.Id) + Name = dto.Name + Age = int32 dto.Age + Balance = { + Amount = decimal dto.BalanceAmount + Currency = dto.BalanceCurrency + } + } + with ex -> + Error ex.Message +``` + ### Why This Pattern? The "boilerplate" of separate DTO types is actually valuable: diff --git a/justfile b/justfile index 5400280..a7e2379 100644 --- a/justfile +++ b/justfile @@ -123,6 +123,21 @@ lint-markdown: # TODO: Re-enable lint-python once Fable code generation issues are fixed lint: lint-markdown +# Run tests (.NET) +test: + dotnet run --project Fable.Literate.Tests/Fable.Literate.Tests.fsproj + +# Build tests to Python +build-tests: + dotnet fable Fable.Literate.Tests/ --lang python --outDir output/Fable.Literate.Tests/ + +# Run tests (Python) +test-python: build-tests + uv run python output/Fable.Literate.Tests/program.py + +# Run all tests (.NET and Python) +test-all: test test-python + # Full build: restore, build, generate docs, format, lint all: restore build generate format lint @echo "Build complete!"