Merge pull request #6 from cardamomcode/add-fable-literate-tests

Add Fable.Literate test infrastructure and fix boilerplate filtering

Author: dbrattli

.vscode/settings.json

@@ -12,6 +12,7 @@
         "Hashnode",
         "pathlib",
         "pyname",
+        "Pyxpecto",
         "stroustrup"
     ]
 }

Fable.Literate.Tests/Fable.Literate.Tests.fsproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <LangVersion>preview</LangVersion>
+    <GenerateProgramFile>false</GenerateProgramFile>
+    <DefineConstants>$(DefineConstants);TESTING</DefineConstants>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Expecto" Version="10.2.1" />
+    <PackageReference Include="Fable.Pyxpecto" Version="2.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="YoloDev.Expecto.TestSdk" Version="0.14.3" />
+    <PackageReference Include="Fable.Core" Version="5.0.0-beta.4" />
+    <PackageReference Include="Fable.Python" Version="5.0.0-alpha.21.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="../Fable.Literate/Python.fs" />
+    <Compile Include="../Fable.Literate/App.fs" />
+    <Compile Include="Tests.fs" />
+    <Compile Include="Program.fs" />
+  </ItemGroup>
+
+</Project>

Fable.Literate.Tests/Program.fs

@@ -0,0 +1,26 @@
+module Fable.Literate.Tests.Program
+
+#if FABLE_COMPILER
+open Fable.Pyxpecto
+#else
+open Expecto
+#endif
+
+open Fable.Literate.Tests.AppTests
+
+let allTests =
+    testList "All" [
+        parserTests
+        transformTests
+        utilsTests
+        markdownPrinterTests
+        pipelineTests
+    ]
+
+[<EntryPoint>]
+let main args =
+#if FABLE_COMPILER
+    Pyxpecto.runTests [||] allTests
+#else
+    runTestsWithCLIArgs [] args allTests
+#endif

Fable.Literate.Tests/Tests.fs

@@ -0,0 +1,155 @@
+module Fable.Literate.Tests.AppTests
+
+#if FABLE_COMPILER
+open Fable.Pyxpecto
+#else
+open Expecto
+#endif
+open Fable.Literate.App
+
+/// Helper to parse a string into lines
+let lines (s: string) = s.Split('\n') |> Array.toSeq
+
+let parserTests =
+    testList "Parser" [
+        testCase "parses simple markdown block" <| fun _ ->
+            let input = lines """(**
+Hello world
+*)"""
+            let result = Parser.parse input
+            Expect.equal result [Markdown "Hello world"] "Should parse markdown block"
+
+        testCase "parses single-line markdown" <| fun _ ->
+            let input = lines "(** Hello *)"
+            let result = Parser.parse input
+            Expect.equal result [Markdown "Hello"] "Should parse single-line markdown"
+
+        testCase "parses F# code block" <| fun _ ->
+            let input = lines "let x = 42"
+            let result = Parser.parse input
+            Expect.equal result [FSharpCode ["let x = 42"]] "Should parse code block"
+
+        testCase "parses hide directive" <| fun _ ->
+            let input = lines """(*** hide ***)
+let secret = 42
+(**
+Visible
+*)"""
+            let result = Parser.parse input
+            Expect.equal result [
+                Hidden ["let secret = 42"]
+                Markdown "Visible"
+            ] "Should parse hidden block followed by markdown"
+
+        testCase "parses include-python directive" <| fun _ ->
+            let input = lines "(*** include-python: foo, bar ***)"
+            let result = Parser.parse input
+            Expect.equal result [IncludePython ["foo"; "bar"]] "Should parse include-python directive"
+
+        testCase "parses module with body" <| fun _ ->
+            let input = lines """module Foo =
+    let x = 1"""
+            let result = Parser.parse input
+            Expect.equal result [FSharpCode ["module Foo ="; "    let x = 1"]] "Should parse module with body"
+    ]
+
+let transformTests =
+    testList "Transform" [
+        testCase "filterHidden removes Hidden blocks" <| fun _ ->
+            let doc = [
+                Markdown "Before"
+                Hidden ["secret"]
+                Markdown "After"
+            ]
+            let result = Transform.filterHidden doc
+            Expect.equal result [Markdown "Before"; Markdown "After"] "Should remove hidden blocks"
+
+        testCase "filterBoilerplate removes standalone module declaration" <| fun _ ->
+            let doc = [FSharpCode ["module Foo"]]
+            let result = Transform.filterBoilerplate doc
+            Expect.equal result [] "Should filter standalone module"
+
+        testCase "filterBoilerplate keeps module with body" <| fun _ ->
+            let doc = [FSharpCode ["module Foo ="; "    let x = 1"]]
+            let result = Transform.filterBoilerplate doc
+            Expect.equal result [FSharpCode ["module Foo ="; "    let x = 1"]] "Should keep module with body"
+
+        testCase "filterBoilerplate removes namespace declaration" <| fun _ ->
+            let doc = [FSharpCode ["namespace MyLib"]]
+            let result = Transform.filterBoilerplate doc
+            Expect.equal result [] "Should filter namespace"
+
+        testCase "filterBoilerplate removes empty code blocks" <| fun _ ->
+            let doc = [FSharpCode [""]; FSharpCode ["   "]]
+            let result = Transform.filterBoilerplate doc
+            Expect.equal result [] "Should filter empty code blocks"
+
+        testCase "filterBoilerplate keeps regular code" <| fun _ ->
+            let doc = [FSharpCode ["let x = 42"]]
+            let result = Transform.filterBoilerplate doc
+            Expect.equal result [FSharpCode ["let x = 42"]] "Should keep regular code"
+    ]
+
+let utilsTests =
+    testList "Utils" [
+        testCase "trimCode preserves indentation" <| fun _ ->
+            let input = "    let x = 1\n        let y = 2"
+            let result = Utils.trimCode input
+            Expect.equal result "    let x = 1\n        let y = 2" "Should preserve indentation"
+
+        testCase "trimCode removes leading empty lines" <| fun _ ->
+            let input = "\n\n    let x = 1"
+            let result = Utils.trimCode input
+            Expect.equal result "    let x = 1" "Should remove leading empty lines"
+
+        testCase "trimCode removes trailing whitespace" <| fun _ ->
+            let input = "let x = 1\n   "
+            let result = Utils.trimCode input
+            Expect.equal result "let x = 1" "Should remove trailing whitespace"
+    ]
+
+let markdownPrinterTests =
+    testList "MarkdownPrinter" [
+        testCase "printMarkdown renders F# code block" <| fun _ ->
+            let doc = [FSharpCode ["let x = 42"]]
+            let result = MarkdownPrinter.printMarkdown doc
+            Expect.stringContains result "```fsharp" "Should have fsharp fence"
+            Expect.stringContains result "let x = 42" "Should contain code"
+    ]
+
+let pipelineTests =
+    testList "Pipeline" [
+        testCase "full pipeline processes simple document" <| fun _ ->
+            let input = lines """(**
+# Hello
+
+Some text
+*)
+
+let greeting = "world"
+
+(*** hide ***)
+let secret = 42
+
+(**
+More text
+*)"""
+            let result = Pipeline.standard None input
+            Expect.stringContains result "# Hello" "Should contain heading"
+            Expect.stringContains result "Some text" "Should contain text"
+            Expect.stringContains result "let greeting" "Should contain visible code"
+            Expect.isFalse (result.Contains "secret") "Should not contain hidden code"
+            Expect.stringContains result "More text" "Should contain text after hidden section"
+
+        testCase "pipeline preserves indentation in code blocks" <| fun _ ->
+            let input = lines """(**
+Text
+*)
+
+module Foo =
+    let x = 1
+    let y = 2"""
+            let result = Pipeline.standard None input
+            Expect.stringContains result "    let x = 1" "Should preserve 4-space indent"
+            Expect.stringContains result "    let y = 2" "Should preserve 4-space indent"
+    ]

Fable.Literate/App.fs

@@ -39,6 +39,8 @@ The input syntax:
 *)
 
 (*** hide ***)
+module Fable.Literate.App
+
 open System
 open Fable.Core
 open Fable.Literate.Python
@@ -147,6 +149,12 @@ module Utils =
         | "*)" -> MarkdownClose
         | _ -> Content
 
+    /// Trim empty lines from front, whitespace from end (preserving indentation).
+    let trimCode (code: string) : string =
+        code.TrimEnd().Split '\n'
+        |> Array.skipWhile String.IsNullOrWhiteSpace
+        |> String.concat "\n"
+
 open Utils
 
 (**
@@ -169,6 +177,7 @@ module Parser =
         Blocks: Block list // Accumulated blocks (in reverse)
     }
 
+    (*** hide ***)
     /// Flush current state to a block if non-empty.
     let private flushState (ctx: ParseContext) : ParseContext =
         match ctx.State with
@@ -234,6 +243,8 @@ module Parser =
         | CollectingMarkdown _, (MarkdownOpen _ | MarkdownSingle _ | IncludePythonCmd _) -> ctx
         | (CollectingCode _ | Ready), MarkdownClose -> ctx
 
+    (** Parse lines into a document AST *)
+
     /// Parse lines into a document AST.
     let parse (lines: string seq) : Document =
         let initial = {
@@ -264,11 +275,14 @@ module Transform =
             | _ -> true)
 
     /// Check if code lines are empty or boilerplate-only.
+    /// Filters standalone module/namespace declarations (e.g., "module Foo" or "namespace Bar")
+    /// but keeps module definitions with bodies (e.g., "module Foo =").
     let private isBoilerplate (lines: string list) : bool =
         let code = lines |> String.concat "\n" |> (fun s -> s.Trim())
 
         String.IsNullOrWhiteSpace code
-        || boilerplatePrefixes |> List.exists (fun prefix -> code.Trim().StartsWith prefix)
+        || code.StartsWith "namespace "
+        || code.StartsWith "module " && not (code.Contains "=")
 
     /// Remove empty or boilerplate-only code blocks.
     let filterBoilerplate (doc: Document) : Document =
@@ -277,6 +291,7 @@ module Transform =
             | FSharpCode lines when isBoilerplate lines -> false
             | _ -> true)
 
+    (*** hide ***)
     /// Resolve IncludePython blocks to actual Python code blocks.
     let resolvePythonIncludes (pythonContent: string option) (doc: Document) : Document =
         doc
@@ -307,7 +322,7 @@ module MarkdownPrinter =
         match block with
         | Markdown content -> content + "\n"
         | FSharpCode lines ->
-            let code = lines |> String.concat "\n" |> (fun s -> s.Trim())
+            let code = lines |> String.concat "\n" |> trimCode
             "\n```fsharp\n" + code + "\n```\n\n"
         | PythonCode content -> "\n```python\n" + content + "\n```\n\n"
         | IncludePython symbols ->
@@ -320,6 +335,7 @@ module MarkdownPrinter =
     let printMarkdown (doc: Document) : string =
         doc |> List.map printBlock |> String.concat ""
 
+    (*** hide ***)
     /// Increases all markdown header levels by one (# becomes ##, etc.).
     /// Preserves headers inside fenced code blocks.
     let adjustHeaderLevels (markdown: string) : string =
@@ -404,7 +420,9 @@ let getPositionalArgs (args: string[]) : string[] =
 /// Main entry point. Converts a literate F# file to Markdown.
 /// Use --increase-headers flag to bump all header levels by one.
 /// Use --python-file <path> to enable include-python directives.
+#if !TESTING
 [<EntryPoint>]
+#endif
 let main (args: string[]) =
     let hasFlag flag = args |> Array.contains flag
     let pythonFilePath = getFlagValue "--python-file" args

Fable.Literate/Python.fs

@@ -52,6 +52,8 @@ let symbolPatterns (symbol: string) = [
     symbol + ": "
     "def " + symbol + "("
     "def " + symbol + "["
+    "async def " + symbol + "("
+    "async def " + symbol + "["
     "class " + symbol + "("
     "class " + symbol + ":"
     "class " + symbol + "["
@@ -79,6 +81,7 @@ let isMultilineDefinition (line: string) : bool =
 
     trimmed.StartsWith "class "
     || trimmed.StartsWith "def "
+    || trimmed.StartsWith "async def "
     || trimmed.EndsWith "("
     || trimmed.EndsWith "["
     || trimmed.EndsWith "{"

docs/async-programming.md

@@ -138,7 +138,22 @@ let processItemTask (item: string) =
 ```
 
 This generates:
-<!-- include-python: processItemTask (not found) -->
+
+```python
+async def process_item_task(item: str) -> str:
+    builder_0040: Any = task()
+
+    def _arrow49(
+        __unit: None = None, item: Any = item
+    ) -> Callable[[FSharpRef[Any]], bool]:
+        def _arrow48(__unit: None = None) -> Callable[[FSharpRef[Any]], bool]:
+            return builder_0040.Return(item.upper())
+
+        return builder_0040.Bind(delay(int32(100)), _arrow48)
+
+    return await builder_0040.Run(builder_0040.Delay(_arrow49))
+```
+
 Now frameworks like FastAPI can detect and handle these as proper async endpoints.
 
 ### Task vs Async: Key Differences
@@ -231,7 +246,20 @@ let simpleTask () =
 ```
 
 In Python, this generates:
-<!-- include-python: simpleTask (not found) -->
+
+```python
+async def simple_task(__unit: None = None) -> int32:
+    builder_0040: Any = task()
+
+    def _arrow60(__unit: None = None) -> Callable[[FSharpRef[Any]], bool]:
+        def _arrow59(__unit: None = None) -> Callable[[FSharpRef[Any]], bool]:
+            return builder_0040.Return(int32(42))
+
+        return builder_0040.Bind(delay(int32(500)), _arrow59)
+
+    return await builder_0040.Run(builder_0040.Delay(_arrow60))
+```
+
 ### Running Tasks from F`#`
 
 To run a task and get its result in F#: