diff --git a/README.md b/README.md index 6de2ed3..5665b6e 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,77 @@ files, since `.go` matches `go`). However, this will fail with other files like [embedmd]:# (file.ext) ``` +## Options + +Options can be added after the regular expressions in an `embedmd` command. + +### Excluding delimiter lines + +Prefix a regexp with `!` to use it as a boundary but exclude the matching line +from the output. This is useful when you use comments in your code to mark +snippet boundaries: + +```go +// snippet-start +fmt.Println("hello") +// snippet-end +``` + +```Markdown +[embedmd]:# (example.go go !/\/\/ snippet-start/ !/\/\/ snippet-end/) +``` + +This embeds only `fmt.Println("hello")`, excluding the marker comments. + +You can also exclude just the start or just the end: + +```Markdown +[embedmd]:# (example.go go !/\/\/ start/ /end/) +``` + +### Stripping indentation + +Code inside functions is typically indented. The `dedent` option removes the +common leading whitespace from all lines, so your Markdown code blocks are not +unnecessarily indented: + +```Markdown +[embedmd]:# (example.go go !/\/\/ snippet-start/ !/\/\/ snippet-end/ dedent) +``` + +### Trimming trailing blank lines + +Go's `gofmt` inserts a blank line between code and a following comment. The +`trim` option removes trailing blank lines from the extracted content: + +```Markdown +[embedmd]:# (example.go go !/\/\/ snippet-start/ !/\/\/ snippet-end/ trim) +``` + +### Text substitution + +The `s/old/new/` option replaces all occurrences of `old` with `new` in the +extracted content. This lets you use placeholder tokens in compilable code: + +```Markdown +[embedmd]:# (example.go go /func demo/ /}/ s/ELLIPSIS/.../) +``` + +Multiple substitutions can be chained: + +```Markdown +[embedmd]:# (example.go go /func/ /}/ s/ELLIPSIS/.../ s/_ = ELLIPSIS/.../) +``` + +### Combining options + +Options can be combined. They are applied in this order: line exclusion, trailing +blank line trimming, dedentation, then text substitution. + +```Markdown +[embedmd]:# (example.go go !/\/\/ snippet-start/ !/\/\/ snippet-end/ dedent trim s/ELLIPSIS/.../) +``` + ## Installation > You can install Go by following [these instructions](https://golang.org/doc/install). diff --git a/embedmd/command.go b/embedmd/command.go index 35ff28e..d915efb 100644 --- a/embedmd/command.go +++ b/embedmd/command.go @@ -15,13 +15,23 @@ package embedmd import ( "errors" + "fmt" "path/filepath" "strings" ) +type substitution struct { + old, new string +} + type command struct { - path, lang string - start, end *string + path, lang string + start, end *string + excludeStart bool + excludeEnd bool + dedent bool + trim bool + substitutions []substitution } func parseCommand(s string) (*command, error) { @@ -40,7 +50,7 @@ func parseCommand(s string) (*command, error) { cmd := &command{path: args[0]} args = args[1:] - if len(args) > 0 && args[0][0] != '/' { + if len(args) > 0 && !isRegexpOrOption(args[0]) { cmd.lang, args = args[0], args[1:] } else { ext := filepath.Ext(cmd.path[1:]) @@ -50,31 +60,119 @@ func parseCommand(s string) (*command, error) { cmd.lang = ext[1:] } - switch { - case len(args) == 1: - cmd.start = &args[0] - case len(args) == 2: - cmd.start, cmd.end = &args[0], &args[1] - case len(args) > 2: - return nil, errors.New("too many arguments") + // Consume regexp arguments (starting with / or !/ or bare $). + for len(args) > 0 { + arg := args[0] + isRegexp := strings.HasPrefix(arg, "/") + isExclude := strings.HasPrefix(arg, "!/") + isDollar := arg == "$" + isExcludeDollar := arg == "!$" + + if isExcludeDollar { + return nil, errors.New("exclude (!) cannot be used with $") + } + + if !isRegexp && !isExclude && !isDollar { + break + } + + if cmd.start == nil && !isDollar { + re := arg + if isExclude { + re = arg[1:] + cmd.excludeStart = true + } + cmd.start = &re + } else if cmd.end == nil { + re := arg + if isExclude { + re = arg[1:] + cmd.excludeEnd = true + } + cmd.end = &re + } else { + return nil, errors.New("too many arguments") + } + args = args[1:] + } + + // Single regexp with exclude is useless (would produce empty output). + if cmd.start != nil && cmd.end == nil && cmd.excludeStart { + return nil, errors.New("exclude (!) cannot be used with a single regexp") + } + + // Consume options: dedent, trim, s/old/new/. + for _, arg := range args { + switch { + case arg == "dedent": + cmd.dedent = true + case arg == "trim": + cmd.trim = true + case strings.HasPrefix(arg, "s/"): + sub, err := parseSubstitution(arg) + if err != nil { + return nil, err + } + cmd.substitutions = append(cmd.substitutions, sub) + default: + return nil, fmt.Errorf("unknown option %q", arg) + } } return cmd, nil } +func parseSubstitution(s string) (substitution, error) { + // s is already validated as a complete s/old/new/ token by fields(). + inner := s[2 : len(s)-1] // strip "s/" and trailing "/" + idx := strings.Index(inner, "/") + if idx < 0 { + return substitution{}, fmt.Errorf("invalid substitution %q", s) + } + return substitution{old: inner[:idx], new: inner[idx+1:]}, nil +} + +// isRegexpOrOption reports whether arg looks like a regexp (/.../, !/.../, $) +// or an option (dedent, trim, s/.../) rather than a language identifier. +func isRegexpOrOption(arg string) bool { + return arg[0] == '/' || arg[0] == '!' || arg == "$" || + strings.HasPrefix(arg, "s/") || arg == "dedent" || arg == "trim" +} + // fields returns a list of the groups of text separated by blanks, // keeping all text surrounded by / as a group. +// It also handles !/regexp/ (exclude) and s/old/new/ (substitution) tokens. func fields(s string) ([]string, error) { var args []string for s = strings.TrimSpace(s); len(s) > 0; s = strings.TrimSpace(s) { - if s[0] == '/' { - sep := nextSlash(s[1:]) + regexpStart := s[0] == '/' + excludeRegexp := len(s) > 1 && s[0] == '!' && s[1] == '/' + substStart := len(s) > 1 && s[0] == 's' && s[1] == '/' + + switch { + case regexpStart, excludeRegexp: + offset := 0 + if excludeRegexp { + offset = 1 + } + sep := nextSlash(s[offset+1:]) if sep < 0 { return nil, errors.New("unbalanced /") } - args, s = append(args, s[:sep+2]), s[sep+2:] - } else { + args, s = append(args, s[:offset+sep+2]), s[offset+sep+2:] + case substStart: + sep1 := nextSlash(s[2:]) + if sep1 < 0 { + return nil, errors.New("unbalanced / in substitution") + } + sep2 := nextSlash(s[2+sep1+1:]) + if sep2 < 0 { + return nil, errors.New("unbalanced / in substitution") + } + end := 2 + sep1 + 1 + sep2 + 1 + args, s = append(args, s[:end]), s[end:] + default: sep := strings.IndexByte(s[1:], ' ') if sep < 0 { return append(args, s), nil diff --git a/embedmd/command_test.go b/embedmd/command_test.go index 630aa76..d79c20d 100644 --- a/embedmd/command_test.go +++ b/embedmd/command_test.go @@ -19,6 +19,68 @@ import ( "github.com/campoy/embedmd/internal/testutil" ) +func TestFields(t *testing.T) { + tc := []struct { + name string + in string + out []string + err string + }{ + { + name: "simple args", + in: "code.go go /start/ /end/", out: []string{"code.go", "go", "/start/", "/end/"}, + }, + { + name: "exclude regexp", + in: "code.go go !/start/ !/end/", out: []string{"code.go", "go", "!/start/", "!/end/"}, + }, + { + name: "exclude regexp with spaces", + in: `code.go go !/start here/ !/end here/`, out: []string{"code.go", "go", "!/start here/", "!/end here/"}, + }, + { + name: "substitution", + in: "code.go go /start/ /end/ s/ELLIPSIS/.../", out: []string{"code.go", "go", "/start/", "/end/", "s/ELLIPSIS/.../"}, + }, + { + name: "substitution with spaces in old", + in: "code.go go s/_ = ELLIPSIS/.../", out: []string{"code.go", "go", "s/_ = ELLIPSIS/.../"}, + }, + { + name: "options after regexps", + in: "code.go go !/start/ !/end/ dedent trim", out: []string{"code.go", "go", "!/start/", "!/end/", "dedent", "trim"}, + }, + { + name: "unbalanced exclude regexp", + in: "code.go !/start", err: "unbalanced /", + }, + { + name: "unbalanced substitution first sep", + in: "code.go s/broken", err: "unbalanced / in substitution", + }, + { + name: "unbalanced substitution second sep", + in: "code.go s/old/broken", err: "unbalanced / in substitution", + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + got, err := fields(tt.in) + if !testutil.EqErr(t, tt.name, err, tt.err) { + return + } + if len(got) != len(tt.out) { + t.Fatalf("case [%s]: expected %d fields %v; got %d fields %v", tt.name, len(tt.out), tt.out, len(got), got) + } + for i := range got { + if got[i] != tt.out[i] { + t.Errorf("case [%s]: field %d: expected %q; got %q", tt.name, i, tt.out[i], got[i]) + } + } + }) + } +} + func TestParseCommand(t *testing.T) { tc := []struct { name string @@ -26,53 +88,160 @@ func TestParseCommand(t *testing.T) { cmd command err string }{ - {name: "start to end", - in: "(code.go /start/ /end/)", - cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/")}}, - {name: "only start", - in: "(code.go /start/)", - cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/")}}, - {name: "empty list", - in: "()", - err: "missing file name"}, - {name: "file with no extension and no lang", - in: "(test)", - err: "language is required when file has no extension"}, - {name: "surrounding blanks", - in: " \t (code.go) \t ", - cmd: command{path: "code.go", lang: "go"}}, - {name: "no parenthesis", - in: "{code.go}", - err: "argument list should be in parenthesis"}, - {name: "only left parenthesis", - in: "(code.go", - err: "argument list should be in parenthesis"}, - {name: "regexp not closed", - in: "(code.go /start)", - err: "unbalanced /"}, - {name: "end regexp not closed", - in: "(code.go /start/ /end)", - err: "unbalanced /"}, - {name: "file name and language", - in: "(test.md markdown)", - cmd: command{path: "test.md", lang: "markdown"}}, - {name: "multi-line comments", - in: `(doc.go /\/\*/ /\*\//)`, - cmd: command{path: "doc.go", lang: "go", start: testutil.Ptr(`/\/\*/`), end: testutil.Ptr(`/\*\//`)}}, - {name: "using $ as end", - in: "(foo.go /start/ $)", - cmd: command{path: "foo.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("$")}}, - {name: "extra arguments", - in: "(foo.go /start/ $ extra)", err: "too many arguments"}, - {name: "file name with directories", - in: "(foo/bar.go)", - cmd: command{path: "foo/bar.go", lang: "go"}}, - {name: "url", - in: "(http://golang.org/sample.go)", - cmd: command{path: "http://golang.org/sample.go", lang: "go"}}, - {name: "bad url", - in: "(http://golang:org:sample.go)", - cmd: command{path: "http://golang:org:sample.go", lang: "go"}}, + { + name: "start to end", + in: "(code.go /start/ /end/)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/")}, + }, + { + name: "only start", + in: "(code.go /start/)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/")}, + }, + { + name: "empty list", + in: "()", + err: "missing file name", + }, + { + name: "file with no extension and no lang", + in: "(test)", + err: "language is required when file has no extension", + }, + { + name: "surrounding blanks", + in: " \t (code.go) \t ", + cmd: command{path: "code.go", lang: "go"}, + }, + { + name: "no parenthesis", + in: "{code.go}", + err: "argument list should be in parenthesis", + }, + { + name: "only left parenthesis", + in: "(code.go", + err: "argument list should be in parenthesis", + }, + { + name: "regexp not closed", + in: "(code.go /start)", + err: "unbalanced /", + }, + { + name: "end regexp not closed", + in: "(code.go /start/ /end)", + err: "unbalanced /", + }, + { + name: "file name and language", + in: "(test.md markdown)", + cmd: command{path: "test.md", lang: "markdown"}, + }, + { + name: "multi-line comments", + in: `(doc.go /\/\*/ /\*\//)`, + cmd: command{path: "doc.go", lang: "go", start: testutil.Ptr(`/\/\*/`), end: testutil.Ptr(`/\*\//`)}, + }, + { + name: "using $ as end", + in: "(foo.go /start/ $)", + cmd: command{path: "foo.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("$")}, + }, + { + name: "exclude start", + in: "(code.go !/start/ /end/)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/"), excludeStart: true}, + }, + { + name: "exclude end", + in: "(code.go /start/ !/end/)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/"), excludeEnd: true}, + }, + { + name: "exclude both", + in: "(code.go !/start/ !/end/)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/"), excludeStart: true, excludeEnd: true}, + }, + { + name: "exclude start with dollar end", + in: "(code.go !/start/ $)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("$"), excludeStart: true}, + }, + { + name: "dedent option", + in: "(code.go /start/ /end/ dedent)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/"), dedent: true}, + }, + { + name: "trim option", + in: "(code.go /start/ /end/ trim)", + cmd: command{path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/"), trim: true}, + }, + { + name: "substitution option", + in: "(code.go /start/ /end/ s/ELLIPSIS/.../)", + cmd: command{ + path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/"), + substitutions: []substitution{{old: "ELLIPSIS", new: "..."}}, + }, + }, + { + name: "multiple substitutions", + in: "(code.go s/ELLIPSIS/.../ s/_ = ELLIPSIS/.../)", + cmd: command{ + path: "code.go", lang: "go", + substitutions: []substitution{{old: "ELLIPSIS", new: "..."}, {old: "_ = ELLIPSIS", new: "..."}}, + }, + }, + { + name: "all options combined", + in: "(code.go !/start/ !/end/ dedent trim s/ELLIPSIS/.../)", + cmd: command{ + path: "code.go", lang: "go", start: testutil.Ptr("/start/"), end: testutil.Ptr("/end/"), + excludeStart: true, excludeEnd: true, dedent: true, trim: true, + substitutions: []substitution{{old: "ELLIPSIS", new: "..."}}, + }, + }, + { + name: "exclude on single regexp is error", + in: "(code.go !/only/)", + err: "exclude (!) cannot be used with a single regexp", + }, + { + name: "exclude on dollar is error", + in: "(code.go /start/ !$)", + err: "exclude (!) cannot be used with $", + }, + { + name: "unknown option", + in: "(code.go /start/ /end/ bogus)", + err: `unknown option "bogus"`, + }, + { + name: "invalid substitution", + in: "(code.go /start/ /end/ s/broken)", + err: "unbalanced / in substitution", + }, + { + name: "extra arguments", + in: "(foo.go /start/ $ extra)", err: "unknown option \"extra\"", + }, + { + name: "file name with directories", + in: "(foo/bar.go)", + cmd: command{path: "foo/bar.go", lang: "go"}, + }, + { + name: "url", + in: "(http://golang.org/sample.go)", + cmd: command{path: "http://golang.org/sample.go", lang: "go"}, + }, + { + name: "bad url", + in: "(http://golang:org:sample.go)", + cmd: command{path: "http://golang:org:sample.go", lang: "go"}, + }, } for _, tt := range tc { @@ -95,6 +264,27 @@ func TestParseCommand(t *testing.T) { if !testutil.EqPtr(want.end, got.end) { t.Errorf("case [%s]: expected end %v; got %v", tt.name, testutil.Str(want.end), testutil.Str(got.end)) } + if want.excludeStart != got.excludeStart { + t.Errorf("case [%s]: expected excludeStart %v; got %v", tt.name, want.excludeStart, got.excludeStart) + } + if want.excludeEnd != got.excludeEnd { + t.Errorf("case [%s]: expected excludeEnd %v; got %v", tt.name, want.excludeEnd, got.excludeEnd) + } + if want.dedent != got.dedent { + t.Errorf("case [%s]: expected dedent %v; got %v", tt.name, want.dedent, got.dedent) + } + if want.trim != got.trim { + t.Errorf("case [%s]: expected trim %v; got %v", tt.name, want.trim, got.trim) + } + if len(want.substitutions) != len(got.substitutions) { + t.Errorf("case [%s]: expected %d substitutions; got %d", tt.name, len(want.substitutions), len(got.substitutions)) + } else { + for i := range want.substitutions { + if want.substitutions[i] != got.substitutions[i] { + t.Errorf("case [%s]: substitution %d: expected %v; got %v", tt.name, i, want.substitutions[i], got.substitutions[i]) + } + } + } }) } } diff --git a/embedmd/embedmd.go b/embedmd/embedmd.go index 52f8c84..a6f6f73 100644 --- a/embedmd/embedmd.go +++ b/embedmd/embedmd.go @@ -133,6 +133,16 @@ func (e *embedder) runCommand(w io.Writer, cmd *command) error { return fmt.Errorf("could not extract content from %s: %v", cmd.path, err) } + // Apply transforms in order: exclude → trim → dedent → substitute. + b = excludeLines(b, cmd.excludeStart, cmd.excludeEnd) + if cmd.trim { + b = trimTrailingBlankLines(b) + } + if cmd.dedent { + b = dedentBytes(b) + } + b = applySubstitutions(b, cmd.substitutions) + if len(b) > 0 && b[len(b)-1] != '\n' { b = append(b, '\n') } diff --git a/embedmd/embedmd_test.go b/embedmd/embedmd_test.go index eb2dbf2..d26437d 100644 --- a/embedmd/embedmd_test.go +++ b/embedmd/embedmd_test.go @@ -42,34 +42,60 @@ func TestExtract(t *testing.T) { out string err string }{ - {name: "no limits", - out: string(content)}, - {name: "only one line", - start: testutil.Ptr("/func main.*\n/"), out: "func main() {\n"}, - {name: "from package to end", - start: testutil.Ptr("/package main/"), end: testutil.Ptr("$"), out: string(content[1:])}, - {name: "not matching", - start: testutil.Ptr("/gopher/"), err: "could not match \"/gopher/\""}, - {name: "part of a line", - start: testutil.Ptr("/fmt.P/"), end: testutil.Ptr("/hello/"), out: "fmt.Println(\"hello"}, - {name: "function call", - start: testutil.Ptr("/fmt\\.[^()]*/"), out: "fmt.Println"}, - {name: "from fmt to end of line", - start: testutil.Ptr("/fmt.P.*\n/"), out: "fmt.Println(\"hello, test\")\n"}, - {name: "from func to end of next line", - start: testutil.Ptr("/func/"), end: testutil.Ptr("/Println.*\n/"), out: "func main() {\n fmt.Println(\"hello, test\")\n"}, - {name: "from func to }", - start: testutil.Ptr("/func main/"), end: testutil.Ptr("/}/"), out: "func main() {\n fmt.Println(\"hello, test\")\n}"}, + { + name: "no limits", + out: string(content), + }, + { + name: "only one line", + start: testutil.Ptr("/func main.*\n/"), out: "func main() {\n", + }, + { + name: "from package to end", + start: testutil.Ptr("/package main/"), end: testutil.Ptr("$"), out: string(content[1:]), + }, + { + name: "not matching", + start: testutil.Ptr("/gopher/"), err: "could not match \"/gopher/\"", + }, + { + name: "part of a line", + start: testutil.Ptr("/fmt.P/"), end: testutil.Ptr("/hello/"), out: "fmt.Println(\"hello", + }, + { + name: "function call", + start: testutil.Ptr("/fmt\\.[^()]*/"), out: "fmt.Println", + }, + { + name: "from fmt to end of line", + start: testutil.Ptr("/fmt.P.*\n/"), out: "fmt.Println(\"hello, test\")\n", + }, + { + name: "from func to end of next line", + start: testutil.Ptr("/func/"), end: testutil.Ptr("/Println.*\n/"), out: "func main() {\n fmt.Println(\"hello, test\")\n", + }, + { + name: "from func to }", + start: testutil.Ptr("/func main/"), end: testutil.Ptr("/}/"), out: "func main() {\n fmt.Println(\"hello, test\")\n}", + }, - {name: "bad start regexp", - start: testutil.Ptr("/(/"), err: "error parsing regexp: missing closing ): `(`"}, - {name: "bad regexp", - start: testutil.Ptr("something"), err: "missing slashes (/) around \"something\""}, - {name: "bad end regexp", - start: testutil.Ptr("/fmt.P/"), end: testutil.Ptr("/)/"), err: "error parsing regexp: unexpected ): `)`"}, + { + name: "bad start regexp", + start: testutil.Ptr("/(/"), err: "error parsing regexp: missing closing ): `(`", + }, + { + name: "bad regexp", + start: testutil.Ptr("something"), err: "missing slashes (/) around \"something\"", + }, + { + name: "bad end regexp", + start: testutil.Ptr("/fmt.P/"), end: testutil.Ptr("/)/"), err: "error parsing regexp: unexpected ): `)`", + }, - {name: "start and end of line ^$", - start: testutil.Ptr("/^func main/"), end: testutil.Ptr("/}$/"), out: "func main() {\n fmt.Println(\"hello, test\")\n}"}, + { + name: "start and end of line ^$", + start: testutil.Ptr("/^func main/"), end: testutil.Ptr("/}$/"), out: "func main() {\n fmt.Println(\"hello, test\")\n}", + }, } for _, tt := range tc { @@ -124,6 +150,56 @@ func TestExtractFromFile(t *testing.T) { files: map[string][]byte{"code.go": []byte(content)}, err: "could not extract content from code.go: could not match \"/potato/\"", }, + { + name: "exclude start and end lines", + cmd: command{ + path: "code.go", lang: "go", + start: testutil.Ptr("/\\/\\/ start/"), end: testutil.Ptr("/\\/\\/ end/"), + excludeStart: true, excludeEnd: true, + }, + files: map[string][]byte{"code.go": []byte("// start\nfmt.Println(\"hello\")\n// end\n")}, + out: "```go\nfmt.Println(\"hello\")\n```\n", + }, + { + name: "dedent strips common indent", + cmd: command{ + path: "code.go", lang: "go", + start: testutil.Ptr("/\\/\\/ start/"), end: testutil.Ptr("/\\/\\/ end/"), + excludeStart: true, excludeEnd: true, dedent: true, + }, + files: map[string][]byte{"code.go": []byte("// start\n\tfmt.Println(\"hello\")\n\tx := 1\n// end\n")}, + out: "```go\nfmt.Println(\"hello\")\nx := 1\n```\n", + }, + { + name: "trim removes trailing blank lines", + cmd: command{ + path: "code.go", lang: "go", + start: testutil.Ptr("/\\/\\/ start/"), end: testutil.Ptr("/\\/\\/ end/"), + excludeStart: true, excludeEnd: true, trim: true, + }, + files: map[string][]byte{"code.go": []byte("// start\nline1\n\n\n// end\n")}, + out: "```go\nline1\n```\n", + }, + { + name: "substitution replaces text", + cmd: command{ + path: "code.go", lang: "go", + substitutions: []substitution{{old: "ELLIPSIS", new: "..."}}, + }, + files: map[string][]byte{"code.go": []byte("x = ELLIPSIS\n")}, + out: "```go\nx = ...\n```\n", + }, + { + name: "all transforms combined", + cmd: command{ + path: "code.go", lang: "go", + start: testutil.Ptr("/\\/\\/ snippet-start/"), end: testutil.Ptr("/\\/\\/ snippet-end/"), + excludeStart: true, excludeEnd: true, dedent: true, trim: true, + substitutions: []substitution{{old: "ELLIPSIS", new: "..."}}, + }, + files: map[string][]byte{"code.go": []byte("// snippet-start\n\ttokenSource := ELLIPSIS\n\tclient := New(ctx, tokenSource)\n\n// snippet-end\n")}, + out: "```go\ntokenSource := ...\nclient := New(ctx, tokenSource)\n```\n", + }, } for _, tt := range tc { diff --git a/embedmd/transform.go b/embedmd/transform.go new file mode 100644 index 0000000..5fdb7a4 --- /dev/null +++ b/embedmd/transform.go @@ -0,0 +1,130 @@ +package embedmd + +import "bytes" + +// splitLines splits b into lines, each retaining its trailing \n. +// Unlike bytes.SplitAfter, it never returns an empty trailing element. +func splitLines(b []byte) [][]byte { + lines := bytes.SplitAfter(b, []byte("\n")) + if len(lines) > 0 && len(lines[len(lines)-1]) == 0 { + lines = lines[:len(lines)-1] + } + return lines +} + +func excludeLines(b []byte, exclStart, exclEnd bool) []byte { + if len(b) == 0 { + return b + } + lines := splitLines(b) + if exclStart && len(lines) > 0 { + lines = lines[1:] + } + if exclEnd && len(lines) > 0 { + lines = lines[:len(lines)-1] + // Ensure trailing newline on the new last line. + if len(lines) > 0 { + l := lines[len(lines)-1] + if len(l) > 0 && l[len(l)-1] != '\n' { + lines[len(lines)-1] = append(l, '\n') + } + } + } + return bytes.Join(lines, nil) +} + +func trimTrailingBlankLines(b []byte) []byte { + if len(b) == 0 { + return b + } + lines := splitLines(b) + // Remove trailing blank/whitespace-only lines. + for len(lines) > 0 { + last := lines[len(lines)-1] + if len(bytes.TrimRight(last, " \t\n\r")) == 0 { + lines = lines[:len(lines)-1] + } else { + break + } + } + if len(lines) == 0 { + return nil + } + // Ensure last line ends with newline + last := lines[len(lines)-1] + if len(last) > 0 && last[len(last)-1] != '\n' { + lines[len(lines)-1] = append(last, '\n') + } + return bytes.Join(lines, nil) +} + +func dedentBytes(b []byte) []byte { + if len(b) == 0 { + return b + } + lines := splitLines(b) + + // Find common whitespace prefix among non-blank lines. + var prefix []byte + first := true + for _, line := range lines { + content := bytes.TrimRight(line, "\n") + if len(bytes.TrimSpace(content)) == 0 { + continue // skip blank lines for prefix calculation + } + linePrefix := leadingWhitespace(content) + if first { + prefix = linePrefix + first = false + } else { + prefix = commonPrefix(prefix, linePrefix) + } + } + + if len(prefix) == 0 { + return b + } + + // Strip common prefix from each line. + var result []byte + for _, line := range lines { + content := bytes.TrimRight(line, "\n") + if len(bytes.TrimSpace(content)) == 0 { + // Whitespace-only line: make it empty + result = append(result, '\n') + } else { + result = append(result, bytes.TrimPrefix(content, prefix)...) + result = append(result, '\n') + } + } + return result +} + +func leadingWhitespace(b []byte) []byte { + for i, c := range b { + if c != ' ' && c != '\t' { + return b[:i] + } + } + return b +} + +func commonPrefix(a, b []byte) []byte { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + if a[i] != b[i] { + return a[:i] + } + } + return a[:n] +} + +func applySubstitutions(b []byte, subs []substitution) []byte { + for _, s := range subs { + b = bytes.ReplaceAll(b, []byte(s.old), []byte(s.new)) + } + return b +} diff --git a/embedmd/transform_test.go b/embedmd/transform_test.go new file mode 100644 index 0000000..2445048 --- /dev/null +++ b/embedmd/transform_test.go @@ -0,0 +1,195 @@ +package embedmd + +import "testing" + +func TestExcludeLines(t *testing.T) { + tc := []struct { + name string + in string + excludeStart, excludeEnd bool + out string + }{ + { + name: "no exclusion", + in: "line1\nline2\nline3\n", out: "line1\nline2\nline3\n", + }, + { + name: "exclude start", + in: "// start\nline1\nline2\n", excludeStart: true, out: "line1\nline2\n", + }, + { + name: "exclude end", + in: "line1\nline2\n// end\n", excludeEnd: true, out: "line1\nline2\n", + }, + { + name: "exclude both", + in: "// start\nline1\nline2\n// end\n", excludeStart: true, excludeEnd: true, out: "line1\nline2\n", + }, + { + name: "exclude both leaves nothing", + in: "// start\n// end\n", excludeStart: true, excludeEnd: true, out: "", + }, + { + name: "single line exclude start", + in: "only\n", excludeStart: true, out: "", + }, + { + name: "no trailing newline exclude end", + in: "line1\nline2", excludeEnd: true, out: "line1\n", + }, + { + name: "no trailing newline exclude both", + in: "start\nmiddle\nend", excludeStart: true, excludeEnd: true, out: "middle\n", + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + got := excludeLines([]byte(tt.in), tt.excludeStart, tt.excludeEnd) + if string(got) != tt.out { + t.Errorf("case [%s]: expected %q; got %q", tt.name, tt.out, string(got)) + } + }) + } +} + +func TestTrimTrailingBlankLines(t *testing.T) { + tc := []struct { + name string + in string + out string + }{ + { + name: "no trailing blanks", + in: "line1\nline2\n", out: "line1\nline2\n", + }, + { + name: "one trailing blank", + in: "line1\nline2\n\n", out: "line1\nline2\n", + }, + { + name: "multiple trailing blanks", + in: "line1\n\n\n\n", out: "line1\n", + }, + { + name: "trailing whitespace-only lines", + in: "line1\n \n\t\n", out: "line1\n", + }, + { + name: "all blank lines", + in: "\n\n\n", out: "", + }, + { + name: "preserves internal blank lines", + in: "line1\n\nline2\n\n", out: "line1\n\nline2\n", + }, + { + name: "empty input", + in: "", out: "", + }, + { + name: "single line no newline", + in: "hello", out: "hello\n", + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + got := trimTrailingBlankLines([]byte(tt.in)) + if string(got) != tt.out { + t.Errorf("case [%s]: expected %q; got %q", tt.name, tt.out, string(got)) + } + }) + } +} + +func TestDedentBytes(t *testing.T) { + tc := []struct { + name string + in string + out string + }{ + { + name: "no indentation", + in: "line1\nline2\n", out: "line1\nline2\n", + }, + { + name: "uniform tab indent", + in: "\tline1\n\tline2\n", out: "line1\nline2\n", + }, + { + name: "uniform space indent", + in: " line1\n line2\n", out: "line1\nline2\n", + }, + { + name: "mixed depth keeps relative indent", + in: "\t\tline1\n\tline2\n\t\t\tline3\n", out: "\tline1\nline2\n\t\tline3\n", + }, + { + name: "blank lines ignored for min calculation", + in: "\tline1\n\n\tline2\n", out: "line1\n\nline2\n", + }, + { + name: "whitespace-only lines ignored for min calculation", + in: "\tline1\n \n\tline2\n", out: "line1\n\nline2\n", + }, + { + name: "already no indent", + in: "a\n b\n", out: "a\n b\n", + }, + { + name: "empty input", + in: "", out: "", + }, + { + name: "single indented line", + in: " hello\n", out: "hello\n", + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + got := dedentBytes([]byte(tt.in)) + if string(got) != tt.out { + t.Errorf("case [%s]: expected %q; got %q", tt.name, tt.out, string(got)) + } + }) + } +} + +func TestApplySubstitutions(t *testing.T) { + tc := []struct { + name string + in string + subs []substitution + out string + }{ + { + name: "no substitutions", + in: "hello world", subs: nil, out: "hello world", + }, + { + name: "single substitution", + in: "x = ELLIPSIS", subs: []substitution{{old: "ELLIPSIS", new: "..."}}, out: "x = ...", + }, + { + name: "multiple substitutions applied in order", + in: "_ = ELLIPSIS\nELLIPSIS\n", + subs: []substitution{{old: "_ = ELLIPSIS", new: "..."}, {old: "ELLIPSIS", new: "..."}}, + out: "...\n...\n", + }, + { + name: "substitution replaces all occurrences", + in: "A and A and A", subs: []substitution{{old: "A", new: "B"}}, out: "B and B and B", + }, + { + name: "no match leaves content unchanged", + in: "hello", subs: []substitution{{old: "MISSING", new: "X"}}, out: "hello", + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + got := applySubstitutions([]byte(tt.in), tt.subs) + if string(got) != tt.out { + t.Errorf("case [%s]: expected %q; got %q", tt.name, tt.out, string(got)) + } + }) + } +}