Plugin Development

This guide shows you how to create your own plugins using the patterns demonstrated in this project.

Overview

There are two types of plugins you can create:

Simple plugins - Self-contained, no host services needed
Plugins with host services - Access filesystem, environment, etc. through the host

Both types use the same infrastructure and can coexist in the same system.

Simple Plugin (No Host Services)

Perfect for plugins that just process data without needing system resources.

Example: Data Formatter Plugin

1. Define your protobuf service:

// shared/proto/formatter/v1/formatter.proto
syntax = "proto3";

package formatter.v1;

service Formatter {
  rpc Format(FormatRequest) returns (FormatResponse);
}

message FormatRequest {
  string text = 1;
  string style = 2;  // "upper", "lower", "title"
}

message FormatResponse {
  string formatted_text = 1;
}

2. Generate code:

buf generate

3. Implement the plugin:

// plugins/formatter/formatter.go
package main

import (
    "context"
    "strings"

    "github.com/hashicorp/go-plugin"
    formatterv1 "github.com/bmj2728/hst/shared/protogen/formatter/v1"
    "google.golang.org/grpc"
)

// Formatter implements the business logic
type Formatter struct{}

func (f *Formatter) Format(ctx context.Context, req *formatterv1.FormatRequest) (*formatterv1.FormatResponse, error) {
    var result string
    switch req.Style {
    case "upper":
        result = strings.ToUpper(req.Text)
    case "lower":
        result = strings.ToLower(req.Text)
    case "title":
        result = strings.ToTitle(req.Text)
    default:
        result = req.Text
    }
    return &formatterv1.FormatResponse{FormattedText: result}, nil
}

// GRPCServer wrapper
type FormatterGRPCServer struct {
    formatterv1.UnimplementedFormatterServer
    Impl *Formatter
}

func (s *FormatterGRPCServer) Format(ctx context.Context, req *formatterv1.FormatRequest) (*formatterv1.FormatResponse, error) {
    return s.Impl.Format(ctx, req)
}

// Plugin interface implementation
type FormatterPlugin struct {
    plugin.Plugin
    Impl *Formatter
}

func (p *FormatterPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error {
    formatterv1.RegisterFormatterServer(s, &FormatterGRPCServer{Impl: p.Impl})
    return nil
}

func (p *FormatterPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) {
    return formatterv1.NewFormatterClient(c), nil
}

func main() {
    plugin.Serve(&plugin.ServeConfig{
        HandshakeConfig: plugin.HandshakeConfig{
            ProtocolVersion:  1,
            MagicCookieKey:   "TEST_KEY",
            MagicCookieValue: "TEST_VALUE",
        },
        Plugins: map[string]plugin.Plugin{
            "formatter": &FormatterPlugin{Impl: &Formatter{}},
        },
        GRPCServer: plugin.DefaultGRPCServer,
    })
}

4. Build:

go build -o plugins/formatter/formatter ./plugins/formatter

That's it! No host services, no complex setup. Just implement your logic and go.

Plugin with Host Services

For plugins that need filesystem access, environment variables, or other system resources.

Example: Log Analyzer Plugin

1. Define your protobuf service:

// shared/proto/loganalyzer/v1/loganalyzer.proto
syntax = "proto3";

package loganalyzer.v1;

service LogAnalyzer {
  rpc Analyze(AnalyzeRequest) returns (AnalyzeResponse);
  rpc EstablishHostServices(EstablishHostServicesRequest) returns (EstablishHostServicesResponse);
}

message AnalyzeRequest {
  string log_file_path = 1;
}

message AnalyzeResponse {
  int32 error_count = 1;
  int32 warning_count = 2;
  repeated string errors = 3;
}

message EstablishHostServicesRequest {
  uint32 host_service = 1;
}

message EstablishHostServicesResponse {
  string client_id = 1;
}

2. Generate code:

buf generate

3. Implement the plugin:

// plugins/loganalyzer/loganalyzer.go
package main

import (
    "context"
    "strings"
    "sync"

    "github.com/hashicorp/go-plugin"
    "github.com/hashicorp/go-hclog"
    loganalyzerv1 "github.com/bmj2728/hst/shared/protogen/loganalyzer/v1"
    hostservev1 "github.com/bmj2728/hst/shared/protogen/hostserve/v1"
    "github.com/bmj2728/hst/shared/pkg/hostserve"
    "google.golang.org/grpc"
)

// LogAnalyzer implements the business logic
type LogAnalyzer struct {
    broker            *plugin.GRPCBroker
    hostServiceClient hostserve.IHostServices
    conn              *grpc.ClientConn
    connMutex         sync.Mutex
    logger            hclog.Logger
}

// Implement HostConnection interface for automatic setup
func (la *LogAnalyzer) SetBroker(broker *plugin.GRPCBroker) {
    la.broker = broker
}

func (la *LogAnalyzer) EstablishHostServices(hostServiceID uint32) (hostserve.ClientID, error) {
    la.connMutex.Lock()
    defer la.connMutex.Unlock()

    conn, err := la.broker.Dial(hostServiceID)
    if err != nil {
        return "", err
    }

    la.conn = conn
    client := hostserve.NewHostServiceGRPCClient(hostservev1.NewHostServiceClient(conn))
    la.hostServiceClient = client

    return client.ClientID(), nil
}

func (la *LogAnalyzer) DisconnectHostServices() {
    la.connMutex.Lock()
    defer la.connMutex.Unlock()

    if la.conn != nil {
        la.conn.Close()
        la.conn = nil
    }
}

// Business logic using host services
func (la *LogAnalyzer) Analyze(ctx context.Context, req *loganalyzerv1.AnalyzeRequest) (*loganalyzerv1.AnalyzeResponse, error) {
    // Use host service to read the log file securely
    data, err := la.hostServiceClient.ReadFile(ctx, "/var/log", req.LogFilePath)
    if err != nil {
        return nil, err
    }

    // Analyze the content
    lines := strings.Split(string(data), "\n")
    var errorCount, warningCount int
    var errors []string

    for _, line := range lines {
        if strings.Contains(line, "ERROR") {
            errorCount++
            errors = append(errors, line)
        } else if strings.Contains(line, "WARNING") {
            warningCount++
        }
    }

    return &loganalyzerv1.AnalyzeResponse{
        ErrorCount:   int32(errorCount),
        WarningCount: int32(warningCount),
        Errors:       errors,
    }, nil
}

// GRPCServer wrapper
type LogAnalyzerGRPCServer struct {
    loganalyzerv1.UnimplementedLogAnalyzerServer
    Impl *LogAnalyzer
}

func (s *LogAnalyzerGRPCServer) Analyze(ctx context.Context, req *loganalyzerv1.AnalyzeRequest) (*loganalyzerv1.AnalyzeResponse, error) {
    return s.Impl.Analyze(ctx, req)
}

func (s *LogAnalyzerGRPCServer) EstablishHostServices(ctx context.Context, req *loganalyzerv1.EstablishHostServicesRequest) (*loganalyzerv1.EstablishHostServicesResponse, error) {
    clientID, err := s.Impl.EstablishHostServices(req.HostService)
    if err != nil {
        return nil, err
    }
    return &loganalyzerv1.EstablishHostServicesResponse{ClientId: string(clientID)}, nil
}

// Plugin interface implementation
type LogAnalyzerPlugin struct {
    plugin.Plugin
    Impl *LogAnalyzer
}

func (p *LogAnalyzerPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error {
    p.Impl.SetBroker(broker)
    loganalyzerv1.RegisterLogAnalyzerServer(s, &LogAnalyzerGRPCServer{Impl: p.Impl})
    return nil
}

func (p *LogAnalyzerPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) {
    return loganalyzerv1.NewLogAnalyzerClient(c), nil
}

func main() {
    logger := hclog.New(&hclog.LoggerOptions{
        Level:      hclog.Debug,
        Output:     os.Stderr,
        JSONFormat: true,
    })

    plugin.Serve(&plugin.ServeConfig{
        HandshakeConfig: plugin.HandshakeConfig{
            ProtocolVersion:  1,
            MagicCookieKey:   "TEST_KEY",
            MagicCookieValue: "TEST_VALUE",
        },
        Plugins: map[string]plugin.Plugin{
            "loganalyzer": &LogAnalyzerPlugin{
                Impl: &LogAnalyzer{logger: logger},
            },
        },
        GRPCServer: plugin.DefaultGRPCServer,
    })
}

4. Build:

go build -o plugins/loganalyzer/loganalyzer ./plugins/loganalyzer

5. Use from host:

// In your host code
raw, _ := rpcClient.Dispense("loganalyzer")
cid, _ := hostconn.EstablishHostServices(raw, hostServices, logger)
if cid != "" {
    hostServices.ActiveClients().AddClient(cid, "loganalyzer")
}

// Call the plugin
analyzer := raw.(loganalyzerv1.LogAnalyzerClient)
result, _ := analyzer.Analyze(ctx, &loganalyzerv1.AnalyzeRequest{
    LogFilePath: "app.log",
})

Host Connection Interface

To use host services, your plugin must implement the HostConnection interface:

type HostConnection interface {
    SetBroker(broker *plugin.GRPCBroker)
    EstablishHostServices(hostServiceID uint32) (ClientID, error)
    DisconnectHostServices()
}

This enables the hostconn.EstablishHostServices() helper to automatically set up the connection.

Adding New Host Service Functions

Want to add a new capability to all plugins? It's easy:

1. Update the proto:

// shared/proto/hostserve/v1/hostserve.proto
service HostService {
  // ... existing methods ...
  rpc ExecuteCommand(ExecuteCommandRequest) returns (ExecuteCommandResponse);
}

message ExecuteCommandRequest {
  string command = 1;
  repeated string args = 2;
}

message ExecuteCommandResponse {
  string stdout = 1;
  string stderr = 2;
  int32 exit_code = 3;
}

2. Regenerate:

buf generate

3. Update the interface:

// shared/pkg/hostserve/host_service.go
type IHostServices interface {
    ExecuteCommand(ctx context.Context, command string, args []string) (stdout, stderr string, exitCode int, err error)
    // ... other methods
}

4. Implement business logic:

// shared/pkg/hostserve/host_exec.go (new file)
func (h *HostServices) ExecuteCommand(ctx context.Context, command string, args []string) (string, string, int, error) {
    cmd := exec.CommandContext(ctx, command, args...)

    var stdout, stderr bytes.Buffer
    cmd.Stdout = &stdout
    cmd.Stderr = &stderr

    err := cmd.Run()
    exitCode := cmd.ProcessState.ExitCode()

    return stdout.String(), stderr.String(), exitCode, err
}

5. Add gRPC client wrapper:

// shared/pkg/hostserve/grpc_client_exec.go (new file)
func (c *HostServiceGRPCClient) ExecuteCommand(ctx context.Context, command string, args []string) (string, string, int, error) {
    resp, err := c.client.ExecuteCommand(ctx, &hostservev1.ExecuteCommandRequest{
        Command: command,
        Args:    args,
    })
    if err != nil {
        return "", "", -1, err
    }
    return resp.Stdout, resp.Stderr, int(resp.ExitCode), nil
}

6. Add gRPC server wrapper:

// shared/pkg/hostserve/grpc_server_exec.go (new file)
func (s *HostServiceGRPCServer) ExecuteCommand(ctx context.Context, req *hostservev1.ExecuteCommandRequest) (*hostservev1.ExecuteCommandResponse, error) {
    return processRequestWithResponse(ctx, s, "ExecuteCommand",
        func(ctx context.Context) ([]any, error) {
            return extractRequestFields(req)
        },
        func(ctx context.Context, hostServices *HostServices) (*hostservev1.ExecuteCommandResponse, error) {
            stdout, stderr, exitCode, err := hostServices.ExecuteCommand(ctx, req.Command, req.Args)
            if err != nil {
                return nil, err
            }
            return &hostservev1.ExecuteCommandResponse{
                Stdout:   stdout,
                Stderr:   stderr,
                ExitCode: int32(exitCode),
            }, nil
        })
}

Done! All existing plugins can now call ExecuteCommand() with zero changes to plugin code.

Best Practices

1. Use Structured Logging

logger := hclog.New(&hclog.LoggerOptions{
    Name:       "myplugin",
    Level:      hclog.Debug,
    JSONFormat: true,
})

logger.Info("processing request", "file", path, "size", size)

2. Handle Context Cancellation

func (p *MyPlugin) DoWork(ctx context.Context, req *Request) (*Response, error) {
    select {
    case <-ctx.Done():
        return nil, ctx.Err()
    default:
        // Do work
    }
}

3. Close Resources Properly

func (p *MyPlugin) ProcessFile(ctx context.Context, path string) error {
    handle, size, err := p.hostServiceClient.FileOpen(ctx, "/data", path, os.O_RDONLY, 0)
    if err != nil {
        return err
    }
    defer p.hostServiceClient.FileClose(ctx, handle)

    // Use handle
}

4. Use Thread-Safe Access

type MyPlugin struct {
    mu                sync.Mutex
    hostServiceClient hostserve.IHostServices
}

func (p *MyPlugin) SetClient(client hostserve.IHostServices) {
    p.mu.Lock()
    defer p.mu.Unlock()
    p.hostServiceClient = client
}

5. Provide Sensible Defaults

func (p *MyPlugin) Process(req *ProcessRequest) (*ProcessResponse, error) {
    chunkSize := req.ChunkSize
    if chunkSize == 0 {
        chunkSize = 64 * 1024 // 64KB default
    }
    // ...
}

Testing Your Plugin

Unit Tests

func TestMyPlugin_Format(t *testing.T) {
    plugin := &MyPlugin{}

    req := &mypluginv1.FormatRequest{
        Text:  "hello",
        Style: "upper",
    }

    resp, err := plugin.Format(context.Background(), req)
    if err != nil {
        t.Fatalf("unexpected error: %v", err)
    }

    if resp.FormattedText != "HELLO" {
        t.Errorf("expected HELLO, got %s", resp.FormattedText)
    }
}

Integration Tests with Mock Host Services

type mockHostServices struct {
    hostserve.IHostServices
    readFileCalled bool
}

func (m *mockHostServices) ReadFile(ctx context.Context, rootDir, path string) ([]byte, error) {
    m.readFileCalled = true
    return []byte("test data"), nil
}

func TestMyPlugin_WithHostServices(t *testing.T) {
    mock := &mockHostServices{}
    plugin := &MyPlugin{hostServiceClient: mock}

    // Test your plugin logic

    if !mock.readFileCalled {
        t.Error("expected ReadFile to be called")
    }
}

Testing with the TUI

The easiest way to test your plugin interactively:

Build your plugin
Add it to main.go in the host
Run ./hst to launch the TUI
Navigate to your plugin and test its functions

Common Patterns

Pattern: File Processing

func (p *MyPlugin) ProcessFile(ctx context.Context, rootDir, path string) error {
    // Open file
    handle, _, err := p.hostServiceClient.FileOpen(ctx, rootDir, path, os.O_RDONLY, 0)
    if err != nil {
        return err
    }
    defer p.hostServiceClient.FileClose(ctx, handle)

    // Stream read
    reader, err := p.hostServiceClient.FileReader(ctx, handle, 64*1024)
    if err != nil {
        return err
    }

    // Process chunks
    buffer := make([]byte, 4096)
    for {
        n, err := reader.Read(buffer)
        if n > 0 {
            // Process buffer[:n]
        }
        if err == io.EOF {
            break
        }
        if err != nil {
            return err
        }
    }

    return nil
}

Pattern: Temporary Files

func (p *MyPlugin) CreateWorkFile(ctx context.Context) error {
    // Create temp file (auto-cleanup)
    handle, err := p.hostServiceClient.FileCreateTemp(ctx, "", "work-*.tmp")
    if err != nil {
        return err
    }
    defer p.hostServiceClient.FileClose(ctx, handle)

    // Write data
    writer, err := p.hostServiceClient.FileWriter(ctx, handle)
    if err != nil {
        return err
    }

    data := []byte("work in progress")
    _, err = writer.Write(data)
    writer.Close()

    // File is automatically deleted when plugin disconnects
    return err
}

Pattern: Directory Operations

func (p *MyPlugin) ProcessDirectory(ctx context.Context, rootDir, path string) error {
    entries, err := p.hostServiceClient.ReadDir(ctx, rootDir, path)
    if err != nil {
        return err
    }

    for _, entry := range entries {
        if entry.IsDir() {
            // Recurse into subdirectory
            subPath := filepath.Join(path, entry.Name())
            p.ProcessDirectory(ctx, rootDir, subPath)
        } else {
            // Process file
            p.ProcessFile(ctx, rootDir, filepath.Join(path, entry.Name()))
        }
    }

    return nil
}

Next Steps

API Reference - Explore all 60+ host service operations
Cross Language Plugins - Create Python plugins
Architecture - Understand the broker and multiplexing

Troubleshooting

Plugin won't start

Check handshake configuration matches host
Verify protobuf definitions are up to date (buf generate)
Check plugin executable has correct permissions

Can't connect to host services

Ensure HostConnection interface is implemented
Call SetBroker() in GRPCServer method
Verify EstablishHostServices is called from host

Host service calls fail

Check context is properly propagated
Verify paths are within allowed rootDir
Ensure client ID is set in gRPC metadata

Build errors

Run buf generate after proto changes
Check import paths match your module
Verify all dependencies are installed: go mod tidy

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Plugin Development

Plugin Development

Overview

Simple Plugin (No Host Services)

Example: Data Formatter Plugin

Plugin with Host Services

Example: Log Analyzer Plugin

Host Connection Interface

Adding New Host Service Functions

Best Practices

1. Use Structured Logging

2. Handle Context Cancellation

3. Close Resources Properly

4. Use Thread-Safe Access

5. Provide Sensible Defaults

Testing Your Plugin

Unit Tests

Integration Tests with Mock Host Services

Testing with the TUI

Common Patterns

Pattern: File Processing

Pattern: Temporary Files

Pattern: Directory Operations

Next Steps

Troubleshooting

Plugin won't start

Can't connect to host services

Host service calls fail

Build errors

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Clone this wiki locally