From ea4e19a180bb69c8e1b54d995d9cbdc912f71fc9 Mon Sep 17 00:00:00 2001 From: XD Date: Wed, 19 Aug 2026 15:37:21 +0800 Subject: [PATCH] Add JSON output to vMCP validation Expose a stable, secret-free validation summary so CI and Agent harnesses can consume vMCP validation results without parsing human-readable logs. Preserve the existing text behavior while adding format validation, focused tests, and generated CLI documentation. Signed-off-by: XD --- cmd/thv/app/vmcp.go | 5 ++ cmd/thv/app/vmcp_test.go | 49 +++++++++++++++ docs/cli/thv_vmcp_validate.md | 1 + pkg/vmcp/cli/validate.go | 47 ++++++++++++++ pkg/vmcp/cli/validate_test.go | 115 ++++++++++++++++++++++++++++++++++ 5 files changed, 217 insertions(+) diff --git a/cmd/thv/app/vmcp.go b/cmd/thv/app/vmcp.go index 26d07296e8..f6fc2ae682 100644 --- a/cmd/thv/app/vmcp.go +++ b/cmd/thv/app/vmcp.go @@ -129,6 +129,7 @@ If neither --output nor --config is provided, the generated YAML is written to s // newVMCPValidateCommand returns the "vmcp validate" subcommand. func newVMCPValidateCommand() *cobra.Command { var configPath string + var format string cmd := &cobra.Command{ Use: "validate", Short: "Validate a vMCP configuration file", @@ -141,10 +142,14 @@ for valid configurations, non-zero with a descriptive error otherwise.`, RunE: func(cmd *cobra.Command, _ []string) error { return vmcpcli.Validate(cmd.Context(), vmcpcli.ValidateConfig{ ConfigPath: configPath, + Format: format, + Writer: cmd.OutOrStdout(), }) }, } cmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to vMCP configuration file (required)") + AddFormatFlag(cmd, &format) + cmd.PreRunE = ValidateFormat(&format) _ = cmd.MarkFlagRequired("config") return cmd } diff --git a/cmd/thv/app/vmcp_test.go b/cmd/thv/app/vmcp_test.go index 24ce1ba7d2..8dc54d0a53 100644 --- a/cmd/thv/app/vmcp_test.go +++ b/cmd/thv/app/vmcp_test.go @@ -4,6 +4,10 @@ package app import ( + "bytes" + "encoding/json" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -56,3 +60,48 @@ func TestNewVMCPCommand_InitRegistered(t *testing.T) { } assert.True(t, found, "expected 'init' to be registered as a subcommand of 'vmcp'") } + +func TestNewVMCPValidateCommand_FormatFlag(t *testing.T) { + t.Parallel() + + cmd := newVMCPValidateCommand() + formatFlag := cmd.Flags().Lookup("format") + require.NotNil(t, formatFlag, "expected --format flag to be registered") + assert.Equal(t, FormatText, formatFlag.DefValue) + + require.NoError(t, formatFlag.Value.Set("yaml")) + err := cmd.PreRunE(cmd, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), `invalid format "yaml"`) +} + +func TestNewVMCPValidateCommand_JSONOutput(t *testing.T) { + t.Parallel() + + configPath := filepath.Join(t.TempDir(), "vmcp.yaml") + require.NoError(t, os.WriteFile(configPath, []byte(` +name: cli-json-vmcp +groupRef: cli-json-group +incomingAuth: + type: anonymous +outgoingAuth: + source: inline + default: + type: unauthenticated +aggregation: + conflictResolution: prefix + conflictResolutionConfig: + prefixFormat: "{workload}_" +`), 0o600)) + + cmd := newVMCPValidateCommand() + var output bytes.Buffer + cmd.SetOut(&output) + cmd.SetArgs([]string{"--config", configPath, "--format", FormatJSON}) + require.NoError(t, cmd.Execute()) + + var summary map[string]any + require.NoError(t, json.Unmarshal(output.Bytes(), &summary)) + assert.Equal(t, true, summary["valid"]) + assert.Equal(t, "cli-json-vmcp", summary["name"]) +} diff --git a/docs/cli/thv_vmcp_validate.md b/docs/cli/thv_vmcp_validate.md index 0f88282be0..2cfbcc8922 100644 --- a/docs/cli/thv_vmcp_validate.md +++ b/docs/cli/thv_vmcp_validate.md @@ -29,6 +29,7 @@ thv vmcp validate [flags] ``` -c, --config string Path to vMCP configuration file (required) + --format string Output format (json, text) (default "text") -h, --help help for validate ``` diff --git a/pkg/vmcp/cli/validate.go b/pkg/vmcp/cli/validate.go index 594c041670..8d60732b0e 100644 --- a/pkg/vmcp/cli/validate.go +++ b/pkg/vmcp/cli/validate.go @@ -5,8 +5,11 @@ package cli import ( "context" + "encoding/json" "fmt" + "io" "log/slog" + "os" "github.com/stacklok/toolhive-core/env" "github.com/stacklok/toolhive/pkg/vmcp/config" @@ -16,6 +19,23 @@ import ( type ValidateConfig struct { // ConfigPath is the path to the vMCP YAML configuration file to validate. ConfigPath string + // Format controls successful output. Empty or "text" preserves the log summary. + Format string + // Writer receives JSON output. Nil uses os.Stdout. + Writer io.Writer +} + +// ValidationSummary is the stable, secret-free machine-readable validation result. +type ValidationSummary struct { + Valid bool `json:"valid"` + Name string `json:"name"` + Group string `json:"group"` + IncomingAuth string `json:"incoming_auth"` + OutgoingAuthSource string `json:"outgoing_auth_source"` + BackendAuthOverrides int `json:"backend_auth_override_count"` + BackendCount int `json:"backend_count"` + ConflictResolution string `json:"conflict_resolution"` + CompositeToolCount int `json:"composite_tool_count"` } // Validate loads and validates a vMCP configuration file, printing a summary @@ -25,6 +45,9 @@ func Validate(_ context.Context, cfg ValidateConfig) error { if cfg.ConfigPath == "" { return fmt.Errorf("no configuration file specified, use --config flag") } + if cfg.Format != "" && cfg.Format != "text" && cfg.Format != "json" { + return fmt.Errorf("unsupported output format %q", cfg.Format) + } slog.Info(fmt.Sprintf("Validating configuration: %s", cfg.ConfigPath)) @@ -44,6 +67,30 @@ func Validate(_ context.Context, cfg ValidateConfig) error { return fmt.Errorf("validation failed: %w", err) } + summary := ValidationSummary{ + Valid: true, + Name: vmcpCfg.Name, + Group: vmcpCfg.Group, + IncomingAuth: vmcpCfg.IncomingAuth.Type, + OutgoingAuthSource: vmcpCfg.OutgoingAuth.Source, + BackendAuthOverrides: len(vmcpCfg.OutgoingAuth.Backends), + BackendCount: len(vmcpCfg.Backends), + ConflictResolution: string(vmcpCfg.Aggregation.ConflictResolution), + CompositeToolCount: len(vmcpCfg.CompositeTools), + } + if cfg.Format == "json" { + writer := cfg.Writer + if writer == nil { + writer = os.Stdout + } + encoder := json.NewEncoder(writer) + encoder.SetIndent("", " ") + if err := encoder.Encode(summary); err != nil { + return fmt.Errorf("failed to encode validation summary: %w", err) + } + return nil + } + slog.Info("✓ Configuration is valid") slog.Info(fmt.Sprintf(" Name: %s", vmcpCfg.Name)) slog.Info(fmt.Sprintf(" Group: %s", vmcpCfg.Group)) diff --git a/pkg/vmcp/cli/validate_test.go b/pkg/vmcp/cli/validate_test.go index 63e0e309b7..23fb417c09 100644 --- a/pkg/vmcp/cli/validate_test.go +++ b/pkg/vmcp/cli/validate_test.go @@ -4,11 +4,15 @@ package cli import ( + "bytes" "context" + "encoding/json" + "log/slog" "os" "path/filepath" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -115,3 +119,114 @@ aggregation: }) } } + +func TestValidateJSONSummaryIsStableAndSecretFree(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "vmcp.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +name: sensitive-vmcp +groupRef: sensitive-group + +backends: + - name: first-backend + url: http://first.example.test/mcp + transport: streamable-http + - name: second-backend + url: http://second.example.test/sse + transport: sse + +incomingAuth: + type: anonymous + +outgoingAuth: + source: inline + default: + type: unauthenticated + backends: + protected-backend: + type: header_injection + headerInjection: + headerName: X-Private-API-Key + headerValue: vmcp-json-secret-value + +aggregation: + conflictResolution: prefix + conflictResolutionConfig: + prefixFormat: "{workload}_" + +telemetry: + headers: + X-Telemetry-Token: vmcp-telemetry-secret-value + +passthroughHeaders: + - X-Tenant-Token +`), 0o600)) + + var output bytes.Buffer + err := Validate(context.Background(), ValidateConfig{ + ConfigPath: path, + Format: "json", + Writer: &output, + }) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(output.Bytes(), &got)) + assert.Equal(t, map[string]any{ + "valid": true, + "name": "sensitive-vmcp", + "group": "sensitive-group", + "incoming_auth": "anonymous", + "outgoing_auth_source": "inline", + "backend_auth_override_count": float64(1), + "backend_count": float64(2), + "conflict_resolution": "prefix", + "composite_tool_count": float64(0), + }, got) + + for _, sensitiveValue := range []string{ + "vmcp-json-secret-value", + "vmcp-telemetry-secret-value", + "X-Private-API-Key", + "X-Telemetry-Token", + "X-Tenant-Token", + "first.example.test", + "second.example.test", + } { + assert.NotContains(t, output.String(), sensitiveValue) + } +} + +//nolint:paralleltest // slog.SetDefault is process-wide and must be restored before parallel tests run. +func TestValidateTextOutputRemainsCompatible(t *testing.T) { + path := filepath.Join(t.TempDir(), "vmcp.yaml") + require.NoError(t, os.WriteFile(path, []byte(validConfigYAML), 0o600)) + + var logs bytes.Buffer + previousLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previousLogger) }) + + for _, format := range []string{"", "text"} { + logs.Reset() + var output bytes.Buffer + require.NoError(t, Validate(context.Background(), ValidateConfig{ + ConfigPath: path, + Format: format, + Writer: &output, + })) + + assert.Empty(t, output.String(), "text output must not be written to the JSON writer") + for _, summaryLine := range []string{ + "Configuration is valid", + "Name: test-vmcp", + "Group: test-group", + "Incoming Auth: anonymous", + "Outgoing Auth: default only (source: inline)", + "Conflict Resolution: prefix", + } { + assert.Contains(t, logs.String(), summaryLine) + } + } +}