From d6df087ea1de28c0b6adc4c6114f571e47002c50 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 14:01:46 +0000 Subject: [PATCH 1/3] Initial plan From ad260de67bffdaf5f9823d4a596be49ed00d0fa9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 14:13:27 +0000 Subject: [PATCH 2/3] Add max-tokens and max-iterations flags to engine configuration Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/parser/schemas/main_workflow_schema.json | 37 ++++++++++- pkg/workflow/agent_validation.go | 68 +++++++++++++++++++- pkg/workflow/agentic_engine.go | 16 +++++ pkg/workflow/claude_engine.go | 12 ++-- pkg/workflow/codex_engine.go | 2 + pkg/workflow/compiler_orchestrator_tools.go | 10 +++ pkg/workflow/copilot_engine.go | 2 + pkg/workflow/custom_engine.go | 2 + pkg/workflow/engine.go | 50 ++++++++++---- 9 files changed, 179 insertions(+), 20 deletions(-) diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index 44c6ae1fc7..7745ec6fef 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -6307,6 +6307,15 @@ "model": "claude-3-5-sonnet-20241022", "max-turns": 15 }, + { + "id": "claude", + "model": "claude-3-5-sonnet-20241022", + "max-tokens": 4096 + }, + { + "id": "custom", + "max-iterations": 3 + }, { "id": "copilot", "version": "beta" @@ -6354,7 +6363,33 @@ "description": "Maximum number of chat iterations per run as a string value" } ], - "description": "Maximum number of chat iterations per run. Helps prevent runaway loops and control costs. Has sensible defaults and can typically be omitted. Note: Only supported by the claude engine." + "description": "Maximum number of chat iterations per run. Helps prevent runaway loops and control costs. Has sensible defaults and can typically be omitted. Note: Only supported by the claude and custom engines." + }, + "max-tokens": { + "oneOf": [ + { + "type": "integer", + "description": "Maximum number of tokens (input + output) per request as an integer value" + }, + { + "type": "string", + "description": "Maximum number of tokens (input + output) per request as a string value" + } + ], + "description": "Maximum number of tokens to use per request. Controls the context window size and response length. Has sensible defaults and can typically be omitted. Note: Only supported by the claude and custom engines." + }, + "max-iterations": { + "oneOf": [ + { + "type": "integer", + "description": "Maximum number of iterations per run as an integer value" + }, + { + "type": "string", + "description": "Maximum number of iterations per run as a string value" + } + ], + "description": "Maximum number of iterations per run. Similar to max-turns but used by some engines as an alias. Helps prevent runaway loops and control costs. Has sensible defaults and can typically be omitted. Note: Only supported by the custom engine." }, "concurrency": { "oneOf": [ diff --git a/pkg/workflow/agent_validation.go b/pkg/workflow/agent_validation.go index c0fca18022..b66740526a 100644 --- a/pkg/workflow/agent_validation.go +++ b/pkg/workflow/agent_validation.go @@ -133,7 +133,13 @@ func (c *Compiler) validateMaxTurnsSupport(frontmatter map[string]any, engine Co // max-turns is specified, check if the engine supports it if !engine.SupportsMaxTurns() { - return fmt.Errorf("max-turns not supported: engine '%s' does not support the max-turns feature. Use engine: copilot or remove max-turns from your configuration. Example:\nengine:\n id: copilot\n max-turns: 5", engine.GetID()) + errorMsg := fmt.Sprintf("max-turns not supported: engine '%s' does not support the max-turns feature. Supported engines: claude, custom. Example:\nengine:\n id: claude\n max-turns: 5", engine.GetID()) + if c.strictMode { + return fmt.Errorf("max-turns not supported: engine '%s' does not support the max-turns feature. Supported engines: claude, custom. Example:\nengine:\n id: claude\n max-turns: 5", engine.GetID()) + } + // In non-strict mode, issue a warning + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(errorMsg)) + c.IncrementWarningCount() } // Engine supports max-turns - additional validation could be added here if needed @@ -142,6 +148,66 @@ func (c *Compiler) validateMaxTurnsSupport(frontmatter map[string]any, engine Co return nil } +// validateMaxTokensSupport validates that max-tokens is only used with engines that support this feature +func (c *Compiler) validateMaxTokensSupport(frontmatter map[string]any, engine CodingAgentEngine) error { + // Check if max-tokens is specified in the engine config + engineSetting, engineConfig := c.ExtractEngineConfig(frontmatter) + _ = engineSetting // Suppress unused variable warning + + hasMaxTokens := engineConfig != nil && engineConfig.MaxTokens != "" + + if !hasMaxTokens { + // No max-tokens specified, no validation needed + return nil + } + + // max-tokens is specified, check if the engine supports it + if !engine.SupportsMaxTokens() { + errorMsg := fmt.Sprintf("max-tokens not supported: engine '%s' does not support the max-tokens feature. Supported engines: claude, custom. Example:\nengine:\n id: claude\n max-tokens: 4096", engine.GetID()) + if c.strictMode { + return fmt.Errorf("max-tokens not supported: engine '%s' does not support the max-tokens feature. Supported engines: claude, custom. Example:\nengine:\n id: claude\n max-tokens: 4096", engine.GetID()) + } + // In non-strict mode, issue a warning + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(errorMsg)) + c.IncrementWarningCount() + } + + // Engine supports max-tokens - additional validation could be added here if needed + // For now, we rely on JSON schema validation for format checking + + return nil +} + +// validateMaxIterationsSupport validates that max-iterations is only used with engines that support this feature +func (c *Compiler) validateMaxIterationsSupport(frontmatter map[string]any, engine CodingAgentEngine) error { + // Check if max-iterations is specified in the engine config + engineSetting, engineConfig := c.ExtractEngineConfig(frontmatter) + _ = engineSetting // Suppress unused variable warning + + hasMaxIterations := engineConfig != nil && engineConfig.MaxIterations != "" + + if !hasMaxIterations { + // No max-iterations specified, no validation needed + return nil + } + + // max-iterations is specified, check if the engine supports it + if !engine.SupportsMaxIterations() { + errorMsg := fmt.Sprintf("max-iterations not supported: engine '%s' does not support the max-iterations feature. Supported engines: custom. Example:\nengine:\n id: custom\n max-iterations: 3", engine.GetID()) + if c.strictMode { + return fmt.Errorf("max-iterations not supported: engine '%s' does not support the max-iterations feature. Supported engines: custom. Example:\nengine:\n id: custom\n max-iterations: 3", engine.GetID()) + } + // In non-strict mode, issue a warning + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(errorMsg)) + c.IncrementWarningCount() + } + + // Engine supports max-iterations - additional validation could be added here if needed + // For now, we rely on JSON schema validation for format checking + + return nil +} + // validateWebSearchSupport validates that web-search tool is only used with engines that support this feature func (c *Compiler) validateWebSearchSupport(tools map[string]any, engine CodingAgentEngine) { // Check if web-search tool is requested diff --git a/pkg/workflow/agentic_engine.go b/pkg/workflow/agentic_engine.go index 2c1a446549..607af9603a 100644 --- a/pkg/workflow/agentic_engine.go +++ b/pkg/workflow/agentic_engine.go @@ -116,6 +116,12 @@ type CapabilityProvider interface { // SupportsMaxTurns returns true if this engine supports the max-turns feature SupportsMaxTurns() bool + // SupportsMaxTokens returns true if this engine supports the max-tokens feature + SupportsMaxTokens() bool + + // SupportsMaxIterations returns true if this engine supports the max-iterations feature + SupportsMaxIterations() bool + // SupportsWebFetch returns true if this engine has built-in support for the web-fetch tool SupportsWebFetch() bool @@ -196,6 +202,8 @@ type BaseEngine struct { supportsToolsAllowlist bool supportsHTTPTransport bool supportsMaxTurns bool + supportsMaxTokens bool + supportsMaxIterations bool supportsWebFetch bool supportsWebSearch bool supportsFirewall bool @@ -229,6 +237,14 @@ func (e *BaseEngine) SupportsMaxTurns() bool { return e.supportsMaxTurns } +func (e *BaseEngine) SupportsMaxTokens() bool { + return e.supportsMaxTokens +} + +func (e *BaseEngine) SupportsMaxIterations() bool { + return e.supportsMaxIterations +} + func (e *BaseEngine) SupportsWebFetch() bool { return e.supportsWebFetch } diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index f788859daf..ca67fc41ea 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -25,11 +25,13 @@ func NewClaudeEngine() *ClaudeEngine { description: "Uses Claude Code with full MCP tool support and allow-listing", experimental: false, supportsToolsAllowlist: true, - supportsHTTPTransport: true, // Claude supports both stdio and HTTP transport - supportsMaxTurns: true, // Claude supports max-turns feature - supportsWebFetch: true, // Claude has built-in WebFetch support - supportsWebSearch: true, // Claude has built-in WebSearch support - supportsFirewall: true, // Claude supports network firewalling via AWF + supportsHTTPTransport: true, // Claude supports both stdio and HTTP transport + supportsMaxTurns: true, // Claude supports max-turns feature + supportsMaxTokens: true, // Claude supports max-tokens feature + supportsMaxIterations: false, // Claude uses max-turns instead + supportsWebFetch: true, // Claude has built-in WebFetch support + supportsWebSearch: true, // Claude has built-in WebSearch support + supportsFirewall: true, // Claude supports network firewalling via AWF }, } } diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index d1aebde3d1..c4208fb274 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -38,6 +38,8 @@ func NewCodexEngine() *CodexEngine { supportsToolsAllowlist: true, supportsHTTPTransport: true, // Codex now supports HTTP transport for remote MCP servers supportsMaxTurns: false, // Codex does not support max-turns feature + supportsMaxTokens: false, // Codex max-tokens needs verification + supportsMaxIterations: false, // Codex max-iterations needs verification supportsWebFetch: false, // Codex does not have built-in web-fetch support supportsWebSearch: true, // Codex has built-in web-search support supportsFirewall: true, // Codex supports network firewalling via AWF diff --git a/pkg/workflow/compiler_orchestrator_tools.go b/pkg/workflow/compiler_orchestrator_tools.go index 8e7086c332..de85ca5006 100644 --- a/pkg/workflow/compiler_orchestrator_tools.go +++ b/pkg/workflow/compiler_orchestrator_tools.go @@ -179,6 +179,16 @@ func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cle return nil, err } + // Validate max-tokens support for the current engine + if err := c.validateMaxTokensSupport(result.Frontmatter, agenticEngine); err != nil { + return nil, err + } + + // Validate max-iterations support for the current engine + if err := c.validateMaxIterationsSupport(result.Frontmatter, agenticEngine); err != nil { + return nil, err + } + // Validate web-search support for the current engine (warning only) c.validateWebSearchSupport(tools, agenticEngine) diff --git a/pkg/workflow/copilot_engine.go b/pkg/workflow/copilot_engine.go index cef6673186..31c6106105 100644 --- a/pkg/workflow/copilot_engine.go +++ b/pkg/workflow/copilot_engine.go @@ -42,6 +42,8 @@ func NewCopilotEngine() *CopilotEngine { supportsToolsAllowlist: true, supportsHTTPTransport: true, // Copilot CLI supports HTTP transport via MCP supportsMaxTurns: false, // Copilot CLI does not support max-turns feature yet + supportsMaxTokens: false, // Copilot CLI max-tokens needs verification + supportsMaxIterations: false, // Copilot CLI max-iterations needs verification supportsWebFetch: true, // Copilot CLI has built-in web-fetch support supportsWebSearch: false, // Copilot CLI does not have built-in web-search support supportsFirewall: true, // Copilot supports network firewalling via AWF diff --git a/pkg/workflow/custom_engine.go b/pkg/workflow/custom_engine.go index f938409a83..1e07311da9 100644 --- a/pkg/workflow/custom_engine.go +++ b/pkg/workflow/custom_engine.go @@ -24,6 +24,8 @@ func NewCustomEngine() *CustomEngine { supportsToolsAllowlist: false, supportsHTTPTransport: false, supportsMaxTurns: true, // Custom engine supports max-turns for consistency + supportsMaxTokens: true, // Custom engine supports max-tokens for consistency + supportsMaxIterations: true, // Custom engine supports max-iterations for consistency supportsWebFetch: false, // Custom engine does not have built-in web-fetch support supportsWebSearch: false, // Custom engine does not have built-in web-search support }, diff --git a/pkg/workflow/engine.go b/pkg/workflow/engine.go index d5484d5a4f..39fac9e458 100644 --- a/pkg/workflow/engine.go +++ b/pkg/workflow/engine.go @@ -12,19 +12,21 @@ var engineLog = logger.New("workflow:engine") // EngineConfig represents the parsed engine configuration type EngineConfig struct { - ID string - Version string - Model string - MaxTurns string - Concurrency string // Agent job-level concurrency configuration (YAML format) - UserAgent string - Command string // Custom executable path (when set, skip installation steps) - Env map[string]string - Steps []map[string]any - Config string - Args []string - Firewall *FirewallConfig // AWF firewall configuration - Agent string // Agent identifier for copilot --agent flag (copilot engine only) + ID string + Version string + Model string + MaxTurns string + MaxTokens string // Maximum number of tokens (input + output) per request + MaxIterations string // Maximum number of iterations (alias for MaxTurns for some engines) + Concurrency string // Agent job-level concurrency configuration (YAML format) + UserAgent string + Command string // Custom executable path (when set, skip installation steps) + Env map[string]string + Steps []map[string]any + Config string + Args []string + Firewall *FirewallConfig // AWF firewall configuration + Agent string // Agent identifier for copilot --agent flag (copilot engine only) } // NetworkPermissions represents network access permissions for workflow execution @@ -116,6 +118,28 @@ func (c *Compiler) ExtractEngineConfig(frontmatter map[string]any) (string, *Eng } } + // Extract optional 'max-tokens' field + if maxTokens, hasMaxTokens := engineObj["max-tokens"]; hasMaxTokens { + if maxTokensInt, ok := maxTokens.(int); ok { + config.MaxTokens = fmt.Sprintf("%d", maxTokensInt) + } else if maxTokensUint64, ok := maxTokens.(uint64); ok { + config.MaxTokens = fmt.Sprintf("%d", maxTokensUint64) + } else if maxTokensStr, ok := maxTokens.(string); ok { + config.MaxTokens = maxTokensStr + } + } + + // Extract optional 'max-iterations' field + if maxIterations, hasMaxIterations := engineObj["max-iterations"]; hasMaxIterations { + if maxIterationsInt, ok := maxIterations.(int); ok { + config.MaxIterations = fmt.Sprintf("%d", maxIterationsInt) + } else if maxIterationsUint64, ok := maxIterations.(uint64); ok { + config.MaxIterations = fmt.Sprintf("%d", maxIterationsUint64) + } else if maxIterationsStr, ok := maxIterations.(string); ok { + config.MaxIterations = maxIterationsStr + } + } + // Extract optional 'concurrency' field (string or object format) if concurrency, hasConcurrency := engineObj["concurrency"]; hasConcurrency { if concurrencyStr, ok := concurrency.(string); ok { From c1970bce5cd3ec7bdc3537a45df6a219053babee Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 14:16:50 +0000 Subject: [PATCH 3/3] Add comprehensive tests for execution bounding flags Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../execution_bounds_validation_test.go | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 pkg/workflow/execution_bounds_validation_test.go diff --git a/pkg/workflow/execution_bounds_validation_test.go b/pkg/workflow/execution_bounds_validation_test.go new file mode 100644 index 0000000000..c002fdcb09 --- /dev/null +++ b/pkg/workflow/execution_bounds_validation_test.go @@ -0,0 +1,319 @@ +//go:build integration + +package workflow + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/github/gh-aw/pkg/testutil" +) + +func TestMaxTokensValidationWithUnsupportedEngine(t *testing.T) { + tests := []struct { + name string + content string + engine string + expectError bool + errorMsg string + }{ + { + name: "max-tokens with codex engine should fail", + content: `--- +on: + workflow_dispatch: +permissions: + contents: read + issues: read + pull-requests: read +engine: + id: codex + max-tokens: 4096 +--- + +# Test Workflow + +This should fail because codex doesn't support max-tokens.`, + engine: "codex", + expectError: true, + errorMsg: "max-tokens not supported: engine 'codex' does not support the max-tokens feature", + }, + { + name: "max-tokens with claude engine should succeed", + content: `--- +on: + workflow_dispatch: +permissions: + contents: read + issues: read + pull-requests: read +engine: + id: claude + max-tokens: 4096 +--- + +# Test Workflow + +This should succeed because claude supports max-tokens.`, + engine: "claude", + expectError: false, + }, + { + name: "max-tokens with custom engine should succeed", + content: `--- +on: + workflow_dispatch: +permissions: + contents: read + issues: read + pull-requests: read +engine: + id: custom + max-tokens: 4096 + steps: + - name: Test Step + run: echo "test" +--- + +# Test Workflow + +This should succeed because custom supports max-tokens.`, + engine: "custom", + expectError: false, + }, + { + name: "codex engine without max-tokens should succeed", + content: `--- +on: + workflow_dispatch: +permissions: + contents: read + issues: read + pull-requests: read +engine: codex +--- + +# Test Workflow + +This should succeed because no max-tokens is specified.`, + engine: "codex", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a temporary directory for the test + tmpDir := testutil.TempDir(t, "max-tokens-validation-test") + + // Create a test workflow file + testFile := filepath.Join(tmpDir, "test.md") + if err := os.WriteFile(testFile, []byte(tt.content), 0644); err != nil { + t.Fatal(err) + } + + // Create a compiler instance + compiler := NewCompiler() + compiler.SetSkipValidation(false) + compiler.SetStrictMode(true) // Enable strict mode for validation errors + + // Try to compile the workflow + err := compiler.CompileWorkflow(testFile) + + if tt.expectError { + // We expect an error + if err == nil { + t.Errorf("Expected error but compilation succeeded") + return + } + + // Check if the error message contains the expected text + if !strings.Contains(err.Error(), tt.errorMsg) { + t.Errorf("Expected error message to contain '%s', but got: %s", tt.errorMsg, err.Error()) + } + } else { + // We don't expect an error + if err != nil { + t.Errorf("Expected compilation to succeed but got error: %v", err) + } + } + }) + } +} + +func TestMaxIterationsValidationWithUnsupportedEngine(t *testing.T) { + tests := []struct { + name string + content string + engine string + expectError bool + errorMsg string + }{ + { + name: "max-iterations with claude engine should fail", + content: `--- +on: + workflow_dispatch: +permissions: + contents: read + issues: read + pull-requests: read +engine: + id: claude + max-iterations: 3 +--- + +# Test Workflow + +This should fail because claude doesn't support max-iterations.`, + engine: "claude", + expectError: true, + errorMsg: "max-iterations not supported: engine 'claude' does not support the max-iterations feature", + }, + { + name: "max-iterations with custom engine should succeed", + content: `--- +on: + workflow_dispatch: +permissions: + contents: read + issues: read + pull-requests: read +engine: + id: custom + max-iterations: 3 + steps: + - name: Test Step + run: echo "test" +--- + +# Test Workflow + +This should succeed because custom supports max-iterations.`, + engine: "custom", + expectError: false, + }, + { + name: "codex engine without max-iterations should succeed", + content: `--- +on: + workflow_dispatch: +permissions: + contents: read + issues: read + pull-requests: read +engine: codex +--- + +# Test Workflow + +This should succeed because no max-iterations is specified.`, + engine: "codex", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a temporary directory for the test + tmpDir := testutil.TempDir(t, "max-iterations-validation-test") + + // Create a test workflow file + testFile := filepath.Join(tmpDir, "test.md") + if err := os.WriteFile(testFile, []byte(tt.content), 0644); err != nil { + t.Fatal(err) + } + + // Create a compiler instance + compiler := NewCompiler() + compiler.SetSkipValidation(false) + compiler.SetStrictMode(true) // Enable strict mode for validation errors + + // Try to compile the workflow + err := compiler.CompileWorkflow(testFile) + + if tt.expectError { + // We expect an error + if err == nil { + t.Errorf("Expected error but compilation succeeded") + return + } + + // Check if the error message contains the expected text + if !strings.Contains(err.Error(), tt.errorMsg) { + t.Errorf("Expected error message to contain '%s', but got: %s", tt.errorMsg, err.Error()) + } + } else { + // We don't expect an error + if err != nil { + t.Errorf("Expected compilation to succeed but got error: %v", err) + } + } + }) + } +} + +func TestEngineSupportsExecutionBounds(t *testing.T) { + tests := []struct { + engineID string + supportsMaxTurns bool + supportsMaxTokens bool + supportsMaxIterations bool + }{ + { + engineID: "claude", + supportsMaxTurns: true, + supportsMaxTokens: true, + supportsMaxIterations: false, + }, + { + engineID: "copilot", + supportsMaxTurns: false, + supportsMaxTokens: false, + supportsMaxIterations: false, + }, + { + engineID: "codex", + supportsMaxTurns: false, + supportsMaxTokens: false, + supportsMaxIterations: false, + }, + { + engineID: "custom", + supportsMaxTurns: true, + supportsMaxTokens: true, + supportsMaxIterations: true, + }, + } + + registry := GetGlobalEngineRegistry() + + for _, tt := range tests { + t.Run(tt.engineID, func(t *testing.T) { + engine, err := registry.GetEngine(tt.engineID) + if err != nil { + t.Fatalf("Failed to get engine '%s': %v", tt.engineID, err) + } + + actualMaxTurns := engine.SupportsMaxTurns() + if actualMaxTurns != tt.supportsMaxTurns { + t.Errorf("Expected engine '%s' to have SupportsMaxTurns() = %v, but got %v", + tt.engineID, tt.supportsMaxTurns, actualMaxTurns) + } + + actualMaxTokens := engine.SupportsMaxTokens() + if actualMaxTokens != tt.supportsMaxTokens { + t.Errorf("Expected engine '%s' to have SupportsMaxTokens() = %v, but got %v", + tt.engineID, tt.supportsMaxTokens, actualMaxTokens) + } + + actualMaxIterations := engine.SupportsMaxIterations() + if actualMaxIterations != tt.supportsMaxIterations { + t.Errorf("Expected engine '%s' to have SupportsMaxIterations() = %v, but got %v", + tt.engineID, tt.supportsMaxIterations, actualMaxIterations) + } + }) + } +}