diff --git a/.cursor/rules/vendors-in-guest.mdc b/.cursor/rules/vendors-in-guest.mdc index eea3c1ea..d6f7a35e 100644 --- a/.cursor/rules/vendors-in-guest.mdc +++ b/.cursor/rules/vendors-in-guest.mdc @@ -9,6 +9,6 @@ The factory builds Go from vendor docs. That guest owns how to connect, send, an The host must not implement Slack, Telegram, Discord, WhatsApp, or other vendor HTTP — no form encoders, users.info, auth.test, conversation-list filters, bundled `contracts/vendors/*.json`, or vendor live/E2E harnesses in this repo. -Host work: forward `http.request` as declared, attach declared secrets as Bearer/Basic, persist `result.emit`, render HostUI. +Do not add code of any kind, for any reason, to fix a vendor-specific cause. That includes host code, contract text, prompts, and hand-edits to one installed guest (error codes, scopes, ids, query flags, display names). If an override is needed, David will say so explicitly. Otherwise do not do it. -If David asks for host-side vendor protocol code, tests, or harnesses, warn first. +Host work: forward `http.request` as declared, attach declared secrets as Bearer/Basic, persist `result.emit`, render HostUI. A live acceptance test performs those same forwards. It must not know which vendor it is calling. diff --git a/packages/LLMAgentClient/Sources/LLMAgentClient/GeminiProvider.swift b/packages/LLMAgentClient/Sources/LLMAgentClient/GeminiProvider.swift index 1a5c350d..92967461 100644 --- a/packages/LLMAgentClient/Sources/LLMAgentClient/GeminiProvider.swift +++ b/packages/LLMAgentClient/Sources/LLMAgentClient/GeminiProvider.swift @@ -35,6 +35,13 @@ public enum GeminiModel: String, CaseIterable, Codable, Sendable, AgentModel { return ModelTokenPricing(inputUSDPer1MTokens: 1.50, outputUSDPer1MTokens: 7.50) } } + + public var requestSupport: ModelRequestSupport { + switch self { + case .gemini25FlashLite, .gemini31FlashLite, .gemini37Flash: + return ModelRequestSupport(temperature: true) + } + } } public struct GeminiProvider: AgentProvider { @@ -59,10 +66,11 @@ public struct GeminiProvider: AgentProvider { urlRequest.httpMethod = "POST" urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") urlRequest.setValue("text/event-stream", forHTTPHeaderField: "Accept") + let allowed = request.constrained(to: model) urlRequest.httpBody = try encode(GeminiStreamRequest( - messages: request.messages, - temperature: request.temperature, - thinking: request.thinking + messages: allowed.messages, + temperature: allowed.temperature, + thinking: allowed.thinking )) let (bytes, response) = try await transport.bytes(for: urlRequest) @@ -105,11 +113,12 @@ public struct GeminiProvider: AgentProvider { urlRequest.httpMethod = "POST" urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") urlRequest.setValue("text/event-stream", forHTTPHeaderField: "Accept") + let allowed = request.constrained(to: model) urlRequest.httpBody = try encode(GeminiJSONStreamRequest( - messages: request.messages, - temperature: request.temperature, - responseSchema: responseSchema, - thinking: request.thinking + messages: allowed.messages, + temperature: allowed.temperature, + responseSchema: responseSchema ?? allowed.responseSchema, + thinking: allowed.thinking )) let (bytes, response) = try await transport.bytes(for: urlRequest) diff --git a/packages/LLMAgentClient/Sources/LLMAgentClient/ModelThinking.swift b/packages/LLMAgentClient/Sources/LLMAgentClient/ModelThinking.swift index 5d0d2f6c..087686a7 100644 --- a/packages/LLMAgentClient/Sources/LLMAgentClient/ModelThinking.swift +++ b/packages/LLMAgentClient/Sources/LLMAgentClient/ModelThinking.swift @@ -35,6 +35,10 @@ extension OpenAIModel { efforts = [.none, .low, .medium, .high, .xhigh] case .gpt56Sol: efforts = [.none, .low, .medium, .high, .xhigh, .max] + case .gpt6Luna, .gpt6Sol: + efforts = [.none, .low, .medium, .high, .xhigh, .max] + case .gpt6Astra: + efforts = [.low, .medium, .high, .xhigh, .max] } return efforts.map { effort in ModelThinkingOption( @@ -49,6 +53,10 @@ extension OpenAIModel { thinkingOptions.first { $0.id == ReasoningEffort.medium.rawValue } ?? thinkingOptions[0] } + + public func acceptsThinking(_ option: ModelThinkingOption) -> Bool { + thinkingOptions.contains { $0.id == option.id && $0.wire == option.wire } + } } extension GeminiModel { @@ -95,6 +103,10 @@ extension GeminiModel { } } + public func acceptsThinking(_ option: ModelThinkingOption) -> Bool { + thinkingOptions.contains { $0.id == option.id && $0.wire == option.wire } + } + private static func levelOptions(_ levels: [ThinkingLevel]) -> [ModelThinkingOption] { levels.map { level in ModelThinkingOption( diff --git a/packages/LLMAgentClient/Sources/LLMAgentClient/OpenAIProvider.swift b/packages/LLMAgentClient/Sources/LLMAgentClient/OpenAIProvider.swift index 0ec14093..bad54ce0 100644 --- a/packages/LLMAgentClient/Sources/LLMAgentClient/OpenAIProvider.swift +++ b/packages/LLMAgentClient/Sources/LLMAgentClient/OpenAIProvider.swift @@ -9,17 +9,31 @@ public enum OpenAIModel: String, CaseIterable, Codable, Sendable, AgentModel { case gpt56Sol = "gpt-5.6-sol" case gpt56Terra = "gpt-5.6-terra" case gpt56Luna = "gpt-5.6-luna" + /// GPT-6 capability tiers. + case gpt6Luna = "gpt-6-luna" + case gpt6Sol = "gpt-6-sol" + case gpt6Astra = "gpt-6-astra" public var id: AgentModelID { .init(provider: "openai", name: rawValue) } public var maxSupportedContextTokens: Int { - 400_000 + switch self { + case .gpt6Luna, .gpt6Sol, .gpt6Astra: + return 1_050_000 + default: + return 400_000 + } } public var maxIdealContextTokens: Int { - 200_000 + switch self { + case .gpt6Luna, .gpt6Sol, .gpt6Astra: + return 272_000 + default: + return 200_000 + } } /// Approximate list prices (USD / 1M tokens). Update when OpenAI changes rates. @@ -37,6 +51,20 @@ public enum OpenAIModel: String, CaseIterable, Codable, Sendable, AgentModel { return ModelTokenPricing(inputUSDPer1MTokens: 1.25, outputUSDPer1MTokens: 10.00) case .gpt56Sol: return ModelTokenPricing(inputUSDPer1MTokens: 2.50, outputUSDPer1MTokens: 15.00) + case .gpt6Luna: + return ModelTokenPricing(inputUSDPer1MTokens: 0.10, outputUSDPer1MTokens: 0.50) + case .gpt6Sol: + return ModelTokenPricing(inputUSDPer1MTokens: 2.00, outputUSDPer1MTokens: 10.00) + case .gpt6Astra: + return ModelTokenPricing(inputUSDPer1MTokens: 10.00, outputUSDPer1MTokens: 50.00) + } + } + + /// These models reject a caller-supplied temperature and use their own default. + public var requestSupport: ModelRequestSupport { + switch self { + case .gpt54Mini, .gpt54, .gpt55, .gpt56Sol, .gpt56Terra, .gpt56Luna, .gpt6Luna, .gpt6Sol, .gpt6Astra: + return ModelRequestSupport(temperature: false) } } } @@ -60,12 +88,13 @@ public struct OpenAIProvider: AgentProvider { urlRequest.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") urlRequest.setValue("text/event-stream", forHTTPHeaderField: "Accept") + let allowed = request.constrained(to: model) urlRequest.httpBody = try encode(OpenAIStreamRequest( model: model.rawValue, - messages: request.messages, - temperature: request.temperature, - responseSchema: request.responseSchema, - thinking: request.thinking + messages: allowed.messages, + temperature: allowed.temperature, + responseSchema: allowed.responseSchema, + thinking: allowed.thinking )) let (bytes, response) = try await transport.bytes(for: urlRequest) @@ -128,13 +157,7 @@ struct OpenAIStreamRequest: Encodable { self.messages = messages.map(OpenAIMessage.init) self.stream = true self.streamOptions = OpenAIStreamOptions(includeUsage: true) - - // OpenAI's reasoning-class models (GPT-5 series) lock temperature internally and reject manual settings with HTTP 400. - if model.contains("gpt-5") { - self.temperature = nil - } else { - self.temperature = temperature - } + self.temperature = temperature if case .openAIReasoningEffort(let effort) = thinking?.wire { self.reasoningEffort = effort diff --git a/packages/LLMAgentClient/Tests/LLMAgentClientTests/ModelTests.swift b/packages/LLMAgentClient/Tests/LLMAgentClientTests/ModelTests.swift index 600480fc..f8becae9 100644 --- a/packages/LLMAgentClient/Tests/LLMAgentClientTests/ModelTests.swift +++ b/packages/LLMAgentClient/Tests/LLMAgentClientTests/ModelTests.swift @@ -7,7 +7,9 @@ import Testing struct ModelTests { @Test func openAIModelIdentifier() { #expect(OpenAIModel.gpt56Luna.id.provider == "openai") - #expect(OpenAIModel.gpt56Luna.id.rawValue == "gpt-5.6-luna") + #expect(OpenAIModel.gpt6Luna.id.rawValue == "gpt-6-luna") + #expect(OpenAIModel.gpt6Sol.id.rawValue == "gpt-6-sol") + #expect(OpenAIModel.gpt6Astra.id.rawValue == "gpt-6-astra") } @Test func geminiModelIdentifier() { @@ -187,18 +189,59 @@ struct ModelTests { } @Test func openAIReasoningEffortSerialization() throws { - let request = OpenAIStreamRequest( - model: "gpt-5.6-sol", + let model = OpenAIModel.gpt56Sol + let allowed = AgentRequest( messages: [.init(role: .user, content: "hello")], temperature: 0.1, - responseSchema: nil, - thinking: OpenAIModel.gpt56Sol.thinkingOptions.first { $0.id == "high" } + thinking: model.thinkingOptions.first { $0.id == "high" } + ).constrained(to: model) + let request = OpenAIStreamRequest( + model: model.rawValue, + messages: allowed.messages, + temperature: allowed.temperature, + responseSchema: allowed.responseSchema, + thinking: allowed.thinking ) let data = try JSONEncoder().encode(request) let jsonString = String(decoding: data, as: UTF8.self) #expect(jsonString.contains("reasoning_effort")) #expect(jsonString.contains("high")) + #expect(jsonString.contains("temperature") == false) + } + + @Test func providerSendsOnlyFieldsTheModelAccepts() throws { + let model = OpenAIModel.gpt6Astra + #expect(model.requestSupport.temperature == false) + let unsupported = ModelThinkingOption( + id: "none", + displayName: "None", + wire: .openAIReasoningEffort("none") + ) + let allowed = AgentRequest( + messages: [.init(role: .user, content: "hello")], + temperature: 0, + thinking: unsupported + ).constrained(to: model) + #expect(allowed.temperature == nil) + #expect(allowed.thinking == nil) + let encoded = OpenAIStreamRequest( + model: model.rawValue, + messages: allowed.messages, + temperature: allowed.temperature, + responseSchema: nil, + thinking: allowed.thinking + ) + let json = String(decoding: try JSONEncoder().encode(encoded), as: UTF8.self) + #expect(!json.contains("temperature")) + #expect(!json.contains("reasoning_effort")) + + let gemini = AgentRequest( + messages: [.init(role: .user, content: "hello")], + temperature: 0 + ).constrained(to: GeminiModel.gemini37Flash) + #expect(GeminiModel.gemini37Flash.requestSupport.temperature) + #expect(gemini.temperature == 0) } @Test func geminiThinkingBudgetSerialization() throws { diff --git a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift index 64973538..96e230ed 100644 --- a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift +++ b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift @@ -6,7 +6,7 @@ import Structure public actor LiveFactoryBuilder: PluginFactoryBuilder { private let apiKey: String - private let model: OpenAIModel = .gpt56Luna + private let model: OpenAIModel = .gpt6Luna public init(apiKey: String) { self.apiKey = apiKey @@ -104,7 +104,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { public actor LiveFactoryReviewer: PluginFactoryReviewer { private let apiKey: String - private let model: OpenAIModel = .gpt56Luna + private let model: OpenAIModel = .gpt6Luna public init(apiKey: String) { self.apiKey = apiKey diff --git a/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift index cc8c9ab6..359d4163 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift @@ -3,7 +3,7 @@ import Plugin import Structure /// Production adapter for the Go plugin factory. -public struct GoPluginFactoryDockerExecutor: PluginFactoryExecutor, PluginFactoryCompiledGuestExecutor, Sendable { +public struct GoPluginFactoryDockerExecutor: PluginFactoryExecutor, PluginFactoryCompiledGuestExecutor, PluginFactoryLiveAcceptanceExecutor, Sendable { private let runtime: GoGuestDockerExecutor public var image: String { runtime.image } @@ -48,6 +48,42 @@ public struct GoPluginFactoryDockerExecutor: PluginFactoryExecutor, PluginFactor } } + public func runLiveAcceptance( + artifact: Data, + testInput: Data + ) async throws -> PluginFactoryHopTestRun { + let script = try PluginFactoryTestScript.parse(testInput) + let liveHops = script.hops.filter { $0.kind != .httpResults && $0.httpResults?.isEmpty != false } + if liveHops.isEmpty { + let result = PluginFactoryExecutionResult( + exitCode: 1, + stderr: Data( + "Live acceptance needs a hop that is not a fixture http_results body.".utf8 + ) + ) + return PluginFactoryHopTestRun(final: result, hopResults: [result]) + } + var hopResults: [PluginFactoryExecutionResult] = [] + var lastResult = PluginFactoryExecutionResult(exitCode: 1) + var threadID: String? + for hop in liveHops { + let live = PluginLiveAcceptance.prepared(hop, discoveredThreadID: threadID) + let input = try live.encodeValidated() + let result = try await PluginHostHopDispatcher.run(initialInput: input) { hopInput in + try await self.runtime.runArtifact(artifact: artifact, input: hopInput) + } + hopResults.append(result) + lastResult = result + if threadID == nil { + threadID = PluginLiveAcceptance.firstThreadID( + in: String(decoding: result.stdout, as: UTF8.self) + ) + } + guard result.exitCode == 0 else { break } + } + return PluginFactoryHopTestRun(final: lastResult, hopResults: hopResults) + } + public func packageGuestSource(source: String) async throws -> Data { try await runtime.compileSource(source) } diff --git a/packages/MCPServer/Sources/MCPServer/Script/OpenAIModelReviewer.swift b/packages/MCPServer/Sources/MCPServer/Script/OpenAIModelReviewer.swift index 23e7bec5..74e7a310 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/OpenAIModelReviewer.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/OpenAIModelReviewer.swift @@ -17,7 +17,7 @@ public struct OpenAIScriptReviewer: ScriptReviewer { public init( apiKey: String, - model: OpenAIModel = .gpt56Luna, + model: OpenAIModel = .gpt6Luna, systemPrompt: String = ReviewerSystemPrompt ) { self.name = "openai-\(model.rawValue)" @@ -28,7 +28,7 @@ public struct OpenAIScriptReviewer: ScriptReviewer { public static func fromEnvironment( variable: String = "OPENAI_API_KEY", - model: OpenAIModel = .gpt56Luna + model: OpenAIModel = .gpt6Luna ) -> OpenAIScriptReviewer? { guard let apiKey = ProcessInfo.processInfo.environment[variable], !apiKey.isEmpty else { return nil diff --git a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift index 88b5342f..0bf8f7c6 100644 --- a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift +++ b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift @@ -58,7 +58,8 @@ public struct PluginFactorySession: Sendable { hostManifest: hostManifest ) } catch { - if ModelProviderLimit.matches(error.localizedDescription) { + if ModelProviderLimit.matches(error.localizedDescription) + || Self.isProviderHTTPFailure(error.localizedDescription) { throw error } let wrapped = PluginFactoryError.invalidSource(error.localizedDescription) @@ -85,6 +86,14 @@ public struct PluginFactorySession: Sendable { throw lastError ?? PluginFactoryError.invalidSource("Factory stopped without a result.") } + /// Model HTTP failures are not a bad Go draft. Retrying them repeats the same refusal. + private static func isProviderHTTPFailure(_ message: String) -> Bool { + let lower = message.lowercased() + return lower.contains("http 4") || lower.contains("http 5") + || lower.contains("invalid_request_error") + || lower.contains("unsupported_value") + } + private static func builderFeedback(from error: PluginFactoryError, userGoal: String) -> String { switch error { case .draftValidationFailed(let findings): @@ -137,71 +146,30 @@ public struct PluginFactory: Sendable { try validateSource(draft.guestSource) try PluginFactoryDraftValidator.validateStructure(draft: draft, manifest: manifest) - let hopRun: PluginFactoryHopTestRun - do { - await logger("[plugin_factory] direct_test_started") - hopRun = try await PluginFactoryHopTestRunner.run( - source: draft.guestSource, - testInput: draft.testInput, - executor: executor - ) - } catch { - await logger("[plugin_factory] direct_test failed=\(pluginFactoryLogValue(error.localizedDescription))") - throw PluginFactoryError.directRunFailed(error.localizedDescription) - } - let direct = hopRun.final - let reviewRun = hopRun.aggregatedDirectRun - await logger( - "[plugin_factory] direct_test exit=\(direct.exitCode) " + - "stdout_chars=\(reviewRun.stdout.count) stderr_chars=\(direct.stderr.count)" + let codeReviewInput = PluginFactoryExecutionResult( + exitCode: 0, + stdout: Data("CODE_REVIEW\nTests have not been run. Review the source and the test plan only.".utf8) ) - guard direct.exitCode == 0 else { - await logger("[plugin_factory] direct_test rejected=\(pluginFactoryLogValue(outputSummary(direct)))") - throw PluginFactoryError.directRunFailed(outputSummary(direct)) - } - do { - try validateOutput(direct.stdout) - try PluginFactoryDraftValidator.validateDirectTest( - draft: draft, - manifest: manifest, - hopRun: hopRun - ) - } catch let error as PluginFactoryError { - switch error { - case .draftValidationFailed: - await logger( - "[plugin_factory] draft_validation failed=\(pluginFactoryLogValue(error.localizedDescription))" - ) - throw error - default: - await logger("[plugin_factory] direct_output invalid=\(pluginFactoryLogValue(error.localizedDescription))") - throw error - } - } catch { - await logger("[plugin_factory] direct_output invalid=\(pluginFactoryLogValue(error.localizedDescription))") - throw PluginFactoryError.invalidDirectOutput(error.localizedDescription) - } - - let review: PluginFactoryReview + let codeReview: PluginFactoryReview do { await logger("[plugin_factory] review_started") - review = try await reviewer.review(draft: draft, directRun: reviewRun) + codeReview = try await reviewer.review(draft: draft, directRun: codeReviewInput) } catch { await logger("[plugin_factory] review failed=\(pluginFactoryLogValue(error.localizedDescription))") throw error } await logger( - "[plugin_factory] review decision=\(review.decision.rawValue) " + - "finding_count=\(review.findings.count) summary=\(pluginFactoryLogValue(review.summary))" + "[plugin_factory] review decision=\(codeReview.decision.rawValue) " + + "finding_count=\(codeReview.findings.count) summary=\(pluginFactoryLogValue(codeReview.summary))" ) - guard review.approved else { - let findingMessages = review.findings.map(\.message) + guard codeReview.approved else { + let findingMessages = codeReview.findings.map(\.message) let detail = findingMessages.isEmpty - ? review.summary - : "\(review.summary) \(findingMessages.joined(separator: " "))" + ? codeReview.summary + : "\(codeReview.summary) \(findingMessages.joined(separator: " "))" await logger("[plugin_factory] review rejected=\(pluginFactoryLogValue(detail))") throw PluginFactoryError.reviewRejected( - summary: review.summary, + summary: codeReview.summary, findings: findingMessages ) } @@ -220,48 +188,60 @@ public struct PluginFactory: Sendable { throw PluginFactoryError.packageFailed("Guest source artifact is empty.") } - let packagedRun: PluginFactoryHopTestRun - do { - await logger("[plugin_factory] packaged_test_started") - packagedRun = try await PluginFactoryHopTestRunner.run( - artifact: artifact, + let review: PluginFactoryReview + if let liveExecutor = executor as? any PluginFactoryLiveAcceptanceExecutor { + await logger("[plugin_factory] live_acceptance_started") + let live: PluginFactoryHopTestRun + do { + live = try await liveExecutor.runLiveAcceptance( + artifact: artifact, + testInput: draft.testInput + ) + } catch { + await logger("[plugin_factory] live_acceptance failed=\(pluginFactoryLogValue(error.localizedDescription))") + throw PluginFactoryError.directRunFailed(error.localizedDescription) + } + let transcript = live.aggregatedDirectRun + let ops = PluginFactoryValidationExpectations.messagingOps(fromManifestJSON: draft.manifestJSON) + let stdout = String(decoding: transcript.stdout, as: UTF8.self) + if let problem = PluginLiveAcceptance.problem( testInput: draft.testInput, - executor: executor + stdout: stdout, + stderr: String(decoding: transcript.stderr, as: UTF8.self), + exitCode: transcript.exitCode, + messagingOps: ops + ) { + await logger("[plugin_factory] live_acceptance failed=\(pluginFactoryLogValue(problem))") + throw PluginFactoryError.directRunFailed(problem) + } + await logger( + "[plugin_factory] live_acceptance exit=\(transcript.exitCode) stdout_chars=\(transcript.stdout.count)" ) - } catch { - await logger("[plugin_factory] packaged_test failed=\(pluginFactoryLogValue(error.localizedDescription))") - throw PluginFactoryError.packagedRunFailed(error.localizedDescription) - } - let packaged = packagedRun.final - await logger( - "[plugin_factory] packaged_test exit=\(packaged.exitCode) " + - "stdout_chars=\(packaged.stdout.count) stderr_chars=\(packaged.stderr.count)" - ) - guard packaged.exitCode == 0 else { - await logger("[plugin_factory] packaged_test rejected=\(pluginFactoryLogValue(outputSummary(packaged)))") - throw PluginFactoryError.packagedRunFailed(outputSummary(packaged)) - } - do { - try validateOutput(packaged.stdout) - try PluginFactoryDraftValidator.validateDirectTest( - draft: draft, - manifest: manifest, - hopRun: packagedRun + let liveInput = PluginFactoryExecutionResult( + exitCode: 0, + stdout: Data("LIVE_TEST\n\(stdout)".utf8), + stderr: transcript.stderr ) - } catch let error as PluginFactoryError { - switch error { - case .draftValidationFailed: - await logger( - "[plugin_factory] packaged_validation failed=\(pluginFactoryLogValue(error.localizedDescription))" - ) - throw error - default: - await logger("[plugin_factory] packaged_output invalid=\(pluginFactoryLogValue(error.localizedDescription))") + do { + await logger("[plugin_factory] review_started") + review = try await reviewer.review(draft: draft, directRun: liveInput) + } catch { + await logger("[plugin_factory] review failed=\(pluginFactoryLogValue(error.localizedDescription))") throw error } - } catch { - await logger("[plugin_factory] packaged_output invalid=\(pluginFactoryLogValue(error.localizedDescription))") - throw PluginFactoryError.invalidPackagedOutput(error.localizedDescription) + guard review.approved else { + let findingMessages = review.findings.map(\.message) + let detail = findingMessages.isEmpty + ? review.summary + : "\(review.summary) \(findingMessages.joined(separator: " "))" + await logger("[plugin_factory] review rejected=\(pluginFactoryLogValue(detail))") + throw PluginFactoryError.reviewRejected( + summary: review.summary, + findings: findingMessages + ) + } + } else { + review = codeReview } let guestPath = PluginFactoryRuntime.guestSourcePackagePath( diff --git a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift index 6337a522..3c9708c5 100644 --- a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift +++ b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift @@ -114,9 +114,9 @@ import Testing var tampered = release.packageFiles() tampered["app.derrick/plugin.go"] = Data("changed".utf8) #expect(!PluginFactoryRelease.verifyIntegrity(files: tampered, expected: release.contentHash)) - #expect(await executor.draftRunCount == 1) + #expect(await executor.draftRunCount == 0) #expect(await executor.packageCount == 1) - #expect(await executor.packagedRunCount == 1) + #expect(await executor.packagedRunCount == 0) #expect(await reviewer.callCount == 1) } @@ -143,41 +143,25 @@ import Testing } } - @Test func invalidDraftOutputStopsBeforeReview() async throws { - let executor = RecordingFactoryExecutor( - draftResult: PluginFactoryExecutionResult( - exitCode: 0, - stdout: Data(#"{"not":"an array"}"#.utf8) - ) + @Test func codeReviewRunsBeforeAnyExecution() async throws { + let executor = RecordingFactoryExecutor() + let reviewer = RecordingFactoryReviewer( + result: PluginFactoryReview(approved: false, summary: "source is not safe") ) - let reviewer = RecordingFactoryReviewer(result: PluginFactoryReview(approved: true, summary: "safe")) - do { _ = try await PluginFactory().build( draft: PluginFactoryDraft( manifestJSON: manifestJSON(), - guestSource: """ - package main - - import ( - "encoding/json" - "os" - ) - - func main() { - _ = json.NewDecoder(os.Stdin).Decode(&map[string]any{}) - os.Stdout.WriteString("not a plugin envelope") - } - """, - testInput: Data(#"{"kind":"manual"}"#.utf8) + guestSource: guestGoSource() ), executor: executor, reviewer: reviewer ) - Issue.record("Expected invalid output") + Issue.record("Expected review rejection") } catch let error as PluginFactoryError { - #expect(error.localizedDescription.contains("invalid plugin output")) - #expect(await reviewer.callCount == 0) + #expect(error == .reviewRejected(summary: "source is not safe", findings: [])) + #expect(await executor.draftRunCount == 0) + #expect(await executor.packageCount == 0) } } @@ -201,11 +185,10 @@ import Testing ) #expect(release.pluginID == "weather-tool") - #expect(await executor.draftRunCount == 2) + #expect(await executor.draftRunCount == 0) #expect(await executor.packageCount == 1) let requests = await builder.requests - #expect(requests.count == 2) - #expect(requests[1].feedback?.contains("compile error") == true) + #expect(requests.count == 1) #expect(await reviewer.callCount == 1) } @@ -238,7 +221,7 @@ import Testing #expect(release.pluginID == "weather-tool") #expect(await reviewer.callCount == 2) - #expect(await executor.draftRunCount == 2) + #expect(await executor.draftRunCount == 0) #expect(await executor.packageCount == 1) let requests = await builder.requests #expect(requests.count == 2) @@ -265,7 +248,7 @@ import Testing } catch let error as PluginFactoryError { #expect(error == .reviewRejected(summary: "The draft remains unsafe.", findings: [])) #expect(await reviewer.callCount == 3) - #expect(await executor.draftRunCount == 3) + #expect(await executor.draftRunCount == 0) #expect(await executor.packageCount == 0) #expect(await builder.requests.count == 3) } @@ -438,11 +421,8 @@ import Testing """ {"hops":[ {"kind":"manual","params":{"messaging_op":"sync_threads"}}, - {"kind":"http_results","http_results":[{"request_id":"sync-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"sync_threads"}}, {"kind":"manual","params":{"messaging_op":"poll_inbox","vendor_thread_id":"C123"}}, - {"kind":"http_results","http_results":[{"request_id":"poll-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"poll_inbox"}}, - {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C123","text":"hello"}}, - {"kind":"http_results","http_results":[{"request_id":"send-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"send_message"}} + {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C123","text":"hello"}} ]} """.utf8 ) @@ -463,13 +443,9 @@ import Testing """ {"hops":[ {"kind":"manual","params":{"messaging_op":"sync_threads"}}, - {"kind":"http_results","http_results":[{"request_id":"sync-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"sync_threads"}}, {"kind":"manual","params":{"messaging_op":"poll_inbox","vendor_thread_id":"C123"}}, - {"kind":"http_results","http_results":[{"request_id":"poll-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"poll_inbox"}}, {"kind":"manual","params":{"messaging_op":"poll_inbox","vendor_thread_id":"C123","parent_vendor_message_id":"171.1"}}, - {"kind":"http_results","http_results":[{"request_id":"replies-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"poll_inbox"}}, - {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C123","text":"hello"}}, - {"kind":"http_results","http_results":[{"request_id":"send-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"send_message"}} + {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C123","text":"hello"}} ]} """.utf8 ), @@ -488,16 +464,9 @@ import Testing """ {"hops":[ {"kind":"manual","params":{"messaging_op":"sync_threads"}}, - {"kind":"http_results","http_results":[{"request_id":"sync-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"sync_threads"}}, {"kind":"manual","params":{"messaging_op":"poll_inbox","vendor_thread_id":"C1"}}, - {"kind":"http_results","http_results":[{"request_id":"poll-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"poll_inbox"}}, {"kind":"manual","params":{"messaging_op":"poll_inbox","vendor_thread_id":"C1","parent_vendor_message_id":"1"}}, - {"kind":"http_results","http_results":[{"request_id":"replies-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"poll_inbox"}}, - {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C1","text":"hi"}}, - {"kind":"http_results","http_results":[ - {"request_id":"send-1","status":200,"body":"{\\"ok\\":true,\\"ts\\":\\"1.0\\"}"}, - {"request_id":"auth-1","status":401,"body":"{\\"ok\\":false,\\"error\\":\\"invalid_auth\\"}"} - ],"params":{"messaging_op":"send_message"}} + {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C1","text":"hi"}} ]} """.utf8 ) @@ -711,13 +680,9 @@ private func validConnectorTestInput() -> Data { """ {"hops":[ {"kind":"manual","params":{"messaging_op":"sync_threads"}}, - {"kind":"http_results","http_results":[{"request_id":"sync-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"sync_threads"}}, {"kind":"manual","params":{"messaging_op":"poll_inbox","vendor_thread_id":"C1"}}, - {"kind":"http_results","http_results":[{"request_id":"poll-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"poll_inbox"}}, {"kind":"manual","params":{"messaging_op":"poll_inbox","vendor_thread_id":"C1","parent_vendor_message_id":"1"}}, - {"kind":"http_results","http_results":[{"request_id":"replies-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"poll_inbox"}}, - {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C1","text":"hi"}}, - {"kind":"http_results","http_results":[{"request_id":"send-1","status":200,"body":"{\\"ok\\":true}"}],"params":{"messaging_op":"send_message"}} + {"kind":"message_in_room","params":{"messaging_op":"send_message","vendor_thread_id":"C1","text":"hi"}} ]} """.utf8 ) diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift index 9445f9ab..a1fb8147 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift @@ -28,142 +28,44 @@ public enum PluginFactoryCreateFailureMessage: Sendable { ) } - if ModelProviderLimit.matches(trimmed) { - return PluginFactoryCreateFailurePresentation( - summary: ModelProviderLimit.summary, - technicalDetail: trimmed - ) - } - - if WorkerImageFailureDisplay.isWorkerImageIssue(trimmed) { - return PluginFactoryCreateFailurePresentation( - summary: """ - The connector was not saved. Derrick’s web tools were not ready. \ - Keep Docker Desktop open and try again. - """, - technicalDetail: trimmed - ) - } - - if isModelTimeout(trimmed) { - let reviewer = trimmed.lowercased().contains("safety reviewer") - return PluginFactoryCreateFailurePresentation( - summary: reviewer - ? """ - The connector was not saved. The safety reviewer did not finish in time. Try again. - """ - : """ - The connector was not saved. The plugin builder did not finish in time. \ - High thinking can take several minutes — try again. - """, - technicalDetail: trimmed - ) - } - - if isReviewRejection(trimmed) || isTechnicalReviewerDetail(trimmed) { - return PluginFactoryCreateFailurePresentation( - summary: """ - The connector was not saved. Derrick built a draft but the safety review could not approve it \ - after several attempts. Try again. - """, - technicalDetail: trimmed - ) + if alreadyExplained(trimmed) { + return PluginFactoryCreateFailurePresentation(summary: trimmed, technicalDetail: nil) } - if isDraftValidationDetail(trimmed) || isFactoryDidNotSaveDetail(trimmed) { - return PluginFactoryCreateFailurePresentation( - summary: """ - The connector was not saved. Derrick could not finish building it after several attempts. \ - Try again. - """, - technicalDetail: trimmed - ) - } - - if isTechnicalFactoryDetail(trimmed) { + if ModelProviderLimit.matches(trimmed) { return PluginFactoryCreateFailurePresentation( - summary: """ - The connector was not saved. Derrick could not finish building it. Try again. - """, + summary: ModelProviderLimit.summary, technicalDetail: trimmed ) } + let nature = excerpt(trimmed) + let unchanged = nature == collapse(trimmed) return PluginFactoryCreateFailurePresentation( - summary: alreadyExplained(trimmed) - ? trimmed - : "The connector was not saved. \(trimmed)", - technicalDetail: nil + summary: "The connector was not saved. \(nature)", + technicalDetail: unchanged ? nil : trimmed ) } - private static func alreadyExplained(_ message: String) -> Bool { - message.hasPrefix("The connector was not saved.") - } - - private static func isModelTimeout(_ message: String) -> Bool { - LLMHTTPTimeouts.isTimeoutDescription(message) - } - - private static func isFactoryDidNotSaveDetail(_ message: String) -> Bool { - let lower = message.lowercased() - return lower.contains("did not return a saved connector") - || lower.contains("plugin factory could not finish") - } + private static let excerptLimit = 360 - private static func isTechnicalFactoryDetail(_ message: String) -> Bool { - if message.count > 160 { return true } - let prefixes = [ - "Invalid Agent Plugin manifest", - "Invalid Go guest source", - "Go draft test failed", - "Plugin review rejected", - "Draft validation failed:", - ] - return prefixes.contains(where: { message.hasPrefix($0) }) - } - - private static func isDraftValidationDetail(_ message: String) -> Bool { - let lower = message.lowercased() - if lower.hasPrefix("draft validation failed:") { return true } - let indicators = [ - "sort http_results", - "test_input_json must", - "http_results must include", - "messaging_ops must declare", - "messaging_op ", - "deterministic draft validation", - "connector test_input_json must", - "direct test must", - ] - return indicators.contains(where: { lower.contains($0) }) + /// A readable slice of the actual error. Every failure keeps its own reason. + private static func excerpt(_ raw: String) -> String { + let collapsed = collapse(raw) + guard collapsed.count > excerptLimit else { return collapsed } + let end = collapsed.index(collapsed.startIndex, offsetBy: excerptLimit) + let head = collapsed[.. head.startIndex { + return String(head[.. Bool { - let lower = message.lowercased() - let indicators = [ - "direct test output", - "safety review", - "approval requires", - "review rejected", - "does not cover", - "test evidence", - "vendor operations", - "thread replies", - ] - return indicators.contains(where: { lower.contains($0) }) + private static func collapse(_ raw: String) -> String { + raw.split(whereSeparator: \.isWhitespace).joined(separator: " ") } - private static func isTechnicalReviewerDetail(_ message: String) -> Bool { - if message.contains(";") && message.count > 120 { return true } - let prefixes = [ - "The plugin source", - "The direct test output", - "The source uses", - "The source reads", - "The source appears", - "The connector synchronizes", - ] - return prefixes.contains(where: { message.hasPrefix($0) }) + private static func alreadyExplained(_ message: String) -> Bool { + message.hasPrefix("The connector was not saved.") } } diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift index ffe2a8dd..4d33fa07 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift @@ -304,6 +304,10 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { extra.append("Host permission labels: \(auth.permissions.joined(separator: ", "))") } } + // Scope owns the blueprint. Only pass the failure oracle from the user/spec. + if description.contains("Unacceptable when:") { + extra.append("User-confirmed behavior:\n\(description)") + } if let skillMarkdown, !skillMarkdown.isEmpty { extra.append("SKILL.md draft:\n\(skillMarkdown)") } @@ -336,6 +340,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { var lines = [ "Create an Agent Plugin capability.", description, + PluginAcceptanceDirections.builderText(forUserGoal: description), ] if let agentPluginSpecSummary, !agentPluginSpecSummary.isEmpty { lines.append( diff --git a/packages/Structure/Sources/AppLayerServices/Plugin/PluginAccessAskPolicy.swift b/packages/Structure/Sources/AppLayerServices/Plugin/PluginAccessAskPolicy.swift index 280a8982..9120a837 100644 --- a/packages/Structure/Sources/AppLayerServices/Plugin/PluginAccessAskPolicy.swift +++ b/packages/Structure/Sources/AppLayerServices/Plugin/PluginAccessAskPolicy.swift @@ -45,6 +45,32 @@ public enum PluginAccessAskPolicy: Sendable { return "Derrick looked at \(link) but could not use it as API setup docs. Paste the page that explains how apps authenticate (a token, key, or login)." } + public static func isGuidanceRequest(_ text: String) -> Bool { + let lowered = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard !lowered.isEmpty else { return false } + if lowered.contains("?") { return true } + let phrases = [ + "how ", "how do", "where ", "what ", "why ", "which ", + "help me", "walk me", "guide me", "show me", + "i don't know", "i dont know", "not sure", "don't understand", "dont understand", + ] + return phrases.contains { lowered.hasPrefix($0) || lowered.contains($0) } + } + + /// Stay on the access step. Point at the setup page without naming a vendor. + public static func guidanceReply(documentationURL: String?) -> String { + var lines = [ + "You do not have to read the whole document. Derrick can walk you through creating the credential.", + ] + if let documentationURL, !documentationURL.isEmpty { + lines.append("[Open the setup docs](\(documentationURL))") + } + lines.append( + "On that page, create the token or key it asks for. When you have it, say yes. Derrick will ask you to enter it here. It stays on this Mac." + ) + return lines.joined(separator: "\n\n") + } + public static func isDocsSearchRequest(_ text: String) -> Bool { if parseHTTPURL(text) != nil { return false } let lowered = text.lowercased() diff --git a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecDraft.swift b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecDraft.swift index 197e16f4..77369947 100644 --- a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecDraft.swift +++ b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecDraft.swift @@ -145,6 +145,7 @@ public struct PluginSpecDraft: Sendable, Hashable, Codable { work.map { "Work: \($0.rawValue)" }, returnClass.map { "Return: \($0.rawValue)" }, present.map { "Present: \($0.rawValue)" }, + wrongness.map { "Unacceptable when: \($0)" }, ].compactMap { $0 } return PluginSkillDraft( goal: outcome, diff --git a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecProcession.swift b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecProcession.swift index 573f40d4..3d87587a 100644 --- a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecProcession.swift +++ b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSpecProcession.swift @@ -230,7 +230,13 @@ public enum PluginSpecProcession: Sendable { } let reply: String - if session.ask == askAtStart { + if session.ask == askAtStart, + isAccessGuidanceTurn(ask: session.ask, utterance: trimmed) { + reply = PluginAccessAskPolicy.guidanceReply( + documentationURL: session.draft.documentationURL + ?? PluginAccessAskPolicy.documentationURL(from: session.draft.connect) + ) + } else if session.ask == askAtStart { reply = notBoundHint(for: session.ask, utterance: trimmed) ?? question(for: session.ask, session: session) } else { @@ -457,6 +463,16 @@ public enum PluginSpecProcession: Sendable { } } + private static func isAccessGuidanceTurn(ask: PluginSpecAsk, utterance: String) -> Bool { + guard PluginAccessAskPolicy.isGuidanceRequest(utterance) else { return false } + switch ask { + case .slot(.access), .accessSecret: + return true + default: + return false + } + } + private static func notBoundHint(for ask: PluginSpecAsk, utterance: String) -> String? { if case .docsURL = ask, PluginAccessAskPolicy.parseHTTPURL(utterance) == nil { if PluginAccessAskPolicy.isDocsSearchRequest(utterance) { diff --git a/packages/Structure/Sources/Contract/ConnectorContractPrompts.swift b/packages/Structure/Sources/Contract/ConnectorContractPrompts.swift index 71b07709..1995526c 100644 --- a/packages/Structure/Sources/Contract/ConnectorContractPrompts.swift +++ b/packages/Structure/Sources/Contract/ConnectorContractPrompts.swift @@ -28,7 +28,9 @@ public enum ConnectorContractPrompts: Sendable { Review connectors against this protocol JSON only. If a rule is not in the JSON, do not require it. \ Do not reject sync_threads for omitting conversation.history or conversation.replies. \ Do not reject emitting messages: [] when the vendor reported success. \ - Reject empty messages as success only when the vendor reported failure. + Reject empty messages as success only when the vendor reported failure. \ + The direct test output is a live acceptance run. Reject it when that run failed or matches what the user called unacceptable. \ + test_input_json fixtures are not evidence. """) } @@ -69,10 +71,11 @@ public enum ConnectorContractPrompts: Sendable { preamble: "Obey this protocol JSON. Do not add ops outside it. Follow crawled vendor docs for HTTP URLs and request shape." ) ) + parts.append(PluginAcceptanceDirections.builderText(forUserGoal: "Scope id: \(scopeID) sync_threads")) parts.append( """ - test_input_json must include a hops array with http_results fixtures that exercise every messaging_op you implement \ - (\(scopeSpec.ops.joined(separator: ", "))) through to result.emit. + test_input_json must include a hops array. Each messaging op starts as its own hop with params.messaging_op and no http_results. \ + The host runs that hop live. Ops: \(scopeSpec.ops.joined(separator: ", ")). """ ) parts.append( diff --git a/packages/Structure/Sources/LLMAgentClient/AgentClientTypes.swift b/packages/Structure/Sources/LLMAgentClient/AgentClientTypes.swift index 0ba015a6..9593042f 100644 --- a/packages/Structure/Sources/LLMAgentClient/AgentClientTypes.swift +++ b/packages/Structure/Sources/LLMAgentClient/AgentClientTypes.swift @@ -333,12 +333,35 @@ public struct AgentModelID: Hashable, Codable, Sendable { } } +/// Request fields a model accepts. The provider sends only these. +public struct ModelRequestSupport: Hashable, Codable, Sendable, Equatable { + public let temperature: Bool + + public init(temperature: Bool) { + self.temperature = temperature + } +} + public protocol AgentModel: Hashable, Codable, Sendable { var id: AgentModelID { get } var maxSupportedContextTokens: Int { get } var maxIdealContextTokens: Int { get } /// List prices for estimated USD (not billed by the chat API). var tokenPricing: ModelTokenPricing { get } + var requestSupport: ModelRequestSupport { get } + func acceptsThinking(_ option: ModelThinkingOption) -> Bool +} + +extension AgentRequest { + /// Drops fields this model does not accept so a caller cannot send them. + public func constrained(to model: Model) -> AgentRequest { + AgentRequest( + messages: messages, + temperature: model.requestSupport.temperature ? temperature : nil, + responseSchema: responseSchema, + thinking: thinking.flatMap { model.acceptsThinking($0) ? $0 : nil } + ) + } } /// Provider list prices used only to *estimate* USD from token counts. diff --git a/packages/Structure/Sources/MemorySystem/Models.swift b/packages/Structure/Sources/MemorySystem/Models.swift index 1e53a706..b42d986a 100644 --- a/packages/Structure/Sources/MemorySystem/Models.swift +++ b/packages/Structure/Sources/MemorySystem/Models.swift @@ -209,7 +209,10 @@ public extension MemoryBudget { .init(provider: "gemini", name: "gemini-3.7-flash"): 50_000, .init(provider: "openai", name: "gpt-5.6-luna"): 25_000, .init(provider: "openai", name: "gpt-5.6-terra"): 50_000, - .init(provider: "openai", name: "gpt-5.6-sol"): 50_000 + .init(provider: "openai", name: "gpt-5.6-sol"): 50_000, + .init(provider: "openai", name: "gpt-6-luna"): 25_000, + .init(provider: "openai", name: "gpt-6-sol"): 50_000, + .init(provider: "openai", name: "gpt-6-astra"): 50_000 ] static func maxTokenCount(forProvider provider: String, modelName: String) -> Int { diff --git a/packages/Structure/Sources/Plugin/Factory/PluginAcceptanceDirections.swift b/packages/Structure/Sources/Plugin/Factory/PluginAcceptanceDirections.swift new file mode 100644 index 00000000..e9e6db31 --- /dev/null +++ b/packages/Structure/Sources/Plugin/Factory/PluginAcceptanceDirections.swift @@ -0,0 +1,155 @@ +import Foundation + +/// How the factory must prove a plugin works. Directions depend on the plugin type, not the vendor. +public enum PluginAcceptanceDirections: Sendable { + public static func builderText(forUserGoal userGoal: String?) -> String { + switch kind(forUserGoal: userGoal) { + case .messagingConnector: + return """ + Acceptance test for a messaging connector (live, not a fixture): + Write the tests after the code. Do not put http_results or any made-up service reply in test_input_json. + Each messaging op is its own hop with params.messaging_op only. + The test must prove the user's statement of what failure is does not happen. + If failure is that it cannot connect, the live test must connect. + If failure is that it cannot send, the live test must send a real test message. + If failure is that it cannot receive, the live test must receive. + After the code review passes, the host runs these hops against the real service and the reviewer confirms they completed. + sync_threads is complete only when result.emit includes threads. + poll_inbox is complete only when result.emit includes messages. + send_message is complete only when result.emit includes sent_message. + """ + case .newsReader: + return """ + Acceptance test for a news reader (live, not a fixture): + The test must start from the source the user named and emit http.request for that fetch. + The host performs that request. The guest must turn the real response into the items the user asked for. + A saved HTML or JSON body in test_input_json is not a test. + The user's statement of what is unacceptable is the failure oracle. + """ + case .customCapability: + return """ + Acceptance test for a custom plugin (live, not a fixture): + If the plugin calls the network, the test hop must emit http.request and must not include a prepared http_results body. + The host performs those requests. The live result must show the outcome the user asked for. + A constant success body is not a test. + The user's statement of what is unacceptable is the failure oracle. + """ + } + } + + public static func reviewerText(forUserGoal userGoal: String?) -> String { + """ + If the direct test output begins with CODE_REVIEW, tests have not run. Judge the source and the test plan only. \ + Reject a test plan that contains http_results or would not prove the user's failure statement. + If the direct test output begins with LIVE_TEST, the tests have been run against the real service. \ + Confirm they completed. Reject when a required action did not complete or the result is what the user called unacceptable. + \(builderText(forUserGoal: userGoal)) + """ + } + + public static func kind(forUserGoal userGoal: String?) -> Kind { + let ops = PluginFactoryValidationExpectations.requiredMessagingOps(from: userGoal) + if !ops.isEmpty { return .messagingConnector } + let text = userGoal?.lowercased() ?? "" + if text.contains("news") { return .newsReader } + return .customCapability + } + + public enum Kind: Sendable { + case messagingConnector + case newsReader + case customCapability + } +} + +/// Decides whether a live acceptance transcript satisfied the plugin type. No vendor knowledge. +public enum PluginLiveAcceptance: Sendable { + public static func liveHops(in testInput: Data) -> [PluginHopEvent] { + guard let script = try? PluginFactoryTestScript.parse(testInput) else { return [] } + return script.hops.filter { $0.kind != .httpResults } + } + + public static func firstThreadID(in stdout: String) -> String? { + guard let range = stdout.range(of: "\"vendor_thread_id\"") else { return nil } + let tail = stdout[range.upperBound...] + guard let colon = tail.firstIndex(of: ":") else { return nil } + let after = tail[tail.index(after: colon)...].drop { $0.isWhitespace } + guard after.first == "\"" else { return nil } + let rest = after.dropFirst() + guard let end = rest.firstIndex(of: "\"") else { return nil } + let id = String(rest[.. PluginHopEvent { + var params = hop.params ?? [:] + let op = params["messaging_op"]?.stringValue + if op == "poll_inbox" || op == "send_message" { + let existing = params["vendor_thread_id"]?.stringValue ?? "" + if existing.isEmpty, let discoveredThreadID { + params["vendor_thread_id"] = .string(discoveredThreadID) + } + } + if op == "send_message" { + let text = params["text"]?.stringValue ?? "" + if text.isEmpty { + params["text"] = .string("Derrick acceptance check") + } + } + return PluginHopEvent(kind: hop.kind, httpResults: nil, params: params) + } + + public static func problem( + testInput: Data, + stdout: String, + stderr: String, + exitCode: Int32, + messagingOps: [String] + ) -> String? { + let hops = liveHops(in: testInput) + if hops.isEmpty { + return """ + Live acceptance failed. The test script has no hop for the host to run. \ + Fixture http_results are not a test. + """ + } + if exitCode != 0 { + return "Live acceptance failed.\n\(clip(stderr))\n\(clip(stdout))" + } + guard !messagingOps.isEmpty else { return nil } + let started = Set(hops.compactMap { $0.params?["messaging_op"]?.stringValue }) + var missing: [String] = [] + for op in messagingOps { + guard started.contains(op) else { + missing.append("\(op) was not started as a live hop") + continue + } + if let field = successField(for: op), !stdout.contains("\"\(field)\"") { + missing.append("\(op) did not return \(field)") + } + } + guard !missing.isEmpty else { return nil } + return """ + Live acceptance failed: \(missing.joined(separator: "; ")). + \(clip(stdout)) + """ + } + + public static func successField(for messagingOp: String) -> String? { + switch messagingOp { + case "sync_threads": return "threads" + case "poll_inbox": return "messages" + case "send_message": return "sent_message" + default: return nil + } + } + + private static func clip(_ text: String) -> String { + let collapsed = text.split(whereSeparator: \.isWhitespace).joined(separator: " ") + if collapsed.count <= 500 { return collapsed } + return String(collapsed.prefix(500)) + "…" + } +} diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryDraftValidator.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryDraftValidator.swift index cd241cb1..deae082e 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryDraftValidator.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryDraftValidator.swift @@ -94,9 +94,9 @@ public enum PluginFactoryDraftValidator: Sendable { ) } - if script.hops.count < 2 { + if script.hops.contains(where: { $0.kind == .httpResults || $0.httpResults?.isEmpty == false }) { findings.append( - "Connector test_input_json must use a hops array with an initial messaging hop and an http_results hop." + "test_input_json must not include http_results. The host runs each hop live." ) } @@ -104,11 +104,6 @@ public enum PluginFactoryDraftValidator: Sendable { findings.append("Connector test_input_json must set params.messaging_op on the first hop.") } - let fixtureCount = script.hops.reduce(0) { $0 + ($1.httpResults?.count ?? 0) } - if fixtureCount == 0 { - findings.append("Connector test_input_json must include http_results fixtures for vendor HTTP replay.") - } - let testedOps = messagingOpsExercised(in: script) for op in requiredOps where !testedOps.contains(op) { findings.append( diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift index 13a5c9ff..efdb2eb2 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift @@ -593,6 +593,11 @@ public protocol PluginFactoryCompiledGuestExecutor: PluginFactoryExecutor { func runGuestSourceHops(source: String, testInput: Data) async throws -> PluginFactoryHopTestRun } +/// Runs the guest's own http.request hops against the network. Fixture bodies are not replayed. +public protocol PluginFactoryLiveAcceptanceExecutor: PluginFactoryExecutor { + func runLiveAcceptance(artifact: Data, testInput: Data) async throws -> PluginFactoryHopTestRun +} + public enum PluginReviewDecision: String, Sendable, Hashable { case approved case rejected diff --git a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift index 345e2e61..5297c61f 100644 --- a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift +++ b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift @@ -1527,6 +1527,22 @@ import Testing ) } + @Test func pluginFactoryCreateFailureMessageIncludesTheError() { + let raw = """ + Invalid Go guest source: HTTP 400: {"error":{"message":"Unsupported value: 'temperature' does not support 0 with this model. Only the default (1) value is supported.","type":"invalid_request_error","param":"temperature","code":"unsupported_value"}} + """ + let presentation = PluginFactoryCreateFailureMessage.presentation(raw) + #expect(presentation.summary.contains("The connector was not saved.")) + #expect(presentation.summary.contains("temperature")) + #expect(presentation.summary.contains("Only the default (1) value is supported.")) + #expect(presentation.summary.contains("could not finish building") == false) + #expect(presentation.technicalDetail == nil) + let docker = "Docker daemon is not running." + let dockerPresentation = PluginFactoryCreateFailureMessage.presentation(docker) + #expect(dockerPresentation.summary.contains(docker)) + #expect(dockerPresentation.technicalDetail == nil) + } + @Test func pluginFactoryCreateFailureMessageExplainsProviderLimitOnce() { let raw = """ Invalid Go guest source: HTTP 429: {"error":{"code":"project_spend_limit_exceeded","message":"Your project has reached its configured enforced spend limit."}} @@ -1541,53 +1557,49 @@ import Testing @Test func pluginFactoryCreateFailureMessageExplainsWebTools() { let raw = "The worker image derrick-worker:go-v1 does not match the version shipped with Derrick. Rebuild or reinstall product images." let presentation = PluginFactoryCreateFailureMessage.presentation(raw) - #expect(presentation.summary.contains("web tools were not ready")) - #expect(presentation.technicalDetail == raw) + #expect(presentation.summary.contains("derrick-worker:go-v1")) + #expect(presentation.technicalDetail == nil) } @Test func pluginFactoryCreateFailureMessageExplainsBuilderTimeout() { let presentation = PluginFactoryCreateFailureMessage.presentation("The request timed out.") - #expect(presentation.summary.contains("plugin builder did not finish in time")) - #expect(presentation.summary.contains("several minutes")) - #expect(presentation.technicalDetail == "The request timed out.") + #expect(presentation.summary.contains("The request timed out.")) + #expect(presentation.technicalDetail == nil) } @Test func pluginFactoryCreateFailureMessageExplainsReviewerTimeout() { let presentation = PluginFactoryCreateFailureMessage.presentation( "The plugin safety reviewer model timed out." ) - #expect(presentation.summary.contains("safety reviewer did not finish in time")) - #expect(presentation.technicalDetail?.contains("safety reviewer") == true) + #expect(presentation.summary.contains("The plugin safety reviewer model timed out.")) + #expect(presentation.technicalDetail == nil) } - @Test func pluginFactoryCreateFailureMessageSanitizesReviewDetail() { + @Test func pluginFactoryCreateFailureMessageIncludesReviewDetail() { let raw = """ The source appears to use the guest runtime envelope, stable de-duplication, channel-specific identifiers, and paginated Slack requests, but the supplied direct test does not cover the connector's required receive/sync operations. """ let presentation = PluginFactoryCreateFailureMessage.presentation(raw) #expect(presentation.summary.contains("was not saved")) - #expect(!presentation.summary.contains("guest runtime envelope")) - #expect(presentation.summary.contains("safety review")) - #expect(!presentation.summary.contains("Send only")) - #expect(!presentation.summary.contains("Full sync")) - #expect(presentation.technicalDetail == raw) + #expect(presentation.summary.contains("guest runtime envelope")) + #expect(presentation.summary.contains("receive/sync operations")) } - @Test func pluginFactoryCreateFailureMessageSanitizesDraftValidationDetail() { + @Test func pluginFactoryCreateFailureMessageIncludesDraftValidationDetail() { let raw = """ Sort http_results by request_id and de-duplicate before lookup. Do not return the first matching entry from an unsorted loop. """ let presentation = PluginFactoryCreateFailureMessage.presentation(raw) #expect(presentation.summary.contains("was not saved")) - #expect(!presentation.summary.contains("http_results")) - #expect(presentation.technicalDetail == raw) + #expect(presentation.summary.contains("http_results")) + #expect(presentation.technicalDetail == nil) } @Test func pluginFactoryCreateFailureMessageExplainsMissingSavedConnector() { let raw = "Plugin factory did not return a saved connector." let presentation = PluginFactoryCreateFailureMessage.presentation(raw) - #expect(presentation.summary.contains("was not saved")) - #expect(presentation.technicalDetail == raw) + #expect(presentation.summary.contains("did not return a saved connector")) + #expect(presentation.technicalDetail == nil) } @Test func connectorManifestMessagingOpsDetectSendOnlyScope() { diff --git a/packages/Structure/Tests/StructureTests/PluginLiveAcceptanceTests.swift b/packages/Structure/Tests/StructureTests/PluginLiveAcceptanceTests.swift new file mode 100644 index 00000000..9214893f --- /dev/null +++ b/packages/Structure/Tests/StructureTests/PluginLiveAcceptanceTests.swift @@ -0,0 +1,47 @@ +import Foundation +import Testing +@testable import Structure + +@Suite struct PluginLiveAcceptanceTests { + @Test func fixtureOnlyScriptIsNotALiveTest() { + let input = Data(#"{"hops":[{"kind":"http_results","http_results":[{"request_id":"sync_threads_page_1","status":200,"body":"{\"ok\":true}"}]}]}"#.utf8) + let problem = PluginLiveAcceptance.problem( + testInput: input, + stdout: #"[{"verb":"result.emit","threads":[]}]"#, + stderr: "", + exitCode: 0, + messagingOps: ["sync_threads"] + ) + #expect(problem?.contains("not a test") == true) + } + + @Test func liveSyncWithoutThreadsFails() { + let input = Data(#"{"hops":[{"kind":"manual","params":{"messaging_op":"sync_threads"}}]}"#.utf8) + let problem = PluginLiveAcceptance.problem( + testInput: input, + stdout: #"[{"verb":"result.emit","title":"Slack","summary":"Slack request failed: missing_scope"}]"#, + stderr: "", + exitCode: 0, + messagingOps: ["sync_threads"] + ) + #expect(problem?.contains("did not return threads") == true) + } + + @Test func liveSyncWithThreadsPassesShape() { + let input = Data(#"{"hops":[{"kind":"manual","params":{"messaging_op":"sync_threads"}}]}"#.utf8) + let problem = PluginLiveAcceptance.problem( + testInput: input, + stdout: #"[{"verb":"result.emit","threads":[{"vendor_thread_id":"C1","title":"general"}]}]"#, + stderr: "", + exitCode: 0, + messagingOps: ["sync_threads"] + ) + #expect(problem == nil) + } + + @Test func newsReaderDirectionsAreNotConnectorDirections() { + let text = PluginAcceptanceDirections.builderText(forUserGoal: "Read tech news headlines") + #expect(text.contains("news reader")) + #expect(!text.contains("sync_threads")) + } +} diff --git a/packages/Structure/Tests/StructureTests/PluginSpecProcessionTests.swift b/packages/Structure/Tests/StructureTests/PluginSpecProcessionTests.swift index 00d04ce7..463025f6 100644 --- a/packages/Structure/Tests/StructureTests/PluginSpecProcessionTests.swift +++ b/packages/Structure/Tests/StructureTests/PluginSpecProcessionTests.swift @@ -248,6 +248,22 @@ import Testing #expect(question.contains(PluginAccessAskPolicy.genericQuestion) == false) } + @Test func accessAskAnswersAnOutOfBandQuestionWithoutLeavingTheStep() { + var session = PluginSpecSession() + session.draft.claimedOutcome = "Connect to a chat service" + session.draft.connect = PluginConnectBinding(klass: .messagingInbox, detail: "a chat service") + session.draft.documentationURL = "https://docs.example.com/auth" + session.ask = .slot(.access) + let turn = PluginSpecProcession.advance( + session: &session, + utterance: "How do I create the key? I don't know what it is." + ) + #expect(session.ask == .slot(.access)) + #expect(turn.reply.contains("https://docs.example.com/auth")) + #expect(turn.reply.contains("say yes")) + #expect(turn.isComplete == false) + } + @Test func accessAskSkipsANavDumpFromAnyDocsPage() { let nav = "Docs Home Changelog TutorialsSuccess!My AppsDiscard ChangesSave ChangesUsing ProductEnterprisePricingSupportGuidesMarket" let dumped = ConnectorAuthDiscovery( @@ -581,6 +597,9 @@ import Testing pluginID: "slack-connector-1" ) #expect(input.pluginID == "slack-connector-1") + #expect(input.description.contains("Unacceptable when: must not post without asking")) + #expect(input.connectorBuildGoal(crawlSummary: nil).contains("Unacceptable when: must not post without asking")) + #expect(input.connectorBuildGoal(crawlSummary: nil).contains("live, not a fixture")) #expect(input.auth?.authScheme.isSupportedInWizard == true) #expect(input.auth?.secrets.map(\.id) == ["bot_token"]) #expect( diff --git a/ui/JobService/JobAgentClient.swift b/ui/JobService/JobAgentClient.swift index 8c68b855..2d4abb6f 100644 --- a/ui/JobService/JobAgentClient.swift +++ b/ui/JobService/JobAgentClient.swift @@ -8,7 +8,7 @@ import Structure final class JobAgentClient: @unchecked Sendable { static let shared = JobAgentClient() - static let defaultModelJSON = Data(#"{"openai":{"_0":"gpt-5.6-luna"}}"#.utf8) + static let defaultModelJSON = Data(#"{"openai":{"_0":"gpt-6-luna"}}"#.utf8) private let lock = NSLock() private var connection: NSXPCConnection? diff --git a/ui/MCPService/MCPServiceScriptReviewer.swift b/ui/MCPService/MCPServiceScriptReviewer.swift index 7da7ba47..4c463023 100644 --- a/ui/MCPService/MCPServiceScriptReviewer.swift +++ b/ui/MCPService/MCPServiceScriptReviewer.swift @@ -57,7 +57,7 @@ struct MCPServiceScriptReviewer: ScriptReviewer { } } - private static let defaultModel: ReviewerModel = .openai(.gpt56Luna) + private static let defaultModel: ReviewerModel = .openai(.gpt6Luna) private static let secondaryDefault: ReviewerModel = .gemini(.gemini25FlashLite) private func resolveSelectedModel() -> ReviewerModel { diff --git a/ui/SharedAgentRuntime/AgentProfiles/AgentProfileBuiltinFactory.swift b/ui/SharedAgentRuntime/AgentProfiles/AgentProfileBuiltinFactory.swift index 56233ee3..ff3d391b 100644 --- a/ui/SharedAgentRuntime/AgentProfiles/AgentProfileBuiltinFactory.swift +++ b/ui/SharedAgentRuntime/AgentProfiles/AgentProfileBuiltinFactory.swift @@ -5,14 +5,13 @@ import Structure /// Built-in agent profiles with product-default models and thinking levels. public enum AgentProfileBuiltinFactory { public static func all() throws -> [AgentProfile] { - let solHigh = try wire(model: .openai(.gpt56Sol), thinkingID: "high") - let terraHigh = try wire(model: .openai(.gpt56Terra), thinkingID: "high") - let lunaHigh = try wire(model: .openai(.gpt56Luna), thinkingID: "high") + let lunaMedium = try wire(model: .openai(.gpt6Luna), thinkingID: "medium") + let lunaHigh = try wire(model: .openai(.gpt6Luna), thinkingID: "high") return [ - AgentProfile.orchestratorDefault(modelJSON: solHigh.modelJSON, thinkingJSON: solHigh.thinkingJSON), - AgentProfile.developerDefault(modelJSON: terraHigh.modelJSON, thinkingJSON: terraHigh.thinkingJSON), - AgentProfile.researcherDefault(modelJSON: terraHigh.modelJSON, thinkingJSON: terraHigh.thinkingJSON), - AgentProfile.generalistDefault(modelJSON: lunaHigh.modelJSON, thinkingJSON: lunaHigh.thinkingJSON), + AgentProfile.orchestratorDefault(modelJSON: lunaHigh.modelJSON, thinkingJSON: lunaHigh.thinkingJSON), + AgentProfile.developerDefault(modelJSON: lunaMedium.modelJSON, thinkingJSON: lunaMedium.thinkingJSON), + AgentProfile.researcherDefault(modelJSON: lunaMedium.modelJSON, thinkingJSON: lunaMedium.thinkingJSON), + AgentProfile.generalistDefault(modelJSON: lunaMedium.modelJSON, thinkingJSON: lunaMedium.thinkingJSON), ] } diff --git a/ui/SharedAgentRuntime/LLMChoice.swift b/ui/SharedAgentRuntime/LLMChoice.swift index 65ddb9a2..bff3ae9c 100644 --- a/ui/SharedAgentRuntime/LLMChoice.swift +++ b/ui/SharedAgentRuntime/LLMChoice.swift @@ -46,7 +46,7 @@ enum LLMProviderChoice: String, CaseIterable, Identifiable, Codable, Sendable { case .google: return .gemini(.gemini37Flash) case .openai: - return .openai(.gpt56Luna) + return .openai(.gpt6Luna) } } } @@ -64,14 +64,17 @@ enum LLMModelChoice: Hashable, Identifiable, Codable, Sendable { .openai(.gpt55), .openai(.gpt56Luna), .openai(.gpt56Terra), - .openai(.gpt56Sol) + .openai(.gpt56Sol), + .openai(.gpt6Luna), + .openai(.gpt6Sol), + .openai(.gpt6Astra) ] /// Default for summarizer, script reviewer, secondary agents, and conversation UI preselection. - static let defaultHelperModel: LLMModelChoice = .openai(.gpt56Luna) + static let defaultHelperModel: LLMModelChoice = .openai(.gpt6Luna) - /// Default plugin factory builder: stronger reasoning than the helper default. - static let defaultPluginBuilderModel: LLMModelChoice = .openai(.gpt56Terra) + /// Default plugin factory builder. Same model as the helper, at medium thinking. + static let defaultPluginBuilderModel: LLMModelChoice = .openai(.gpt6Luna) var id: String { switch self { diff --git a/ui/SharedAgentRuntime/Support/ConnectorAuthClassifier.swift b/ui/SharedAgentRuntime/Support/ConnectorAuthClassifier.swift index 7cbe918b..19226711 100644 --- a/ui/SharedAgentRuntime/Support/ConnectorAuthClassifier.swift +++ b/ui/SharedAgentRuntime/Support/ConnectorAuthClassifier.swift @@ -71,7 +71,7 @@ enum ConnectorAuthClassifier { guard let json, let wire = try? HelperModelWire.decodeJSON(json) else { - return .openai(.gpt56Luna) + return .openai(.gpt6Luna) } switch wire.provider { case LLMProviderChoice.openai.rawValue: @@ -81,7 +81,7 @@ enum ConnectorAuthClassifier { default: break } - return .openai(.gpt56Luna) + return .openai(.gpt6Luna) } private static func thinkingID(_ json: String?) -> String? { diff --git a/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift b/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift index 053530fe..5ede479e 100644 --- a/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift +++ b/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift @@ -70,7 +70,7 @@ actor ConfiguredScriptReviewer: ScriptReviewer { debugLog("Default helper reviewer also failed; denying review.") let failure = LLMFailureClassifier.classify(error, provider: selectedModel.provider) if case .outOfCredits = failure { - ModelProviderLimitCenter.shared.report(raw: error.localizedDescription) + ModelProviderLimitRouting.report(raw: error.localizedDescription) } else { LLMFailureReporter.shared.report(failure) } diff --git a/ui/SharedAgentRuntime/Support/LLM/LLMModelThinkingSettings.swift b/ui/SharedAgentRuntime/Support/LLM/LLMModelThinkingSettings.swift index b237e65c..3aba0c71 100644 --- a/ui/SharedAgentRuntime/Support/LLM/LLMModelThinkingSettings.swift +++ b/ui/SharedAgentRuntime/Support/LLM/LLMModelThinkingSettings.swift @@ -48,7 +48,7 @@ final class LLMModelThinkingSettings: ObservableObject { let match = model.thinkingOptions.first(where: { $0.id == id }) { return match } - return model.preferredHighThinkingOption + return model.preferredMediumThinkingOption } func setThinking(_ option: ModelThinkingOption, for model: LLMModelChoice) { diff --git a/ui/SharedAgentRuntime/Support/LLM/ModelProviderLimitRouting.swift b/ui/SharedAgentRuntime/Support/LLM/ModelProviderLimitRouting.swift new file mode 100644 index 00000000..af36e7d4 --- /dev/null +++ b/ui/SharedAgentRuntime/Support/LLM/ModelProviderLimitRouting.swift @@ -0,0 +1,17 @@ +import Foundation + +/// Shared runtime is compiled into services that do not include the app UI. +/// The app installs `handler` so a spend limit can use the create-plugin modal. +/// Services with no handler fall back to the global failure reporter. +enum ModelProviderLimitRouting { + nonisolated(unsafe) static var handler: (@Sendable (String) -> Void)? + + @MainActor + static func report(raw: String) { + if let handler { + handler(raw) + return + } + LLMFailureReporter.shared.report(.outOfCredits(provider: "your model provider")) + } +} diff --git a/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift b/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift index d701dc2c..3262fb72 100644 --- a/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift +++ b/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift @@ -52,7 +52,7 @@ actor ConfiguredMemorySummarizer: MemorySummarizer { ) let failure = LLMFailureClassifier.classify(error, provider: selectedModel.provider) if case .outOfCredits = failure { - ModelProviderLimitCenter.shared.report(raw: error.localizedDescription) + ModelProviderLimitRouting.report(raw: error.localizedDescription) } else { LLMFailureReporter.shared.report(failure) } diff --git a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift index 148cdfc1..5f54f936 100644 --- a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift +++ b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift @@ -305,7 +305,8 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { private static func reviewerSystemPrompt(for userGoal: String?) -> String { """ - Review the user's goal, manifest, test_input_json, exact Go source, and direct test output. + Review the user's goal, manifest, test plan, and exact Go source. + \(PluginAcceptanceDirections.reviewerText(forUserGoal: userGoal)) \(ScriptExecContractPrompts.pluginFactoryReviewerGuide()) \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) """ @@ -455,14 +456,18 @@ func collectFactoryModelStream( timeoutNanoseconds: UInt64 = LLMHTTPTimeouts.resourceNanoseconds, onFirstText: (@Sendable () async -> Void)? = nil ) async throws -> (text: String, usage: AgentTokenUsage?) { + // Detached so the deadline is not queued behind the builder actor. A stream + // that keeps the actor busy otherwise never reaches the timer, and the + // create dialog stays on the last progress line. try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(String, AgentTokenUsage?), Error>) in let reply = FactoryModelReplyOnce(continuation) - let worker = Task { + let worker = Task.detached { do { - reply.resume( - returning: try await collectFactoryAgentStream(stream, onFirstText: onFirstText) - ) + let result = try await collectFactoryAgentStream(stream, onFirstText: onFirstText) + reply.finish() + reply.resume(returning: result) } catch { + reply.finish() if LLMHTTPTimeouts.isTimeout(error) { reply.resume(throwing: PluginFactoryModelError.timedOut(role)) } else { @@ -470,7 +475,7 @@ func collectFactoryModelStream( } } } - Task { + let timer = Task.detached { do { try await Task.sleep(nanoseconds: timeoutNanoseconds) } catch { @@ -479,6 +484,7 @@ func collectFactoryModelStream( worker.cancel() reply.resume(throwing: PluginFactoryModelError.timedOut(role)) } + reply.onFinish = { timer.cancel() } } } @@ -490,6 +496,7 @@ private func collectFactoryAgentStream( var usage: AgentTokenUsage? var didSignalFirstText = false for try await event in stream { + try Task.checkCancellation() switch event { case .text(let chunk): text += chunk @@ -507,12 +514,22 @@ private func collectFactoryAgentStream( private final class FactoryModelReplyOnce: @unchecked Sendable { private let lock = NSLock() private var continuation: CheckedContinuation<(String, AgentTokenUsage?), Error>? + var onFinish: (@Sendable () -> Void)? init(_ continuation: CheckedContinuation<(String, AgentTokenUsage?), Error>) { self.continuation = continuation } + func finish() { + lock.lock() + let onFinish = self.onFinish + self.onFinish = nil + lock.unlock() + onFinish?() + } + func resume(returning result: (String, AgentTokenUsage?)) { + finish() lock.lock() let continuation = self.continuation self.continuation = nil @@ -521,6 +538,7 @@ private final class FactoryModelReplyOnce: @unchecked Sendable { } func resume(throwing error: Error) { + finish() lock.lock() let continuation = self.continuation self.continuation = nil diff --git a/ui/ui/Plugins/PluginCreationController.swift b/ui/ui/Plugins/PluginCreationController.swift index 2c285146..e3823827 100644 --- a/ui/ui/Plugins/PluginCreationController.swift +++ b/ui/ui/Plugins/PluginCreationController.swift @@ -44,7 +44,7 @@ final class PluginCreationController: ObservableObject { ("skill", "Package skills"), ("factory", "Build guest program"), ("review", "Safety review"), - ("trial", "Trial run"), + ("tests", "Run tests"), ] @Published private(set) var phase: Phase = .idle @@ -550,13 +550,13 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("docs") markProgressCompleted("skill") setProgressStep("factory", status: .failed) - case "trial": + case "tests": markProgressCompleted("credentials") markProgressCompleted("spec") markProgressCompleted("docs") markProgressCompleted("skill") markProgressCompleted("factory") - setProgressStep("trial", status: .failed) + setProgressStep("tests", status: .failed) case "review": markProgressCompleted("credentials") markProgressCompleted("spec") @@ -591,13 +591,14 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("docs") markProgressCompleted("skill") markProgressActive("factory") - case "trial": + case "tests": markProgressCompleted("credentials") markProgressCompleted("spec") markProgressCompleted("docs") markProgressCompleted("skill") markProgressCompleted("factory") - markProgressActive("trial") + markProgressCompleted("review") + markProgressActive("tests") case "review": markProgressCompleted("credentials") markProgressCompleted("spec") @@ -612,7 +613,7 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("skill") markProgressCompleted("factory") markProgressCompleted("review") - markProgressCompleted("trial") + markProgressCompleted("tests") default: break } @@ -625,10 +626,14 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("skill") markProgressActive("factory") } + if message.contains("live_acceptance_started") { + markProgressCompleted("factory") + markProgressCompleted("review") + markProgressActive("tests") + } if message.contains("review decision=approved") { markProgressCompleted("factory") markProgressCompleted("review") - markProgressActive("trial") } if message.contains("review decision=rejected") || message.contains("review rejected=") { markProgressCompleted("factory") @@ -664,7 +669,7 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("skill") markProgressCompleted("factory") markProgressCompleted("review") - markProgressCompleted("trial") + markProgressCompleted("tests") await PluginFactoryListStore.shared.reload() if let saved = parseSuccessResult(result.resultJSON) { phase = .succeeded(pluginID: saved.pluginID, outcome: .plugin) diff --git a/ui/ui/Views/AgentProfileSettingsView.swift b/ui/ui/Views/AgentProfileSettingsView.swift index 8a53d33c..99b7b191 100644 --- a/ui/ui/Views/AgentProfileSettingsView.swift +++ b/ui/ui/Views/AgentProfileSettingsView.swift @@ -14,7 +14,7 @@ struct AgentProfileSettingsView: View { @State private var draftAlias = "" @State private var draftInstructions = "" @State private var draftModel: LLMModelChoice = .defaultHelperModel - @State private var draftThinking: ModelThinkingOption = OpenAIModel.gpt56Luna.defaultThinkingOption + @State private var draftThinking: ModelThinkingOption = OpenAIModel.gpt6Luna.defaultThinkingOption @State private var draftRAG = AgentProfileRAGConfig.default @State private var draftEnabled = true @State private var draftCapabilities = AgentProfileCapabilities() diff --git a/ui/ui/uiApp.swift b/ui/ui/uiApp.swift index 394fff7b..854d55b8 100644 --- a/ui/ui/uiApp.swift +++ b/ui/ui/uiApp.swift @@ -19,6 +19,11 @@ struct uiApp: App { JobResultPanelSession.isPanelOnlyLaunch = true JobResultPanelSession.allowsTermination = false } + ModelProviderLimitRouting.handler = { raw in + Task { @MainActor in + ModelProviderLimitCenter.shared.report(raw: raw) + } + } RuntimeLog.shared.addUISinkOnce { message in Task { await ServiceLogRecorder.shared.record( diff --git a/ui/uiTests/AgentProfileBuiltinFactoryTests.swift b/ui/uiTests/AgentProfileBuiltinFactoryTests.swift index 8ef5fd6b..0c76fa5a 100644 --- a/ui/uiTests/AgentProfileBuiltinFactoryTests.swift +++ b/ui/uiTests/AgentProfileBuiltinFactoryTests.swift @@ -19,15 +19,20 @@ import Testing let researcherModel = try JSONDecoder().decode(LLMModelChoice.self, from: researcher.modelJSON) let generalistModel = try JSONDecoder().decode(LLMModelChoice.self, from: generalist.modelJSON) - #expect(orchestratorModel.id == "openai:gpt-5.6-sol") - #expect(developerModel.id == "openai:gpt-5.6-terra") - #expect(researcherModel.id == "openai:gpt-5.6-terra") - #expect(generalistModel.id == "openai:gpt-5.6-luna") + #expect(orchestratorModel.id == "openai:gpt-6-luna") + #expect(developerModel.id == "openai:gpt-6-luna") + #expect(researcherModel.id == "openai:gpt-6-luna") + #expect(generalistModel.id == "openai:gpt-6-luna") let orchestratorThinking = try JSONDecoder().decode( ModelThinkingOption.self, from: orchestrator.thinkingJSON! ) + let developerThinking = try JSONDecoder().decode( + ModelThinkingOption.self, + from: developer.thinkingJSON! + ) #expect(orchestratorThinking.id == "high") + #expect(developerThinking.id == "medium") } } diff --git a/ui/uiTests/FactoryModelStreamTimeoutTests.swift b/ui/uiTests/FactoryModelStreamTimeoutTests.swift new file mode 100644 index 00000000..cbbdc76c --- /dev/null +++ b/ui/uiTests/FactoryModelStreamTimeoutTests.swift @@ -0,0 +1,34 @@ +import Foundation +import Structure +import Testing +@testable import ui + +@Suite struct FactoryModelStreamTimeoutTests { + private actor BuilderActor { + func collect( + _ stream: AsyncThrowingStream, + timeoutNanoseconds: UInt64 + ) async throws -> String { + let (text, _) = try await collectFactoryModelStream( + stream, + role: "builder", + timeoutNanoseconds: timeoutNanoseconds + ) + return text + } + } + + @Test func stalledBuilderStreamFailsInsteadOfHanging() async throws { + let stream = AsyncThrowingStream { continuation in + continuation.yield(.text("partial draft")) + } + let started = ContinuousClock().now + do { + _ = try await BuilderActor().collect(stream, timeoutNanoseconds: 300_000_000) + Issue.record("A stream that never finishes must time out") + } catch let error as PluginFactoryModelError { + #expect(error == .timedOut("builder")) + } + #expect(ContinuousClock().now - started < .seconds(5)) + } +} diff --git a/ui/uiTests/uiTests.swift b/ui/uiTests/uiTests.swift index 6287904a..a952e81d 100644 --- a/ui/uiTests/uiTests.swift +++ b/ui/uiTests/uiTests.swift @@ -194,7 +194,7 @@ import DBRepository @Test func llmProviderDefaultsToExpectedModels() { #expect(LLMProviderChoice.google.defaultModel.displayName == "gemini-3.7-flash") - #expect(LLMProviderChoice.openai.defaultModel.displayName == "gpt-5.6-luna") + #expect(LLMProviderChoice.openai.defaultModel.displayName == "gpt-6-luna") #expect(LLMProviderChoice.google.apiKeyEnvironmentKeys.contains("GEMINI_API_KEY")) #expect(LLMProviderChoice.openai.apiKeyEnvironmentKeys.contains("OPENAI_API_KEY")) } @@ -307,16 +307,19 @@ import DBRepository } @Test func helperModelChoicesExposeEverySupportedModel() { - #expect(LLMModelChoice.allCases.count == 9) + #expect(LLMModelChoice.allCases.count == 12) #expect(LLMModelChoice.allCases.contains(.gemini(.gemini37Flash))) #expect(LLMModelChoice.allCases.contains(.gemini(.gemini25FlashLite))) #expect(LLMModelChoice.allCases.contains(.openai(.gpt55))) #expect(LLMModelChoice.allCases.contains(.openai(.gpt56Luna))) #expect(LLMModelChoice.allCases.contains(.openai(.gpt56Terra))) #expect(LLMModelChoice.allCases.contains(.openai(.gpt56Sol))) - #expect(LLMModelChoice.defaultHelperModel == .openai(.gpt56Luna)) - #expect(LLMModelChoice.defaultPluginBuilderModel == .openai(.gpt56Terra)) - #expect(LLMModelChoice.defaultPluginBuilderModel.preferredHighThinkingOption.id == "high") + #expect(LLMModelChoice.allCases.contains(.openai(.gpt6Luna))) + #expect(LLMModelChoice.allCases.contains(.openai(.gpt6Sol))) + #expect(LLMModelChoice.allCases.contains(.openai(.gpt6Astra))) + #expect(LLMModelChoice.defaultHelperModel == .openai(.gpt6Luna)) + #expect(LLMModelChoice.defaultPluginBuilderModel == .openai(.gpt6Luna)) + #expect(LLMModelChoice.defaultPluginBuilderModel.preferredMediumThinkingOption.id == "medium") } @MainActor @Test func modelThinkingSettingsPersistsPerModelSelection() async { @@ -332,7 +335,7 @@ import DBRepository #expect(reloaded.thinking(for: .openai(.gpt56Sol)).id == "high") #expect(reloaded.thinking(for: .openai(.gpt56Luna)).id == "medium") #expect(reloaded.pluginSafetyReviewerThinking(for: .openai(.gpt56Luna)).id == "medium") - #expect(reloaded.pluginBuilderThinking(for: .openai(.gpt56Terra)).id == "high") + #expect(reloaded.pluginBuilderThinking(for: .openai(.gpt56Terra)).id == "medium") let medium = OpenAIModel.gpt56Terra.thinkingOptions.first { $0.id == "medium" }! settings.setPluginBuilderThinking(medium, for: .openai(.gpt56Terra)) try? await Task.sleep(nanoseconds: 100_000_000)