From d329a5fe6efb305ee27640aa9542c57f6451bfad Mon Sep 17 00:00:00 2001 From: "vinit.ka" Date: Sun, 12 Apr 2026 22:04:25 +0530 Subject: [PATCH] stash changes: client executed tools and tool approval flow --- docs/core-concepts/tools-function-calling.md | 236 +++ src/Concerns/CallsTools.php | 256 +++- src/Enums/StreamEventType.php | 1 + .../ToolApprovalRequestBroadcast.php | 7 + src/Exceptions/PrismException.php | 14 + src/Providers/Anthropic/Handlers/Stream.php | 12 +- .../Anthropic/Handlers/Structured.php | 22 +- src/Providers/Anthropic/Handlers/Text.php | 16 +- src/Providers/DeepSeek/Handlers/Stream.php | 12 +- src/Providers/DeepSeek/Handlers/Text.php | 25 +- src/Providers/Gemini/Handlers/Stream.php | 13 +- src/Providers/Gemini/Handlers/Structured.php | 17 +- src/Providers/Gemini/Handlers/Text.php | 22 +- src/Providers/Groq/Handlers/Stream.php | 12 +- src/Providers/Groq/Handlers/Text.php | 22 +- src/Providers/Mistral/Handlers/Stream.php | 12 +- src/Providers/Mistral/Handlers/Text.php | 22 +- src/Providers/Ollama/Handlers/Stream.php | 12 +- src/Providers/Ollama/Handlers/Text.php | 34 +- src/Providers/OpenAI/Handlers/Stream.php | 12 +- src/Providers/OpenAI/Handlers/Structured.php | 15 +- src/Providers/OpenAI/Handlers/Text.php | 22 +- src/Providers/OpenRouter/Handlers/Stream.php | 26 +- .../OpenRouter/Handlers/Structured.php | 20 +- src/Providers/OpenRouter/Handlers/Text.php | 25 +- src/Providers/XAI/Handlers/Stream.php | 12 +- src/Providers/XAI/Handlers/Text.php | 19 +- src/Providers/Z/Handlers/Structured.php | 88 +- src/Providers/Z/Handlers/Text.php | 66 +- src/Providers/Z/Maps/ToolCallMap.php | 23 + src/Streaming/Adapters/BroadcastAdapter.php | 3 + .../Adapters/DataProtocolAdapter.php | 41 +- .../Events/ToolApprovalRequestEvent.php | 42 + src/Streaming/StreamCollector.php | 34 +- src/Structured/PendingRequest.php | 2 + src/Structured/Request.php | 10 + src/Structured/Response.php | 4 + src/Structured/ResponseBuilder.php | 1 + src/Structured/Step.php | 4 + src/Text/PendingRequest.php | 6 +- src/Text/Request.php | 10 + src/Text/Response.php | 4 + src/Text/ResponseBuilder.php | 2 + src/Text/Step.php | 4 + src/Tool.php | 79 +- .../Messages/AssistantMessage.php | 6 +- .../Messages/ToolResultMessage.php | 17 +- src/ValueObjects/ToolApprovalRequest.php | 33 + src/ValueObjects/ToolApprovalResponse.php | 32 + tests/Concerns/CallsToolsConcurrentTest.php | 21 +- tests/Concerns/CallsToolsTest.php | 85 +- .../anthropic/stream-with-approval-tool-1.sse | 30 + .../stream-with-client-executed-tool-1.sse | 31 + .../structured-with-approval-phase2-1.json | 1 + .../structured-with-approval-tool-1.json | 1 + ...tructured-with-client-executed-tool-1.json | 2 + .../text-with-approval-phase2-1.json | 1 + .../anthropic/text-with-approval-tool-1.json | 1 + .../text-with-client-executed-tool-1.json | 2 + .../anthropic/text-with-mixed-tools-1.json | 1 + .../deepseek/stream-with-approval-tool-1.sse | 7 + .../stream-with-client-executed-tool-1.sse | 9 + .../deepseek/text-with-approval-tool-1.json | 1 + .../text-with-client-executed-tool-1.json | 2 + .../gemini/stream-with-approval-tool-1.json | 1 + .../stream-with-client-executed-tool-1.json | 3 + .../structured-with-approval-phase2-1.json | 35 + .../structured-with-approval-tool-1.json | 39 + ...tructured-with-client-executed-tool-1.json | 40 + .../gemini/text-with-approval-tool-1.json | 39 + .../text-with-client-executed-tool-1.json | 40 + .../groq/stream-with-approval-tool-1.sse | 7 + .../stream-with-client-executed-tool-1.sse | 9 + .../groq/text-with-approval-tool-1.json | 1 + .../text-with-client-executed-tool-1.json | 2 + .../mistral/stream-with-approval-phase2-1.sse | 7 + .../mistral/stream-with-approval-tool-1.sse | 5 + .../stream-with-client-executed-tool-1.sse | 6 + .../mistral/text-with-approval-phase2-1.json | 21 + .../mistral/text-with-approval-tool-1.json | 31 + .../text-with-client-executed-tool-1.json | 32 + .../ollama/stream-with-approval-tool-1.sse | 1 + .../stream-with-client-executed-tool-1.sse | 3 + .../ollama/text-with-approval-tool-1.json | 1 + .../text-with-client-executed-tool-1.json | 2 + .../openai/stream-with-approval-phase2-1.json | 26 + .../openai/stream-with-approval-tool-1.json | 20 + .../stream-with-client-executed-tool-1.json | 22 + .../structured-with-approval-phase2-1.json | 35 + .../structured-with-approval-tool-1.json | 30 + ...tructured-with-client-executed-tool-1.json | 31 + .../openai/text-with-approval-phase2-1.json | 34 + .../openai/text-with-approval-tool-1.json | 30 + .../text-with-client-executed-tool-1.json | 31 + .../stream-with-approval-phase2-1.sse | 7 + .../stream-with-approval-tool-1.sse | 7 + .../stream-with-client-executed-tool-1.sse | 9 + .../structured-with-approval-phase2-1.json | 1 + .../structured-with-approval-tool-1.json | 1 + ...tructured-with-client-executed-tool-1.json | 1 + .../structured-with-multiple-tools-1.json | 1 + .../structured-with-multiple-tools-2.json | 1 + .../structured-with-single-tool-1.json | 1 + .../structured-with-single-tool-2.json | 1 + .../structured-with-tool-orchestration-1.json | 1 + .../structured-with-tool-orchestration-2.json | 1 + .../structured-with-tool-orchestration-3.json | 1 + .../structured-without-tool-calls-1.json | 1 + .../openrouter/text-with-approval-tool-1.json | 1 + .../text-with-client-executed-tool-1.json | 2 + .../xai/stream-with-approval-tool-1.json | 7 + .../stream-with-client-executed-tool-1.json | 9 + .../xai/text-with-approval-tool-1.json | 33 + .../xai/text-with-client-executed-tool-1.json | 34 + .../z/generate-text-with-missing-meta-1.json | 19 + .../z/generate-text-with-missing-usage-1.json | 16 + .../z/structured-missing-usage-1.json | 16 + .../z/structured-with-approval-phase2-1.json | 21 + .../z/structured-with-missing-meta-1.json | 19 + .../z/text-with-approval-phase2-1.json | 21 + .../Fixtures/z/text-with-approval-tool-1.json | 31 + .../z/text-with-client-executed-tool-1.json | 31 + .../Anthropic/AnthropicTextRequestTest.php | 6 +- .../Providers/Anthropic/AnthropicTextTest.php | 125 ++ tests/Providers/Anthropic/StreamTest.php | 79 + .../Anthropic/StructuredWithToolsTest.php | 116 ++ tests/Providers/DeepSeek/StreamTest.php | 83 + tests/Providers/DeepSeek/TextTest.php | 47 + tests/Providers/Gemini/GeminiStreamTest.php | 83 + tests/Providers/Gemini/GeminiTextTest.php | 49 + .../Gemini/StructuredWithToolsTest.php | 113 ++ tests/Providers/Groq/GroqTextTest.php | 46 + tests/Providers/Groq/StreamTest.php | 83 + tests/Providers/Mistral/MistralTextTest.php | 81 + tests/Providers/Mistral/StreamTest.php | 167 ++ tests/Providers/Ollama/StreamTest.php | 83 + tests/Providers/Ollama/TextTest.php | 48 + tests/Providers/OpenAI/StreamTest.php | 168 ++ .../OpenAI/StructuredWithToolsTest.php | 113 ++ tests/Providers/OpenAI/TextTest.php | 85 ++ tests/Providers/OpenRouter/StreamTest.php | 167 ++ .../OpenRouter/StructuredWithToolsTest.php | 297 ++++ tests/Providers/OpenRouter/TextTest.php | 47 + tests/Providers/XAI/StreamTest.php | 83 + tests/Providers/XAI/XAITextTest.php | 47 + tests/Providers/Z/StructuredWithToolsTest.php | 132 ++ tests/Providers/Z/ZStructuredTest.php | 179 ++- tests/Providers/Z/ZTextTest.php | 128 ++ .../Adapters/BroadcastAdapterTest.php | 7 +- .../Adapters/DataProtocolAdapterTest.php | 140 ++ tests/Streaming/StreamCollectorTest.php | 110 ++ tests/Structured/PendingRequestTest.php | 13 + tests/Text/PendingRequestTest.php | 11 + tests/ToolApprovalTest.php | 1351 +++++++++++++++++ tests/ToolTest.php | 12 + 155 files changed, 6579 insertions(+), 222 deletions(-) create mode 100644 src/Events/Broadcasting/ToolApprovalRequestBroadcast.php create mode 100644 src/Providers/Z/Maps/ToolCallMap.php create mode 100644 src/Streaming/Events/ToolApprovalRequestEvent.php create mode 100644 src/ValueObjects/ToolApprovalRequest.php create mode 100644 src/ValueObjects/ToolApprovalResponse.php create mode 100644 tests/Fixtures/anthropic/stream-with-approval-tool-1.sse create mode 100644 tests/Fixtures/anthropic/stream-with-client-executed-tool-1.sse create mode 100644 tests/Fixtures/anthropic/structured-with-approval-phase2-1.json create mode 100644 tests/Fixtures/anthropic/structured-with-approval-tool-1.json create mode 100644 tests/Fixtures/anthropic/structured-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/anthropic/text-with-approval-phase2-1.json create mode 100644 tests/Fixtures/anthropic/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/anthropic/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/anthropic/text-with-mixed-tools-1.json create mode 100644 tests/Fixtures/deepseek/stream-with-approval-tool-1.sse create mode 100644 tests/Fixtures/deepseek/stream-with-client-executed-tool-1.sse create mode 100644 tests/Fixtures/deepseek/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/deepseek/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/gemini/stream-with-approval-tool-1.json create mode 100644 tests/Fixtures/gemini/stream-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/gemini/structured-with-approval-phase2-1.json create mode 100644 tests/Fixtures/gemini/structured-with-approval-tool-1.json create mode 100644 tests/Fixtures/gemini/structured-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/gemini/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/gemini/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/groq/stream-with-approval-tool-1.sse create mode 100644 tests/Fixtures/groq/stream-with-client-executed-tool-1.sse create mode 100644 tests/Fixtures/groq/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/groq/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/mistral/stream-with-approval-phase2-1.sse create mode 100644 tests/Fixtures/mistral/stream-with-approval-tool-1.sse create mode 100644 tests/Fixtures/mistral/stream-with-client-executed-tool-1.sse create mode 100644 tests/Fixtures/mistral/text-with-approval-phase2-1.json create mode 100644 tests/Fixtures/mistral/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/mistral/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/ollama/stream-with-approval-tool-1.sse create mode 100644 tests/Fixtures/ollama/stream-with-client-executed-tool-1.sse create mode 100644 tests/Fixtures/ollama/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/ollama/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/openai/stream-with-approval-phase2-1.json create mode 100644 tests/Fixtures/openai/stream-with-approval-tool-1.json create mode 100644 tests/Fixtures/openai/stream-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/openai/structured-with-approval-phase2-1.json create mode 100644 tests/Fixtures/openai/structured-with-approval-tool-1.json create mode 100644 tests/Fixtures/openai/structured-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/openai/text-with-approval-phase2-1.json create mode 100644 tests/Fixtures/openai/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/openai/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/openrouter/stream-with-approval-phase2-1.sse create mode 100644 tests/Fixtures/openrouter/stream-with-approval-tool-1.sse create mode 100644 tests/Fixtures/openrouter/stream-with-client-executed-tool-1.sse create mode 100644 tests/Fixtures/openrouter/structured-with-approval-phase2-1.json create mode 100644 tests/Fixtures/openrouter/structured-with-approval-tool-1.json create mode 100644 tests/Fixtures/openrouter/structured-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/openrouter/structured-with-multiple-tools-1.json create mode 100644 tests/Fixtures/openrouter/structured-with-multiple-tools-2.json create mode 100644 tests/Fixtures/openrouter/structured-with-single-tool-1.json create mode 100644 tests/Fixtures/openrouter/structured-with-single-tool-2.json create mode 100644 tests/Fixtures/openrouter/structured-with-tool-orchestration-1.json create mode 100644 tests/Fixtures/openrouter/structured-with-tool-orchestration-2.json create mode 100644 tests/Fixtures/openrouter/structured-with-tool-orchestration-3.json create mode 100644 tests/Fixtures/openrouter/structured-without-tool-calls-1.json create mode 100644 tests/Fixtures/openrouter/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/openrouter/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/xai/stream-with-approval-tool-1.json create mode 100644 tests/Fixtures/xai/stream-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/xai/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/xai/text-with-client-executed-tool-1.json create mode 100644 tests/Fixtures/z/generate-text-with-missing-meta-1.json create mode 100644 tests/Fixtures/z/generate-text-with-missing-usage-1.json create mode 100644 tests/Fixtures/z/structured-missing-usage-1.json create mode 100644 tests/Fixtures/z/structured-with-approval-phase2-1.json create mode 100644 tests/Fixtures/z/structured-with-missing-meta-1.json create mode 100644 tests/Fixtures/z/text-with-approval-phase2-1.json create mode 100644 tests/Fixtures/z/text-with-approval-tool-1.json create mode 100644 tests/Fixtures/z/text-with-client-executed-tool-1.json create mode 100644 tests/Providers/OpenRouter/StructuredWithToolsTest.php create mode 100644 tests/Providers/Z/StructuredWithToolsTest.php create mode 100644 tests/ToolApprovalTest.php diff --git a/docs/core-concepts/tools-function-calling.md b/docs/core-concepts/tools-function-calling.md index 0fdf9dff1..4aa9e3033 100644 --- a/docs/core-concepts/tools-function-calling.md +++ b/docs/core-concepts/tools-function-calling.md @@ -388,6 +388,242 @@ use Prism\Prism\Facades\Tool; $tool = Tool::make(CurrentWeatherTool::class); ``` +## Client-Executed Tools + +Sometimes you need tools that are executed by the client (e.g., frontend application) rather than on the server. Use the `clientExecuted()` method to explicitly mark a tool as client-executed: + +```php +use Prism\Prism\Facades\Tool; + +$clientTool = Tool::as('browser_action') + ->for('Perform an action in the user\'s browser') + ->withStringParameter('action', 'The action to perform') + ->clientExecuted(); +``` + +When the AI calls a client-executed tool, Prism will: +1. Stop execution and return control to your application +2. Set the response's `finishReason` to `FinishReason::ToolCalls` +3. Include the tool calls in the response for your client to execute + +### Handling Client-Executed Tools + +```php +use Prism\Prism\Facades\Prism; +use Prism\Prism\Enums\FinishReason; + +$response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$clientTool]) + ->withMaxSteps(3) + ->withPrompt('Click the submit button') + ->asText(); + +``` + +### Streaming with Client-Executed Tools + +When streaming, client-executed tools emit a `ToolCallEvent` but no `ToolResultEvent`: + +```php + +$response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$clientTool]) + ->withMaxSteps(3) + ->withPrompt('Click the submit button') + ->asStream(); +``` + +> [!NOTE] +> Client-executed tools are useful for scenarios like browser automation, UI interactions, or any operation that must run on the user's device rather than the server. + +## Tool Approval (Human-in-the-Loop) + +Some tools perform sensitive or irreversible actions — sending emails, making payments, deleting records. Tool approval lets you require human confirmation before Prism executes these tools, giving you a human-in-the-loop safety layer. + +### How It Works + +The approval flow is **stateless** and operates in two phases: + +1. **Phase 1 — Approval Request**: The AI calls a tool that requires approval. Prism stops execution and returns the tool call details to your application, giving the user a chance to approve or deny. +2. **Phase 2 — Approval Resolution**: Your application sends back the approval decision. Prism executes approved tools (or records denial reasons), then continues the conversation with the AI. + +### Marking Tools as Requiring Approval + +Use the `requiresApproval()` method to mark a tool: + +```php +use Prism\Prism\Facades\Tool; + +$deleteTool = Tool::as('delete_record') + ->for('Delete a record from the database') + ->withStringParameter('id', 'The record ID to delete') + ->using(function (string $id): string { + // Deletion logic + return "Record {$id} deleted."; + }) + ->requiresApproval(); +``` + +### Conditional Approval + +You can pass a closure to `requiresApproval()` to dynamically decide whether approval is needed based on the tool call arguments: + +```php +use Prism\Prism\Facades\Tool; + +$transferTool = Tool::as('transfer_funds') + ->for('Transfer funds between accounts') + ->withNumberParameter('amount', 'The amount to transfer') + ->withStringParameter('to', 'The destination account') + ->using(function (float $amount, string $to): string { + return "Transferred \${$amount} to {$to}."; + }) + ->requiresApproval(function (array $args): bool { + // Only require approval for large transfers + return ($args['amount'] ?? 0) > 1000; + }); +``` + +### Phase 1 — Handling the Approval Request + +When the AI calls a tool that requires approval, Prism stops and returns control to your application with a `FinishReason::ToolCalls` finish reason, just like client-executed tools: + +```php +use Prism\Prism\Facades\Prism; +use Prism\Prism\Enums\FinishReason; + +$response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$deleteTool]) + ->withMaxSteps(3) + ->withPrompt('Delete record abc-123') + ->asText(); + +if ($response->finishReason === FinishReason::ToolCalls) { + // Check which tool calls need approval + foreach ($response->toolApprovalRequests as $approvalRequest) { + echo "Approval needed for tool call: {$approvalRequest->toolCallId}\n"; + } + + // Inspect the full tool calls for details + foreach ($response->toolCalls as $toolCall) { + echo "The AI wants to call: {$toolCall->name}\n"; + echo "With arguments: " . json_encode($toolCall->arguments()) . "\n"; + } +} +``` + +### Phase 2 — Sending the Approval Response + +Once the user approves or denies, send the decision back by including a `ToolResultMessage` with `ToolApprovalResponse` objects in the message history: + +```php +use Prism\Prism\Facades\Prism; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; +use Prism\Prism\ValueObjects\Messages\UserMessage; +use Prism\Prism\ValueObjects\ToolApprovalResponse; + +// Reconstruct the conversation with the approval response +$response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$deleteTool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete record abc-123'), + // The assistant's response from Phase 1 (contains toolCalls and toolApprovalRequests) + new AssistantMessage( + content: '', + toolCalls: $previousResponse->toolCalls, + toolApprovalRequests: $previousResponse->toolApprovalRequests, + ), + // Your approval decision + new ToolResultMessage( + toolResults: [], + toolApprovalResponses: [ + new ToolApprovalResponse( + approvalId: $previousResponse->toolCalls[0]->id, + approved: true, // or false to deny + ), + ], + ), + ]) + ->asText(); + +// The tool is now executed and the AI continues with the result +echo $response->text; +``` + +To deny a tool call, set `approved: false` and optionally provide a reason: + +```php +new ToolApprovalResponse( + approvalId: $toolCall->id, + approved: false, + reason: 'User decided not to delete the record', +), +``` + +When denied, the denial reason is passed to the AI as the tool result, allowing it to respond appropriately. + +### Streaming with Tool Approval + +In streaming mode, Phase 1 emits a `ToolApprovalRequestEvent` for each tool that needs approval: + +```php +use Prism\Prism\Facades\Prism; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; + +$stream = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$deleteTool]) + ->withMaxSteps(3) + ->withPrompt('Delete record abc-123') + ->asStream(); + +foreach ($stream as $event) { + if ($event instanceof ToolApprovalRequestEvent) { + // Present approval UI to the user + echo "Approval needed for: {$event->toolCall->name}\n"; + echo "Arguments: " . json_encode($event->toolCall->arguments()) . "\n"; + } +} +``` + +Phase 2 works the same way as non-streaming — include the `ToolApprovalResponse` in the messages and call `asStream()` again. Prism will emit `ToolResultEvent`s for the resolved tools before continuing the AI response. + +### Mixing Approval Tools with Regular Tools + +Approval-required tools can coexist with regular server-executed tools and client-executed tools. In a single step, Prism will: + +1. Execute regular server-side tools immediately +2. Collect approval-required tools and emit approval requests +3. Skip client-executed tools (returning them for client handling) + +```php +$regularTool = Tool::as('lookup') + ->for('Look up a record') + ->withStringParameter('id', 'The record ID') + ->using(fn (string $id): string => "Record: {$id}"); + +$approvalTool = Tool::as('delete') + ->for('Delete a record') + ->withStringParameter('id', 'The record ID') + ->using(fn (string $id): string => "Deleted: {$id}") + ->requiresApproval(); + +$response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$regularTool, $approvalTool]) + ->withMaxSteps(5) + ->withPrompt('Look up record abc-123 and then delete it') + ->asText(); +``` + +In this example, if the AI calls both tools in one step, `lookup` executes immediately while `delete` pauses for approval. The results from `lookup` are included alongside the approval request. + ## Tool Choice Options You can control how the AI uses tools with the `withToolChoice` method: diff --git a/src/Concerns/CallsTools.php b/src/Concerns/CallsTools.php index eae5ac824..f0038b708 100644 --- a/src/Concerns/CallsTools.php +++ b/src/Concerns/CallsTools.php @@ -8,11 +8,23 @@ use Illuminate\Support\Facades\Concurrency; use Illuminate\Support\ItemNotFoundException; use Illuminate\Support\MultipleItemsFoundException; +use Prism\Prism\Enums\FinishReason; use Prism\Prism\Exceptions\PrismException; use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; +use Prism\Prism\Streaming\StreamState; +use Prism\Prism\Structured\Request as StructuredRequest; +use Prism\Prism\Text\Request as TextRequest; use Prism\Prism\Tool; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolError; use Prism\Prism\ValueObjects\ToolOutput; @@ -25,15 +37,20 @@ trait CallsTools * * @param Tool[] $tools * @param ToolCall[] $toolCalls + * @param ToolApprovalRequest[] $approvalRequests Tool calls requiring approval (collected by reference) * @return ToolResult[] */ - protected function callTools(array $tools, array $toolCalls): array + protected function callTools(array $tools, array $toolCalls, bool &$hasPendingToolCalls, array &$approvalRequests = []): array { $toolResults = []; - // Consume generator to execute all tools and collect results - foreach ($this->callToolsAndYieldEvents($tools, $toolCalls, EventID::generate(), $toolResults) as $event) { - // Events are discarded for non-streaming handlers + foreach ($this->callToolsAndYieldEvents($tools, $toolCalls, EventID::generate(), $toolResults, $hasPendingToolCalls) as $event) { + if ($event instanceof ToolApprovalRequestEvent) { + $approvalRequests[] = new ToolApprovalRequest( + approvalId: $event->approvalId, + toolCallId: $event->toolCall->id, + ); + } } return $toolResults; @@ -45,15 +62,18 @@ protected function callTools(array $tools, array $toolCalls): array * @param Tool[] $tools * @param ToolCall[] $toolCalls * @param ToolResult[] $toolResults Results are collected into this array by reference - * @return Generator + * @return Generator */ - protected function callToolsAndYieldEvents(array $tools, array $toolCalls, string $messageId, array &$toolResults): Generator + protected function callToolsAndYieldEvents(array $tools, array $toolCalls, string $messageId, array &$toolResults, bool &$hasPendingToolCalls): Generator { - $groupedToolCalls = $this->groupToolCallsByConcurrency($tools, $toolCalls); + $approvalRequiredToolCalls = []; + $serverToolCalls = $this->filterServerExecutedToolCalls($tools, $toolCalls, $hasPendingToolCalls, $approvalRequiredToolCalls); + + $groupedToolCalls = $this->groupToolCallsByConcurrency($tools, $serverToolCalls); $executionResults = $this->executeToolsWithConcurrency($tools, $groupedToolCalls, $messageId); - foreach (array_keys($toolCalls) as $index) { + foreach (collect($executionResults)->keys()->sort() as $index) { $result = $executionResults[$index]; $toolResults[] = $result['toolResult']; @@ -62,11 +82,60 @@ protected function callToolsAndYieldEvents(array $tools, array $toolCalls, strin yield $event; } } + + foreach ($approvalRequiredToolCalls as $toolCall) { + yield new ToolApprovalRequestEvent( + id: EventID::generate(), + timestamp: time(), + toolCall: $toolCall, + messageId: $messageId, + approvalId: EventID::generate('apr'), + ); + } } /** + * Filter out client-executed and approval-required tool calls, setting the pending flag if any are found. + * * @param Tool[] $tools * @param ToolCall[] $toolCalls + * @param ToolCall[] $approvalRequiredToolCalls Collected approval-required tool calls (by reference) + * @return array Server-executed tool calls with original indices preserved + */ + protected function filterServerExecutedToolCalls(array $tools, array $toolCalls, bool &$hasPendingToolCalls, array &$approvalRequiredToolCalls = []): array + { + $serverToolCalls = []; + + foreach ($toolCalls as $index => $toolCall) { + try { + $tool = $this->resolveTool($toolCall->name, $tools); + + if ($tool->isClientExecuted()) { + $hasPendingToolCalls = true; + + continue; + } + + if ($tool->needsApproval($toolCall->arguments())) { + $hasPendingToolCalls = true; + $approvalRequiredToolCalls[] = $toolCall; + + continue; + } + + $serverToolCalls[$index] = $toolCall; + } catch (PrismException) { + // Unknown tool - keep it so error handling works in executeToolCall + $serverToolCalls[$index] = $toolCall; + } + } + + return $serverToolCalls; + } + + /** + * @param Tool[] $tools + * @param array $toolCalls * @return array{concurrent: array, sequential: array} */ protected function groupToolCallsByConcurrency(array $tools, array $toolCalls): array @@ -222,8 +291,179 @@ protected function executeToolCall(array $tools, ToolCall $toolCall, string $mes } } + /** + * Yield stream completion events when client-executed tools are pending. + * + * @return Generator + */ + protected function yieldToolCallsFinishEvents(StreamState $state): Generator + { + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls, + usage: $state->usage(), + citations: $state->citations(), + ); + } + + /** + * Resolve pending tool approvals from a previous request (non-streaming). + * + * Scans request messages for a ToolResultMessage with toolApprovalResponses after + * the last AssistantMessage. If found, executes approved tools, creates denial + * results for denied tools, and replaces it with a ToolResultMessage containing + * merged tool results (existing + resolved) and the consumed approval responses. + */ + protected function resolveToolApprovals(StructuredRequest|TextRequest $request): void + { + foreach ($this->resolveToolApprovalsAndYieldEvents($request, EventID::generate()) as $event) { + // Events are discarded for non-streaming handlers + } + } + + /** + * @return Generator + */ + protected function resolveToolApprovalsAndYieldEvents(StructuredRequest|TextRequest $request, string $messageId, ?StreamState $state = null): Generator + { + $messages = $request->messages(); + + $assistantMessage = null; + $assistantMessageIndex = null; + + for ($i = count($messages) - 1; $i >= 0; $i--) { + if ($messages[$i] instanceof AssistantMessage && $messages[$i]->toolCalls !== []) { + $assistantMessage = $messages[$i]; + $assistantMessageIndex = $i; + + break; + } + } + + if (! $assistantMessage instanceof AssistantMessage || $assistantMessageIndex === null) { + return; + } + + $toolsByName = collect($request->tools())->keyBy(fn (Tool $tool): string => $tool->name()); + $isAnyToolApprovalConfigured = collect($assistantMessage->toolCalls)->contains( + fn (ToolCall $toolCall): bool => $toolsByName->get($toolCall->name)?->hasApprovalConfigured() === true, + ); + + if (! $isAnyToolApprovalConfigured) { + return; + } + + $toolMessage = null; + $toolMessageIndex = null; + $counter = count($messages); + + for ($i = $assistantMessageIndex + 1; $i < $counter; $i++) { + if ($messages[$i] instanceof ToolResultMessage) { + $toolMessage = $messages[$i]; + $toolMessageIndex = $i; + + break; + } + } + + if (! $toolMessage instanceof ToolResultMessage) { + $toolMessage = new ToolResultMessage; + $toolMessageIndex = null; + } + + $toolCallIdToApprovalId = []; + foreach ($assistantMessage->toolApprovalRequests as $approvalRequest) { + $toolCallIdToApprovalId[$approvalRequest->toolCallId] = $approvalRequest->approvalId; + } + + $approvalResolvedToolResults = []; + + foreach ($assistantMessage->toolCalls as $toolCall) { + $approvalId = $toolCallIdToApprovalId[$toolCall->id] ?? null; + $approval = $approvalId !== null ? $toolMessage->findByApprovalId($approvalId) : null; + + if (! $approval instanceof ToolApprovalResponse) { + if (collect($toolMessage->toolResults)->contains(fn (ToolResult $tr): bool => $tr->toolCallId === $toolCall->id)) { // tool already executed + continue; + } + if ($toolsByName->get($toolCall->name)?->hasApprovalConfigured() !== true) { + continue; + } + + $approval = new ToolApprovalResponse($approvalId ?? EventID::generate('apr'), false, 'No approval response provided'); + } + + if ($state instanceof StreamState && $state->shouldEmitStreamStart()) { + yield new StreamStartEvent( + id: EventID::generate(), + timestamp: time(), + model: $request->model(), + provider: $request->provider(), + ); + + $state->markStreamStarted(); + } + + if ($approval->approved) { + $result = $this->executeToolCall($request->tools(), $toolCall, $messageId); + + $approvalResolvedToolResults[] = $result['toolResult']; + + foreach ($result['events'] as $event) { + yield $event; + } + + continue; + } + + $reason = $approval->reason ?? 'User denied tool execution'; + + $toolResult = new ToolResult( + toolCallId: $toolCall->id, + toolName: $toolCall->name, + args: $toolCall->arguments(), + result: $reason, + toolCallResultId: $toolCall->resultId, + ); + + $approvalResolvedToolResults[] = $toolResult; + + yield new ToolResultEvent( + id: EventID::generate(), + timestamp: time(), + toolResult: $toolResult, + messageId: $messageId, + success: false, + error: $reason, + ); + } + + if ($toolMessageIndex !== null) { // remove old tool result message + $updatedMessages = array_values(array_filter( + $messages, + fn (int $index): bool => $index !== $toolMessageIndex, + ARRAY_FILTER_USE_KEY, + )); + $request->setMessages($updatedMessages); + } + + // Add new tool result message which also contains results of approval resolved tools + $request->addMessage(new ToolResultMessage( + array_merge($toolMessage->toolResults, $approvalResolvedToolResults), + $toolMessage->toolApprovalResponses + )); + } + /** * @param Tool[] $tools + * + * @throws PrismException */ protected function resolveTool(string $name, array $tools): Tool { diff --git a/src/Enums/StreamEventType.php b/src/Enums/StreamEventType.php index 187d80dbe..afbf75788 100644 --- a/src/Enums/StreamEventType.php +++ b/src/Enums/StreamEventType.php @@ -17,6 +17,7 @@ enum StreamEventType: string case ToolCallDelta = 'tool_call_delta'; case ProviderToolEvent = 'provider_tool_event'; case ToolResult = 'tool_result'; + case ToolApprovalRequest = 'tool_approval_request'; case Citation = 'citation'; case Artifact = 'artifact'; case Error = 'error'; diff --git a/src/Events/Broadcasting/ToolApprovalRequestBroadcast.php b/src/Events/Broadcasting/ToolApprovalRequestBroadcast.php new file mode 100644 index 000000000..91f25ada4 --- /dev/null +++ b/src/Events/Broadcasting/ToolApprovalRequestBroadcast.php @@ -0,0 +1,7 @@ +resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $this->state->reset(); $response = $this->sendRequest($request); @@ -499,7 +501,15 @@ protected function handleToolCalls(Request $request, int $depth): Generator // Execute tools and emit results $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $toolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $toolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } // Add messages to request for next turn if ($toolResults !== []) { diff --git a/src/Providers/Anthropic/Handlers/Structured.php b/src/Providers/Anthropic/Handlers/Structured.php index 244f1747b..33a032fbd 100644 --- a/src/Providers/Anthropic/Handlers/Structured.php +++ b/src/Providers/Anthropic/Handlers/Structured.php @@ -33,6 +33,7 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -53,6 +54,8 @@ public function __construct(protected PendingRequest $client, protected Structur public function handle(): Response { + $this->resolveToolApprovals($this->request); + $this->strategy->appendMessages(); $this->sendRequest(); @@ -168,8 +171,11 @@ protected function handleToolCalls(array $toolCalls, Response $tempResponse): Re protected function executeCustomToolsAndFinalize(array $toolCalls, Response $tempResponse): Response { $customToolCalls = $this->filterCustomToolCalls($toolCalls); - $toolResults = $this->callTools($this->request->tools(), $customToolCalls); - $this->addStep($toolCalls, $tempResponse, $toolResults); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools($this->request->tools(), $customToolCalls, $hasPendingToolCalls, $approvalRequests); + + $this->addStep($toolCalls, $tempResponse, $toolResults, $approvalRequests); return $this->responseBuilder->toResponse(); } @@ -180,7 +186,9 @@ protected function executeCustomToolsAndFinalize(array $toolCalls, Response $tem protected function executeCustomToolsAndContinue(array $toolCalls, Response $tempResponse): Response { $customToolCalls = $this->filterCustomToolCalls($toolCalls); - $toolResults = $this->callTools($this->request->tools(), $customToolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools($this->request->tools(), $customToolCalls, $hasPendingToolCalls, $approvalRequests); $message = new ToolResultMessage($toolResults); if ($toolResultCacheType = $this->request->providerOptions('tool_result_cache_type')) { @@ -189,9 +197,9 @@ protected function executeCustomToolsAndContinue(array $toolCalls, Response $tem $this->request->addMessage($message); $this->request->resetToolChoice(); - $this->addStep($toolCalls, $tempResponse, $toolResults); + $this->addStep($toolCalls, $tempResponse, $toolResults, $approvalRequests); - if ($this->canContinue()) { + if (! $hasPendingToolCalls && $this->canContinue()) { return $this->handle(); } @@ -263,8 +271,9 @@ protected function hasStructuredToolCall(array $toolCalls): bool /** * @param ToolCall[] $toolCalls * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $toolCalls, Response $tempResponse, array $toolResults = []): void + protected function addStep(array $toolCalls, Response $tempResponse, array $toolResults = [], array $toolApprovalRequests = []): void { $data = $this->httpResponse->json(); $isStructuredStep = $this->determineIfStructuredStep($toolCalls, $toolResults); @@ -281,6 +290,7 @@ protected function addStep(array $toolCalls, Response $tempResponse, array $tool toolCalls: $toolCalls, providerToolCalls: $this->extractProviderToolCalls($data), toolResults: $toolResults, + toolApprovalRequests: $toolApprovalRequests, raw: $data, )); } diff --git a/src/Providers/Anthropic/Handlers/Text.php b/src/Providers/Anthropic/Handlers/Text.php index ee6b2e37a..070d5b31f 100644 --- a/src/Providers/Anthropic/Handlers/Text.php +++ b/src/Providers/Anthropic/Handlers/Text.php @@ -30,6 +30,7 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -49,6 +50,8 @@ public function __construct(protected PendingRequest $client, protected TextRequ public function handle(): Response { + $this->resolveToolApprovals($this->request); + $this->sendRequest(); $this->prepareTempResponse(); @@ -95,14 +98,17 @@ public static function buildHttpRequestPayload(PrismRequest $request): array protected function handleToolCalls(): Response { - $toolResults = $this->callTools($this->request->tools(), $this->tempResponse->toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools($this->request->tools(), $this->tempResponse->toolCalls, $hasPendingToolCalls, $approvalRequests); - $this->addStep($toolResults); + $this->addStep($toolResults, $approvalRequests); $this->request->addMessage(new AssistantMessage( $this->tempResponse->text, $this->tempResponse->toolCalls, $this->tempResponse->additionalContent, + $approvalRequests, )); $toolResultMessage = new ToolResultMessage($toolResults); @@ -110,7 +116,7 @@ protected function handleToolCalls(): Response $this->request->addMessage($toolResultMessage); $this->request->resetToolChoice(); - if ($this->responseBuilder->steps->count() < $this->request->maxSteps()) { + if (! $hasPendingToolCalls && $this->responseBuilder->steps->count() < $this->request->maxSteps()) { return $this->handle(); } @@ -126,8 +132,9 @@ protected function handleStop(): Response /** * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $toolResults = []): void + protected function addStep(array $toolResults = [], array $toolApprovalRequests = []): void { $data = $this->httpResponse->json(); @@ -141,6 +148,7 @@ protected function addStep(array $toolResults = []): void meta: $this->tempResponse->meta, messages: $this->request->messages(), systemPrompts: $this->request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: $this->tempResponse->additionalContent, raw: $data, )); diff --git a/src/Providers/DeepSeek/Handlers/Stream.php b/src/Providers/DeepSeek/Handlers/Stream.php index 8ae26e57b..9c0f1982f 100644 --- a/src/Providers/DeepSeek/Handlers/Stream.php +++ b/src/Providers/DeepSeek/Handlers/Stream.php @@ -57,6 +57,8 @@ public function __construct(protected PendingRequest $client) */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $response = $this->sendRequest($request); yield from $this->processStream($response, $request); @@ -381,7 +383,15 @@ protected function handleToolCalls(Request $request, string $text, array $toolCa } $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } $this->state->markStepFinished(); yield new StepFinishEvent( diff --git a/src/Providers/DeepSeek/Handlers/Text.php b/src/Providers/DeepSeek/Handlers/Text.php index c8a15d253..da6e08a32 100644 --- a/src/Providers/DeepSeek/Handlers/Text.php +++ b/src/Providers/DeepSeek/Handlers/Text.php @@ -23,6 +23,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -41,6 +42,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): TextResponse { + $this->resolveToolApprovals($request); + $data = $this->sendRequest($request); $this->validateResponse($data); @@ -59,19 +62,27 @@ protected function handleToolCalls(array $data, Request $request): TextResponse { $toolCalls = ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $this->addStep($data, $request, $toolResults); + $this->addStep($data, $request, $toolResults, $approvalRequests); $request = $request->addMessage(new AssistantMessage( data_get($data, 'choices.0.message.content') ?? '', $toolCalls, - [] + [], + $approvalRequests, )); $request = $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -118,9 +129,10 @@ protected function sendRequest(Request $request): array /** * @param array $data - * @param array $toolResults + * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, array $toolResults = []): void + protected function addStep(array $data, Request $request, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: data_get($data, 'choices.0.message.content') ?? '', @@ -138,6 +150,7 @@ protected function addStep(array $data, Request $request, array $toolResults = [ ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: [], raw: $data, )); diff --git a/src/Providers/Gemini/Handlers/Stream.php b/src/Providers/Gemini/Handlers/Stream.php index e7acfad33..6e37d3a4d 100644 --- a/src/Providers/Gemini/Handlers/Stream.php +++ b/src/Providers/Gemini/Handlers/Stream.php @@ -58,6 +58,8 @@ public function __construct( */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $this->state->reset(); $this->currentThoughtSignature = null; $response = $this->sendRequest($request); @@ -326,6 +328,7 @@ protected function handleToolCalls( array $data = [] ): Generator { $mappedToolCalls = []; + $hasPendingToolCalls = false; // Convert tool calls to ToolCall objects foreach ($this->state->toolCalls() as $toolCallData) { @@ -334,8 +337,16 @@ protected function handleToolCalls( // Execute tools and emit results $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } + // Add messages for next turn and continue streaming if ($toolResults !== []) { // Emit step finish after tool calls $this->state->markStepFinished(); diff --git a/src/Providers/Gemini/Handlers/Structured.php b/src/Providers/Gemini/Handlers/Structured.php index 6d4870f59..52d5853dd 100644 --- a/src/Providers/Gemini/Handlers/Structured.php +++ b/src/Providers/Gemini/Handlers/Structured.php @@ -28,6 +28,7 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -47,6 +48,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): StructuredResponse { + $this->resolveToolApprovals($request); + $data = $this->sendRequest($request); $this->validateResponse($data); @@ -202,17 +205,21 @@ protected function handleStop(array $data, Request $request, FinishReason $finis */ protected function handleToolCalls(array $data, Request $request): StructuredResponse { + $hasPendingToolCalls = false; + $approvalRequests = []; $toolResults = $this->callTools( $request->tools(), - ToolCallMap::map(data_get($data, 'candidates.0.content.parts', [])) + ToolCallMap::map(data_get($data, 'candidates.0.content.parts', [])), + $hasPendingToolCalls, + $approvalRequests, ); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - $this->addStep($data, $request, FinishReason::ToolCalls, $toolResults); + $this->addStep($data, $request, FinishReason::ToolCalls, $toolResults, $approvalRequests); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -222,8 +229,9 @@ protected function handleToolCalls(array $data, Request $request): StructuredRes /** * @param array $data * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, FinishReason $finishReason, array $toolResults = []): void + protected function addStep(array $data, Request $request, FinishReason $finishReason, array $toolResults = [], array $toolApprovalRequests = []): void { $isStructuredStep = $finishReason !== FinishReason::ToolCalls; $thoughtSummaries = $this->extractThoughtSummaries($data); @@ -255,6 +263,7 @@ protected function addStep(array $data, Request $request, FinishReason $finishRe structured: $isStructuredStep ? $this->extractStructuredData(data_get($data, 'candidates.0.content.parts.0.text') ?? '') : [], toolCalls: $finishReason === FinishReason::ToolCalls ? ToolCallMap::map(data_get($data, 'candidates.0.content.parts', [])) : [], toolResults: $toolResults, + toolApprovalRequests: $toolApprovalRequests, raw: $data, ) ); diff --git a/src/Providers/Gemini/Handlers/Text.php b/src/Providers/Gemini/Handlers/Text.php index fce051b67..4e22c5c63 100644 --- a/src/Providers/Gemini/Handlers/Text.php +++ b/src/Providers/Gemini/Handlers/Text.php @@ -25,6 +25,7 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -43,6 +44,8 @@ public function __construct( public function handle(Request $request): TextResponse { + $this->resolveToolApprovals($request); + $response = $this->sendRequest($request); $this->validateResponse($response); @@ -142,18 +145,27 @@ protected function handleToolCalls(array $data, Request $request): TextResponse { $toolCalls = ToolCallMap::map(data_get($data, 'candidates.0.content.parts', [])); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $this->addStep($data, $request, FinishReason::ToolCalls, $toolResults); + $this->addStep($data, $request, FinishReason::ToolCalls, $toolResults, $approvalRequests); $request->addMessage(new AssistantMessage( $this->extractTextContent($data), $toolCalls, + [], + $approvalRequests, )); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -168,8 +180,9 @@ protected function shouldContinue(Request $request): bool /** * @param array $data * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, FinishReason $finishReason, array $toolResults = []): void + protected function addStep(array $data, Request $request, FinishReason $finishReason, array $toolResults = [], array $toolApprovalRequests = []): void { $providerOptions = $request->providerOptions(); @@ -195,6 +208,7 @@ protected function addStep(array $data, Request $request, FinishReason $finishRe ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: Arr::whereNotNull([ 'citations' => CitationMapper::mapFromGemini(data_get($data, 'candidates.0', [])) ?: null, 'searchEntryPoint' => data_get($data, 'candidates.0.groundingMetadata.searchEntryPoint'), diff --git a/src/Providers/Groq/Handlers/Stream.php b/src/Providers/Groq/Handlers/Stream.php index eaddf4323..47e6e02b8 100644 --- a/src/Providers/Groq/Handlers/Stream.php +++ b/src/Providers/Groq/Handlers/Stream.php @@ -56,6 +56,8 @@ public function __construct(protected PendingRequest $client) */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $response = $this->sendRequest($request); yield from $this->processStream($response, $request); @@ -277,7 +279,15 @@ protected function handleToolCalls( } $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } // Emit step finish after tool calls $this->state->markStepFinished(); diff --git a/src/Providers/Groq/Handlers/Text.php b/src/Providers/Groq/Handlers/Text.php index a88c3007e..c61376453 100644 --- a/src/Providers/Groq/Handlers/Text.php +++ b/src/Providers/Groq/Handlers/Text.php @@ -23,6 +23,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -40,6 +41,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): TextResponse { + $this->resolveToolApprovals($request); + $response = $this->sendRequest($request); $this->validateResponse($response); @@ -81,18 +84,27 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse { $toolCalls = $this->mapToolCalls(data_get($data, 'choices.0.message.tool_calls', []) ?? []); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $this->addStep($data, $request, $clientResponse, FinishReason::ToolCalls, $toolResults); + $this->addStep($data, $request, $clientResponse, FinishReason::ToolCalls, $toolResults, $approvalRequests); $request->addMessage(new AssistantMessage( data_get($data, 'choices.0.message.content') ?? '', $toolCalls, + [], + $approvalRequests, )); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -117,8 +129,9 @@ protected function shouldContinue(Request $request): bool /** * @param array $data * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, ClientResponse $clientResponse, FinishReason $finishReason, array $toolResults = []): void + protected function addStep(array $data, Request $request, ClientResponse $clientResponse, FinishReason $finishReason, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: data_get($data, 'choices.0.message.content') ?? '', @@ -137,6 +150,7 @@ protected function addStep(array $data, Request $request, ClientResponse $client ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: [], raw: $data, )); diff --git a/src/Providers/Mistral/Handlers/Stream.php b/src/Providers/Mistral/Handlers/Stream.php index 00e890c3e..1d054fb27 100644 --- a/src/Providers/Mistral/Handlers/Stream.php +++ b/src/Providers/Mistral/Handlers/Stream.php @@ -59,6 +59,8 @@ public function __construct( */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $response = $this->sendRequest($request); yield from $this->processStream($response, $request); @@ -321,7 +323,15 @@ protected function handleToolCalls( } $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } $this->state->markStepFinished(); yield new StepFinishEvent( diff --git a/src/Providers/Mistral/Handlers/Text.php b/src/Providers/Mistral/Handlers/Text.php index ebb8fc6b9..771c4b0a9 100644 --- a/src/Providers/Mistral/Handlers/Text.php +++ b/src/Providers/Mistral/Handlers/Text.php @@ -25,6 +25,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -47,6 +48,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): Response { + $this->resolveToolApprovals($request); + $response = $this->sendRequest($request); $this->validateResponse($response); @@ -67,18 +70,27 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse { $toolCalls = $this->mapToolCalls(data_get($data, 'choices.0.message.tool_calls', [])); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $this->addStep($data, $request, $clientResponse, $toolResults); + $this->addStep($data, $request, $clientResponse, $toolResults, $approvalRequests); $request->addMessage(new AssistantMessage( $this->extractText(data_get($data, 'choices.0.message', [])), $toolCalls, + [], + $approvalRequests, )); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -107,8 +119,9 @@ protected function shouldContinue(Request $request): bool /** * @param array $data * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, ClientResponse $clientResponse, array $toolResults = []): void + protected function addStep(array $data, Request $request, ClientResponse $clientResponse, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: $this->extractText(data_get($data, 'choices.0.message', [])), @@ -127,6 +140,7 @@ protected function addStep(array $data, Request $request, ClientResponse $client ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: $this->extractThinking(data_get($data, 'choices.0.message', [])), raw: $data, )); diff --git a/src/Providers/Ollama/Handlers/Stream.php b/src/Providers/Ollama/Handlers/Stream.php index b5544c111..6b84d2ffd 100644 --- a/src/Providers/Ollama/Handlers/Stream.php +++ b/src/Providers/Ollama/Handlers/Stream.php @@ -53,6 +53,8 @@ public function __construct(protected PendingRequest $client) */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $response = $this->sendRequest($request); yield from $this->processStream($response, $request); @@ -291,7 +293,15 @@ protected function handleToolCalls( // Execute tools and emit results $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } // Emit step finish after tool calls $this->state->markStepFinished(); diff --git a/src/Providers/Ollama/Handlers/Text.php b/src/Providers/Ollama/Handlers/Text.php index 1c4928fa9..0dbcfef97 100644 --- a/src/Providers/Ollama/Handlers/Text.php +++ b/src/Providers/Ollama/Handlers/Text.php @@ -20,6 +20,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -39,6 +40,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): Response { + $this->resolveToolApprovals($request); + $data = $this->sendRequest($request); $this->validateResponse($data); @@ -97,18 +100,27 @@ protected function handleToolCalls(array $data, Request $request): Response { $toolCalls = $this->mapToolCalls(data_get($data, 'message.tool_calls', [])); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $this->addStep($data, $request, $toolResults); + $this->addStep($data, $request, $toolResults, $approvalRequests); $request->addMessage(new AssistantMessage( data_get($data, 'message.content') ?? '', $toolCalls, + [], + $approvalRequests, )); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -133,13 +145,22 @@ protected function shouldContinue(Request $request): bool /** * @param array $data * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, array $toolResults = []): void + protected function addStep(array $data, Request $request, array $toolResults = [], array $toolApprovalRequests = []): void { + $toolCalls = $this->mapToolCalls(data_get($data, 'message.tool_calls', []) ?? []); + + // Ollama sends done_reason: "stop" even when there are tool calls + // Override finish reason to ToolCalls when tool calls are present + $finishReason = $toolCalls === [] + ? $this->mapFinishReason($data) + : FinishReason::ToolCalls; + $this->responseBuilder->addStep(new Step( text: data_get($data, 'message.content') ?? '', - finishReason: $this->mapFinishReason($data), - toolCalls: $this->mapToolCalls(data_get($data, 'message.tool_calls', []) ?? []), + finishReason: $finishReason, + toolCalls: $toolCalls, toolResults: $toolResults, providerToolCalls: [], usage: new Usage( @@ -152,6 +173,7 @@ protected function addStep(array $data, Request $request, array $toolResults = [ ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: [], raw: $data, )); diff --git a/src/Providers/OpenAI/Handlers/Stream.php b/src/Providers/OpenAI/Handlers/Stream.php index adcbadaf1..d182882a0 100644 --- a/src/Providers/OpenAI/Handlers/Stream.php +++ b/src/Providers/OpenAI/Handlers/Stream.php @@ -61,6 +61,8 @@ public function __construct(protected PendingRequest $client) */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $response = $this->sendRequest($request); yield from $this->processStream($response, $request); @@ -396,7 +398,15 @@ protected function handleToolCalls(Request $request, int $depth): Generator { $mappedToolCalls = $this->mapToolCalls($this->state->toolCalls()); $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } // Emit step finish after tool calls $this->state->markStepFinished(); diff --git a/src/Providers/OpenAI/Handlers/Structured.php b/src/Providers/OpenAI/Handlers/Structured.php index ffa2f0e53..426106e57 100644 --- a/src/Providers/OpenAI/Handlers/Structured.php +++ b/src/Providers/OpenAI/Handlers/Structured.php @@ -31,6 +31,7 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -53,6 +54,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): StructuredResponse { + $this->resolveToolApprovals($request); + $response = match ($request->mode()) { StructuredMode::Auto => $this->handleAutoMode($request), StructuredMode::Structured => $this->handleStructuredMode($request), @@ -100,17 +103,21 @@ public function handle(Request $request): StructuredResponse */ protected function handleToolCalls(array $data, Request $request, ClientResponse $clientResponse): StructuredResponse { + $hasPendingToolCalls = false; + $approvalRequests = []; $toolResults = $this->callTools( $request->tools(), ToolCallMap::map($this->extractFunctionCalls($data)), + $hasPendingToolCalls, + $approvalRequests, ); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - $this->addStep($data, $request, $clientResponse, $toolResults); + $this->addStep($data, $request, $clientResponse, $toolResults, $approvalRequests); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -130,8 +137,9 @@ protected function handleFinalStop(array $data, Request $request, ClientResponse /** * @param array $data * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, ClientResponse $clientResponse, array $toolResults = []): void + protected function addStep(array $data, Request $request, ClientResponse $clientResponse, array $toolResults = [], array $toolApprovalRequests = []): void { $finishReason = $this->mapFinishReason($data); $isStructuredStep = $finishReason !== FinishReason::ToolCalls; @@ -166,6 +174,7 @@ protected function addStep(array $data, Request $request, ClientResponse $client structured: $isStructuredStep ? $this->extractStructuredData(data_get($data, 'output.{last}.content.0.text') ?? '') : [], toolCalls: $toolCalls, toolResults: $toolResults, + toolApprovalRequests: $toolApprovalRequests, raw: $data, )); } diff --git a/src/Providers/OpenAI/Handlers/Text.php b/src/Providers/OpenAI/Handlers/Text.php index e370964dd..38f96ce36 100644 --- a/src/Providers/OpenAI/Handlers/Text.php +++ b/src/Providers/OpenAI/Handlers/Text.php @@ -27,6 +27,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -51,6 +52,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): Response { + $this->resolveToolApprovals($request); + $response = $this->sendRequest($request); $this->validateResponse($response); @@ -86,9 +89,16 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse array_filter(data_get($data, 'output', []), fn (array $output): bool => $output['type'] === 'reasoning'), ); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $this->addStep($data, $request, $clientResponse, $toolResults); + $this->addStep($data, $request, $clientResponse, $toolResults, $approvalRequests); $providerToolCalls = ProviderToolCallMap::map(data_get($data, 'output', [])); @@ -99,11 +109,12 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse 'citations' => $this->citations, 'provider_tool_calls' => $providerToolCalls === [] ? null : $providerToolCalls, ]), + toolApprovalRequests: $approvalRequests, )); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -158,12 +169,14 @@ protected function sendRequest(Request $request): ClientResponse /** * @param array $data * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ protected function addStep( array $data, Request $request, ClientResponse $clientResponse, - array $toolResults = [] + array $toolResults = [], + array $toolApprovalRequests = [], ): void { /** @var array> $output */ $output = data_get($data, 'output', []); @@ -188,6 +201,7 @@ protected function addStep( ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: Arr::whereNotNull([ 'citations' => $this->citations, 'searchQueries' => collect($output) diff --git a/src/Providers/OpenRouter/Handlers/Stream.php b/src/Providers/OpenRouter/Handlers/Stream.php index b9a895388..d9bbf78a8 100644 --- a/src/Providers/OpenRouter/Handlers/Stream.php +++ b/src/Providers/OpenRouter/Handlers/Stream.php @@ -56,6 +56,8 @@ public function __construct(protected PendingRequest $client) */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $response = $this->sendRequest($request); yield from $this->processStream($response, $request); @@ -102,6 +104,13 @@ protected function processStream(Response $response, Request $request, int $dept ); } + // Extract usage from any chunk that has it + // OpenRouter sends usage in a separate final chunk when stream_options.include_usage=true + $usage = $this->extractUsage($data); + if ($usage instanceof Usage) { + $this->state->addUsage($usage); + } + if ($this->hasToolCalls($data)) { $toolCalls = $this->extractToolCalls($data, $toolCalls); @@ -233,13 +242,6 @@ protected function processStream(Response $response, Request $request, int $dept $this->state->withFinishReason($finishReason); } - - // Extract usage from any chunk that has it - // OpenRouter sends usage in a separate final chunk when stream_options.include_usage=true - $usage = $this->extractUsage($data); - if ($usage instanceof Usage) { - $this->state->addUsage($usage); - } } $this->state->markStepFinished(); @@ -389,7 +391,15 @@ protected function handleToolCalls( } $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } $this->state->markStepFinished(); yield new StepFinishEvent( diff --git a/src/Providers/OpenRouter/Handlers/Structured.php b/src/Providers/OpenRouter/Handlers/Structured.php index 43b8376fd..8387e8971 100644 --- a/src/Providers/OpenRouter/Handlers/Structured.php +++ b/src/Providers/OpenRouter/Handlers/Structured.php @@ -22,6 +22,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -41,6 +42,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): StructuredResponse { + $this->resolveToolApprovals($request); + $data = $this->sendRequest($request); $this->validateResponse($data); @@ -99,19 +102,22 @@ protected function handleToolCalls(array $data, Request $request): StructuredRes { $toolCalls = ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools($request->tools(), $toolCalls, $hasPendingToolCalls, $approvalRequests); - $this->addStep($data, $request, $toolResults); + $this->addStep($data, $request, $toolResults, $approvalRequests); $request = $request->addMessage(new AssistantMessage( data_get($data, 'choices.0.message.content') ?? '', $toolCalls, - [] + [], + $approvalRequests, )); $request = $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -146,9 +152,10 @@ protected function shouldContinue(Request $request): bool /** * @param array $data - * @param array $toolResults + * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, array $toolResults = []): void + protected function addStep(array $data, Request $request, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: data_get($data, 'choices.0.message.content') ?? '', @@ -167,6 +174,7 @@ protected function addStep(array $data, Request $request, array $toolResults = [ toolCalls: ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])), providerToolCalls: [], toolResults: $toolResults, + toolApprovalRequests: $toolApprovalRequests, raw: $data, )); } diff --git a/src/Providers/OpenRouter/Handlers/Text.php b/src/Providers/OpenRouter/Handlers/Text.php index ac2b78cf0..8dc030f30 100644 --- a/src/Providers/OpenRouter/Handlers/Text.php +++ b/src/Providers/OpenRouter/Handlers/Text.php @@ -21,6 +21,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -40,6 +41,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): TextResponse { + $this->resolveToolApprovals($request); + $data = $this->sendRequest($request); $this->validateResponse($data); @@ -58,19 +61,27 @@ protected function handleToolCalls(array $data, Request $request): TextResponse { $toolCalls = ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])); - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $this->addStep($data, $request, $toolResults); + $this->addStep($data, $request, $toolResults, $approvalRequests); $request = $request->addMessage(new AssistantMessage( data_get($data, 'choices.0.message.content') ?? '', $toolCalls, - [] + [], + $approvalRequests, )); $request = $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -112,9 +123,10 @@ protected function sendRequest(Request $request): array /** * @param array $data - * @param array $toolResults + * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, array $toolResults = []): void + protected function addStep(array $data, Request $request, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: data_get($data, 'choices.0.message.content') ?? '', @@ -132,6 +144,7 @@ protected function addStep(array $data, Request $request, array $toolResults = [ ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: [], raw: $data, )); diff --git a/src/Providers/XAI/Handlers/Stream.php b/src/Providers/XAI/Handlers/Stream.php index 1edac9f2a..229ed9c3e 100644 --- a/src/Providers/XAI/Handlers/Stream.php +++ b/src/Providers/XAI/Handlers/Stream.php @@ -57,6 +57,8 @@ public function __construct(protected PendingRequest $client) */ public function handle(Request $request): Generator { + yield from $this->resolveToolApprovalsAndYieldEvents($request, EventID::generate(), $this->state); + $response = $this->sendRequest($request); yield from $this->processStream($response, $request); @@ -368,7 +370,15 @@ protected function handleToolCalls( } $toolResults = []; - yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults); + $hasPendingToolCalls = false; + yield from $this->callToolsAndYieldEvents($request->tools(), $mappedToolCalls, $this->state->messageId(), $toolResults, $hasPendingToolCalls); + + if ($hasPendingToolCalls) { + $this->state->markStepFinished(); + yield from $this->yieldToolCallsFinishEvents($this->state); + + return; + } $this->state->markStepFinished(); yield new StepFinishEvent( diff --git a/src/Providers/XAI/Handlers/Text.php b/src/Providers/XAI/Handlers/Text.php index 036b0e22b..e5ecacf56 100644 --- a/src/Providers/XAI/Handlers/Text.php +++ b/src/Providers/XAI/Handlers/Text.php @@ -22,6 +22,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -41,6 +42,8 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): TextResponse { + $this->resolveToolApprovals($request); + $response = $this->sendRequest($request); $this->validateResponse($response); @@ -70,18 +73,22 @@ protected function handleToolCalls(array $data, Request $request): TextResponse throw new PrismException('XAI: finish reason is tool_calls but no tool calls found in response'); } - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools($request->tools(), $toolCalls, $hasPendingToolCalls, $approvalRequests); - $this->addStep($data, $request, $toolResults); + $this->addStep($data, $request, $toolResults, $approvalRequests); $request->addMessage(new AssistantMessage( data_get($data, 'choices.0.message.content') ?? '', $toolCalls, + [], + $approvalRequests, )); $request->addMessage(new ToolResultMessage($toolResults)); $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -147,9 +154,10 @@ protected function mapToolCalls(array $toolCalls): array /** * @param array $data - * @param array $toolResults + * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, array $toolResults = []): void + protected function addStep(array $data, Request $request, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: data_get($data, 'choices.0.message.content') ?? '', @@ -167,6 +175,7 @@ protected function addStep(array $data, Request $request, array $toolResults = [ ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: [], raw: $data, )); diff --git a/src/Providers/Z/Handlers/Structured.php b/src/Providers/Z/Handlers/Structured.php index 37a94eec7..fd23c0756 100644 --- a/src/Providers/Z/Handlers/Structured.php +++ b/src/Providers/Z/Handlers/Structured.php @@ -7,18 +7,29 @@ use Illuminate\Http\Client\PendingRequest; use Illuminate\Http\Client\Response as ClientResponse; use Illuminate\Support\Arr; +use Prism\Prism\Concerns\CallsTools; +use Prism\Prism\Enums\FinishReason; +use Prism\Prism\Exceptions\PrismException; +use Prism\Prism\Exceptions\PrismStructuredDecodingException; use Prism\Prism\Providers\Z\Concerns\MapsFinishReason; use Prism\Prism\Providers\Z\Maps\StructuredMap; +use Prism\Prism\Providers\Z\Maps\ToolCallMap; +use Prism\Prism\Providers\Z\Maps\ToolChoiceMap; +use Prism\Prism\Providers\Z\Maps\ToolMap; use Prism\Prism\Structured\Request; use Prism\Prism\Structured\Response as StructuredResponse; use Prism\Prism\Structured\ResponseBuilder; use Prism\Prism\Structured\Step; use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; class Structured { + use CallsTools; use MapsFinishReason; protected ResponseBuilder $responseBuilder; @@ -30,21 +41,74 @@ public function __construct(protected PendingRequest $client) public function handle(Request $request): StructuredResponse { + $this->resolveToolApprovals($request); + $response = $this->sendRequest($request); $data = $response->json(); - $content = data_get($data, 'choices.0.message.content'); + return match ($this->mapFinishReason($data)) { + FinishReason::ToolCalls => $this->handleToolCalls($data, $request), + FinishReason::Stop, FinishReason::Length => $this->handleStop($data, $request), + default => throw new PrismException('Z: unknown finish reason'), + }; + } - $responseMessage = new AssistantMessage($content); + /** + * @param array $data + */ + protected function handleToolCalls(array $data, Request $request): StructuredResponse + { + $toolCalls = ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])); - $request->addMessage($responseMessage); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools($request->tools(), $toolCalls, $hasPendingToolCalls, $approvalRequests); - $this->addStep($data, $request); + $this->addStep($data, $request, $toolResults, $approvalRequests); + + $request = $request->addMessage(new AssistantMessage( + data_get($data, 'choices.0.message.content') ?? '', + $toolCalls, + [], + $approvalRequests, + )); + $request = $request->addMessage(new ToolResultMessage($toolResults)); + $request->resetToolChoice(); + + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { + return $this->handle($request); + } return $this->responseBuilder->toResponse(); } + /** + * @param array $data + */ + protected function handleStop(array $data, Request $request): StructuredResponse + { + $this->addStep($data, $request); + + try { + return $this->responseBuilder->toResponse(); + } catch (PrismStructuredDecodingException $e) { + $context = sprintf( + "\nModel: %s\nFinish reason: %s\nRaw choices: %s", + data_get($data, 'model', 'unknown'), + data_get($data, 'choices.0.finish_reason', 'unknown'), + json_encode(data_get($data, 'choices'), JSON_PRETTY_PRINT) + ); + + throw new PrismStructuredDecodingException($e->getMessage().$context); + } + } + + protected function shouldContinue(Request $request): bool + { + return $this->responseBuilder->steps->count() < $request->maxSteps(); + } + protected function sendRequest(Request $request): ClientResponse { $structured = new StructuredMap($request->messages(), $request->systemPrompts(), $request->schema()); @@ -61,6 +125,9 @@ protected function sendRequest(Request $request): ClientResponse ], Arr::whereNotNull([ 'temperature' => $request->temperature(), 'top_p' => $request->topP(), + 'max_tokens' => $request->maxTokens(), + 'tools' => ToolMap::map($request->tools()), + 'tool_choice' => ToolChoiceMap::map($request->toolChoice()), ])); /** @var ClientResponse $response */ @@ -71,8 +138,10 @@ protected function sendRequest(Request $request): ClientResponse /** * @param array $data + * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request): void + protected function addStep(array $data, Request $request, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: data_get($data, 'choices.0.message.content') ?? '', @@ -82,13 +151,18 @@ protected function addStep(array $data, Request $request): void completionTokens: data_get($data, 'usage.completion_tokens', 0), ), meta: new Meta( - id: data_get($data, 'id'), - model: data_get($data, 'model'), + id: data_get($data, 'id', ''), + model: data_get($data, 'model', $request->model()), ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), additionalContent: [], structured: [], + toolCalls: ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])), + providerToolCalls: [], + toolResults: $toolResults, + toolApprovalRequests: $toolApprovalRequests, + raw: $data, )); } } diff --git a/src/Providers/Z/Handlers/Text.php b/src/Providers/Z/Handlers/Text.php index dbdc356cf..2497c7a7b 100644 --- a/src/Providers/Z/Handlers/Text.php +++ b/src/Providers/Z/Handlers/Text.php @@ -12,6 +12,7 @@ use Prism\Prism\Exceptions\PrismException; use Prism\Prism\Providers\Z\Concerns\MapsFinishReason; use Prism\Prism\Providers\Z\Maps\MessageMap; +use Prism\Prism\Providers\Z\Maps\ToolCallMap; use Prism\Prism\Providers\Z\Maps\ToolChoiceMap; use Prism\Prism\Providers\Z\Maps\ToolMap; use Prism\Prism\Text\Request; @@ -21,7 +22,7 @@ use Prism\Prism\ValueObjects\Messages\AssistantMessage; use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; -use Prism\Prism\ValueObjects\ToolCall; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -42,20 +43,13 @@ public function __construct(protected PendingRequest $client) */ public function handle(Request $request): TextResponse { + $this->resolveToolApprovals($request); + $response = $this->sendRequest($request); $data = $response->json(); - $responseMessage = new AssistantMessage( - data_get($data, 'choices.0.message.content') ?? '', - $this->mapToolCalls(data_get($data, 'choices.0.message.tool_calls', [])), - ); - - $request->addMessage($responseMessage); - - $finishReason = $this->mapFinishReason($data); - - return match ($finishReason) { + return match ($this->mapFinishReason($data)) { FinishReason::ToolCalls => $this->handleToolCalls($data, $request), FinishReason::Stop, FinishReason::Length => $this->handleStop($data, $request), default => throw new PrismException('Z: unknown finish reason'), @@ -69,19 +63,33 @@ public function handle(Request $request): TextResponse */ protected function handleToolCalls(array $data, Request $request): TextResponse { - $toolCalls = $this->mapToolCalls(data_get($data, 'choices.0.message.tool_calls', [])); + $toolCalls = ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])); if ($toolCalls === []) { throw new PrismException('Z: finish reason is tool_calls but no tool calls found in response'); } - $toolResults = $this->callTools($request->tools(), $toolCalls); + $hasPendingToolCalls = false; + $approvalRequests = []; + $toolResults = $this->callTools( + $request->tools(), + $toolCalls, + $hasPendingToolCalls, + $approvalRequests, + ); - $request->addMessage(new ToolResultMessage($toolResults)); + $this->addStep($data, $request, $toolResults, $approvalRequests); - $this->addStep($data, $request, $toolResults); + $request = $request->addMessage(new AssistantMessage( + data_get($data, 'choices.0.message.content') ?? '', + $toolCalls, + [], + $approvalRequests, + )); + $request = $request->addMessage(new ToolResultMessage($toolResults)); + $request->resetToolChoice(); - if ($this->shouldContinue($request)) { + if (! $hasPendingToolCalls && $this->shouldContinue($request)) { return $this->handle($request); } @@ -122,29 +130,17 @@ protected function sendRequest(Request $request): ClientResponse return $response; } - /** - * @param array> $toolCalls - * @return array - */ - protected function mapToolCalls(array $toolCalls): array - { - return array_map(fn (array $toolCall): ToolCall => new ToolCall( - id: data_get($toolCall, 'id'), - name: data_get($toolCall, 'function.name'), - arguments: data_get($toolCall, 'function.arguments'), - ), $toolCalls); - } - /** * @param array $data - * @param array $toolResults + * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests */ - protected function addStep(array $data, Request $request, array $toolResults = []): void + protected function addStep(array $data, Request $request, array $toolResults = [], array $toolApprovalRequests = []): void { $this->responseBuilder->addStep(new Step( text: data_get($data, 'choices.0.message.content') ?? '', finishReason: $this->mapFinishReason($data), - toolCalls: $this->mapToolCalls(data_get($data, 'choices.0.message.tool_calls', [])), + toolCalls: ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', [])), toolResults: $toolResults, providerToolCalls: [], usage: new Usage( @@ -152,12 +148,14 @@ protected function addStep(array $data, Request $request, array $toolResults = [ data_get($data, 'usage.completion_tokens', 0), ), meta: new Meta( - id: data_get($data, 'id'), - model: data_get($data, 'model'), + id: data_get($data, 'id', ''), + model: data_get($data, 'model', $request->model()), ), messages: $request->messages(), systemPrompts: $request->systemPrompts(), + toolApprovalRequests: $toolApprovalRequests, additionalContent: [], + raw: $data, )); } } diff --git a/src/Providers/Z/Maps/ToolCallMap.php b/src/Providers/Z/Maps/ToolCallMap.php new file mode 100644 index 000000000..44ab00ebb --- /dev/null +++ b/src/Providers/Z/Maps/ToolCallMap.php @@ -0,0 +1,23 @@ +> $toolCalls + * @return array + */ + public static function map(array $toolCalls): array + { + return array_map(fn (array $toolCall): ToolCall => new ToolCall( + id: data_get($toolCall, 'id'), + name: data_get($toolCall, 'function.name'), + arguments: data_get($toolCall, 'function.arguments'), + ), $toolCalls); + } +} diff --git a/src/Streaming/Adapters/BroadcastAdapter.php b/src/Streaming/Adapters/BroadcastAdapter.php index bf8b064cb..45ff03589 100644 --- a/src/Streaming/Adapters/BroadcastAdapter.php +++ b/src/Streaming/Adapters/BroadcastAdapter.php @@ -23,6 +23,7 @@ use Prism\Prism\Events\Broadcasting\ThinkingBroadcast; use Prism\Prism\Events\Broadcasting\ThinkingCompleteBroadcast; use Prism\Prism\Events\Broadcasting\ThinkingStartBroadcast; +use Prism\Prism\Events\Broadcasting\ToolApprovalRequestBroadcast; use Prism\Prism\Events\Broadcasting\ToolCallBroadcast; use Prism\Prism\Events\Broadcasting\ToolCallDeltaBroadcast; use Prism\Prism\Events\Broadcasting\ToolResultBroadcast; @@ -41,6 +42,7 @@ use Prism\Prism\Streaming\Events\ThinkingCompleteEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; use Prism\Prism\Streaming\Events\ThinkingStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallDeltaEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; @@ -85,6 +87,7 @@ protected function broadcastEvent(StreamEvent $event): ShouldBroadcast ThinkingEvent::class => new ThinkingBroadcast($event, $this->channels), ThinkingCompleteEvent::class => new ThinkingCompleteBroadcast($event, $this->channels), ToolCallEvent::class => new ToolCallBroadcast($event, $this->channels), + ToolApprovalRequestEvent::class => new ToolApprovalRequestBroadcast($event, $this->channels), ToolCallDeltaEvent::class => new ToolCallDeltaBroadcast($event, $this->channels), ToolResultEvent::class => new ToolResultBroadcast($event, $this->channels), ArtifactEvent::class => new ArtifactBroadcast($event, $this->channels), diff --git a/src/Streaming/Adapters/DataProtocolAdapter.php b/src/Streaming/Adapters/DataProtocolAdapter.php index 8d164dd15..5dd6d94eb 100644 --- a/src/Streaming/Adapters/DataProtocolAdapter.php +++ b/src/Streaming/Adapters/DataProtocolAdapter.php @@ -21,6 +21,7 @@ use Prism\Prism\Streaming\Events\ThinkingCompleteEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; use Prism\Prism\Streaming\Events\ThinkingStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Prism\Prism\Text\PendingRequest; @@ -32,14 +33,27 @@ class DataProtocolAdapter { /** - * Track tool call IDs that have sent tool-input-available. - * This prevents "tool-output-error must be preceded by a tool-input-available" - * errors when tool outputs arrive without corresponding inputs. + * Track tool call IDs for provider tool events. + * tool-output-available may be sent without a prior tool-input-available (e.g. Dispatched in subsequent request for tools needing approval). * * @var array */ protected array $startedToolCallIds = []; + /** + * @param string|null $responseMessageId When set, the start event echoes this ID instead of the + * provider-generated one. The Vercel AI SDK client sends a + * messageId when it needs the server to continue an existing + * assistant message — for example after a client-executed tool + * result (addToolResult), a tool output (addToolOutput), or any + * automatic resubmission triggered by sendAutomaticallyWhen. + * Without this, the UI would create a new message bubble instead + * of appending to the current one. + */ + public function __construct( + protected ?string $responseMessageId = null, + ) {} + /** * @param callable(PendingRequest, Collection): void|null $callback */ @@ -136,6 +150,7 @@ protected function handleEventConversion(StreamEvent $event): ?string ThinkingEvent::class => $this->handleThinkingDelta($event), ThinkingCompleteEvent::class => $this->handleThinkingComplete($event), ToolCallEvent::class => $this->handleToolCall($event), + ToolApprovalRequestEvent::class => $this->handleToolApprovalRequest($event), ToolResultEvent::class => $this->handleToolResult($event), ArtifactEvent::class => $this->handleArtifact($event), ProviderToolEvent::class => $this->handleProviderTool($event), @@ -165,7 +180,7 @@ protected function handleStreamStart(StreamStartEvent $event): array { return [ 'type' => 'start', - 'messageId' => $event->id, + 'messageId' => $this->responseMessageId ?? $event->id, ]; } @@ -252,6 +267,20 @@ protected function handleToolCall(ToolCallEvent $event): array ]; } + /** + * @return array + */ + protected function handleToolApprovalRequest(ToolApprovalRequestEvent $event): array + { + $this->startedToolCallIds[$event->toolCall->id] = true; + + return [ + 'type' => 'tool-approval-request', + 'approvalId' => $event->approvalId, + 'toolCallId' => $event->toolCall->id, + ]; + } + /** * @return array|null */ @@ -259,10 +288,6 @@ protected function handleToolResult(ToolResultEvent $event): ?array { $toolCallId = $event->toolResult->toolCallId; - if (! isset($this->startedToolCallIds[$toolCallId])) { - return null; - } - return [ 'type' => 'tool-output-available', 'toolCallId' => $toolCallId, diff --git a/src/Streaming/Events/ToolApprovalRequestEvent.php b/src/Streaming/Events/ToolApprovalRequestEvent.php new file mode 100644 index 000000000..8379b451b --- /dev/null +++ b/src/Streaming/Events/ToolApprovalRequestEvent.php @@ -0,0 +1,42 @@ + + */ + public function toArray(): array + { + return [ + 'id' => $this->id, + 'timestamp' => $this->timestamp, + 'approval_id' => $this->approvalId, + 'tool_id' => $this->toolCall->id, + 'tool_name' => $this->toolCall->name, + 'arguments' => $this->toolCall->arguments(), + 'message_id' => $this->messageId, + ]; + } +} diff --git a/src/Streaming/StreamCollector.php b/src/Streaming/StreamCollector.php index 97b93cddf..101c0f330 100644 --- a/src/Streaming/StreamCollector.php +++ b/src/Streaming/StreamCollector.php @@ -14,6 +14,7 @@ use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\TextStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Prism\Prism\Text\PendingRequest; @@ -23,6 +24,7 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderToolCall; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -46,6 +48,8 @@ public function collect(): Generator $accumulatedText = ''; /** @var ToolCall[] $toolCalls */ $toolCalls = []; + /** @var ToolApprovalRequest[] $toolApprovalRequests */ + $toolApprovalRequests = []; /** @var ToolResult[] $toolResults */ $toolResults = []; /** @var ProviderToolCall[] $providerToolCalls */ @@ -60,17 +64,22 @@ public function collect(): Generator yield $event; if ($event instanceof TextStartEvent) { - $this->handleTextStart($accumulatedText, $toolCalls, $toolResults, $providerToolCalls, $messages); + $this->handleTextStart($accumulatedText, $toolCalls, $toolApprovalRequests, $toolResults, $providerToolCalls, $messages); } elseif ($event instanceof TextDeltaEvent) { $accumulatedText .= $event->delta; } elseif ($event instanceof ToolCallEvent) { $toolCalls[] = $event->toolCall; + } elseif ($event instanceof ToolApprovalRequestEvent) { + $toolApprovalRequests[] = new ToolApprovalRequest( + approvalId: $event->approvalId, + toolCallId: $event->toolCall->id, + ); } elseif ($event instanceof ToolResultEvent) { $toolResults[] = $event->toolResult; } elseif ($event instanceof ProviderToolEvent) { // Finalize any accumulated text before tool events to preserve order if ($accumulatedText !== '' && empty($providerToolCalls)) { - $this->finalizeCurrentMessage($accumulatedText, $toolCalls, $toolResults, $providerToolCalls, $messages); + $this->finalizeCurrentMessage($accumulatedText, $toolCalls, $toolApprovalRequests, $toolResults, $providerToolCalls, $messages); } $providerToolCalls[] = new ProviderToolCall( @@ -83,8 +92,10 @@ public function collect(): Generator $finishReason = $event->finishReason; $usage = $event->usage; $additionalContent = $event->additionalContent; - $this->handleStreamEnd($accumulatedText, + $this->handleStreamEnd( + $accumulatedText, $toolCalls, + $toolApprovalRequests, $toolResults, $providerToolCalls, $messages, @@ -98,6 +109,7 @@ public function collect(): Generator /** * @param ToolCall[] $toolCalls + * @param ToolApprovalRequest[] $toolApprovalRequests * @param ToolResult[] $toolResults * @param ProviderToolCall[] $providerToolCalls * @param Message[] $messages @@ -105,15 +117,17 @@ public function collect(): Generator protected function handleTextStart( string &$accumulatedText, array &$toolCalls, + array &$toolApprovalRequests, array &$toolResults, array &$providerToolCalls, array &$messages ): void { - $this->finalizeCurrentMessage($accumulatedText, $toolCalls, $toolResults, $providerToolCalls, $messages); + $this->finalizeCurrentMessage($accumulatedText, $toolCalls, $toolApprovalRequests, $toolResults, $providerToolCalls, $messages); } /** * @param ToolCall[] $toolCalls + * @param ToolApprovalRequest[] $toolApprovalRequests * @param ToolResult[] $toolResults * @param ProviderToolCall[] $providerToolCalls * @param Message[] $messages @@ -122,6 +136,7 @@ protected function handleTextStart( protected function handleStreamEnd( string &$accumulatedText, array &$toolCalls, + array &$toolApprovalRequests, array &$toolResults, array &$providerToolCalls, array &$messages, @@ -129,7 +144,7 @@ protected function handleStreamEnd( ?Usage $usage, array $additionalContent ): void { - $this->finalizeCurrentMessage($accumulatedText, $toolCalls, $toolResults, $providerToolCalls, $messages); + $this->finalizeCurrentMessage($accumulatedText, $toolCalls, $toolApprovalRequests, $toolResults, $providerToolCalls, $messages); if ($this->onCompleteCallback instanceof Closure) { $messagesCollection = new Collection($messages); @@ -178,6 +193,10 @@ protected function handleStreamEnd( usage: $usage ?? new Usage(0, 0), meta: new Meta(id: '', model: '', rateLimits: []), messages: $messagesCollection, + toolApprovalRequests: $messagesCollection + ->filter(fn (Message $msg): bool => $msg instanceof AssistantMessage) + ->flatMap(fn (Message $msg): array => $msg instanceof AssistantMessage ? $msg->toolApprovalRequests : []) + ->all(), additionalContent: $additionalContent ); @@ -187,6 +206,7 @@ protected function handleStreamEnd( /** * @param ToolCall[] $toolCalls + * @param ToolApprovalRequest[] $toolApprovalRequests * @param ToolResult[] $toolResults * @param ProviderToolCall[] $providerToolCalls * @param Message[] $messages @@ -194,14 +214,16 @@ protected function handleStreamEnd( protected function finalizeCurrentMessage( string &$accumulatedText, array &$toolCalls, + array &$toolApprovalRequests, array &$toolResults, array &$providerToolCalls, array &$messages ): void { if ($accumulatedText !== '' || $toolCalls !== []) { - $messages[] = new AssistantMessage($accumulatedText, $toolCalls); + $messages[] = new AssistantMessage($accumulatedText, $toolCalls, [], $toolApprovalRequests); $accumulatedText = ''; $toolCalls = []; + $toolApprovalRequests = []; } if ($toolResults !== []) { diff --git a/src/Structured/PendingRequest.php b/src/Structured/PendingRequest.php index 063a85def..3c0649df4 100644 --- a/src/Structured/PendingRequest.php +++ b/src/Structured/PendingRequest.php @@ -71,6 +71,8 @@ public function toRequest(): Request throw new PrismException('A schema is required for structured output'); } + collect($this->tools)->each->ensureRunnable(); + return new Request( systemPrompts: $this->systemPrompts, model: $this->model, diff --git a/src/Structured/Request.php b/src/Structured/Request.php index 7f9675392..0dc4a408e 100644 --- a/src/Structured/Request.php +++ b/src/Structured/Request.php @@ -131,6 +131,16 @@ public function addMessage(Message $message): self return $this; } + /** + * @param array $messages + */ + public function setMessages(array $messages): self + { + $this->messages = $messages; + + return $this; + } + /** * @return array */ diff --git a/src/Structured/Response.php b/src/Structured/Response.php index b3c0c1035..a548b0325 100644 --- a/src/Structured/Response.php +++ b/src/Structured/Response.php @@ -9,6 +9,7 @@ use Prism\Prism\Enums\FinishReason; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderToolCall; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -24,6 +25,7 @@ * @param array $toolCalls * @param array $providerToolCalls * @param array $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests * @param array $additionalContent * @param array|null $raw */ @@ -37,6 +39,7 @@ public function __construct( public array $toolCalls = [], public array $providerToolCalls = [], public array $toolResults = [], + public array $toolApprovalRequests = [], public array $additionalContent = [], public ?array $raw = null ) {} @@ -57,6 +60,7 @@ public function toArray(): array 'tool_calls' => array_map(fn (ToolCall $toolCall): array => $toolCall->toArray(), $this->toolCalls), 'provider_tool_calls' => array_map(fn (ProviderToolCall $providerToolCall): array => $providerToolCall->toArray(), $this->providerToolCalls), 'tool_results' => array_map(fn (ToolResult $toolResult): array => $toolResult->toArray(), $this->toolResults), + 'tool_approval_requests' => array_map(fn (ToolApprovalRequest $req): array => $req->toArray(), $this->toolApprovalRequests), 'additional_content' => $this->additionalContent, 'raw' => $this->raw, ]; diff --git a/src/Structured/ResponseBuilder.php b/src/Structured/ResponseBuilder.php index 1fd3bfe49..a9f5628ab 100644 --- a/src/Structured/ResponseBuilder.php +++ b/src/Structured/ResponseBuilder.php @@ -44,6 +44,7 @@ public function toResponse(): Response toolCalls: $this->aggregateToolCalls(), providerToolCalls: $this->aggregateProviderToolCalls(), toolResults: $this->aggregateToolResults(), + toolApprovalRequests: $finalStep->toolApprovalRequests, additionalContent: $finalStep->additionalContent, raw: $finalStep->raw, ); diff --git a/src/Structured/Step.php b/src/Structured/Step.php index 7d1a4fb32..aad32a27b 100644 --- a/src/Structured/Step.php +++ b/src/Structured/Step.php @@ -13,6 +13,7 @@ use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderToolCall; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -30,6 +31,7 @@ * @param array $toolCalls * @param array $providerToolCalls * @param array $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests * @param array|null $raw */ public function __construct( @@ -44,6 +46,7 @@ public function __construct( public array $toolCalls = [], public array $providerToolCalls = [], public array $toolResults = [], + public array $toolApprovalRequests = [], public ?array $raw = null ) {} @@ -65,6 +68,7 @@ public function toArray(): array 'tool_calls' => array_map(fn (ToolCall $toolCall): array => $toolCall->toArray(), $this->toolCalls), 'tool_results' => array_map(fn (ToolResult $toolResult): array => $toolResult->toArray(), $this->toolResults), 'provider_tool_calls' => array_map(fn (ProviderToolCall $providerToolCall): array => $providerToolCall->toArray(), $this->providerToolCalls), + 'tool_approval_requests' => array_map(fn (ToolApprovalRequest $req): array => $req->toArray(), $this->toolApprovalRequests), 'raw' => $this->raw, ]; } diff --git a/src/Text/PendingRequest.php b/src/Text/PendingRequest.php index c9807ee32..6c38f9e3b 100644 --- a/src/Text/PendingRequest.php +++ b/src/Text/PendingRequest.php @@ -87,9 +87,9 @@ public function asStream(): Generator /** * @param callable(PendingRequest, Collection): void|null $callback */ - public function asDataStreamResponse(?callable $callback = null): StreamedResponse + public function asDataStreamResponse(?callable $callback = null, ?string $responseMessageId = null): StreamedResponse { - return (new DataProtocolAdapter)($this->asStream(), $this, $callback); + return (new DataProtocolAdapter($responseMessageId))($this->asStream(), $this, $callback); } /** @@ -123,6 +123,8 @@ public function toRequest(): Request $tools = $this->tools; + collect($tools)->each->ensureRunnable(); + if (! $this->toolErrorHandlingEnabled && filled($tools)) { $tools = array_map( callback: fn (Tool $tool): Tool => is_null($tool->failedHandler()) ? $tool : $tool->withoutErrorHandling(), diff --git a/src/Text/Request.php b/src/Text/Request.php index fc3355513..a8297bb86 100644 --- a/src/Text/Request.php +++ b/src/Text/Request.php @@ -143,6 +143,16 @@ public function addMessage(Message $message): self return $this; } + /** + * @param array $messages + */ + public function setMessages(array $messages): self + { + $this->messages = $messages; + + return $this; + } + public function resetToolChoice(): self { if (is_string($this->toolChoice) || $this->toolChoice === ToolChoice::Any) { diff --git a/src/Text/Response.php b/src/Text/Response.php index 30ce7fff9..d7f44c452 100644 --- a/src/Text/Response.php +++ b/src/Text/Response.php @@ -13,6 +13,7 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\Meta; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -26,6 +27,7 @@ * @param Collection $steps * @param ToolCall[] $toolCalls * @param ToolResult[] $toolResults + * @param ToolApprovalRequest[] $toolApprovalRequests * @param Collection $messages * @param array $additionalContent * @param array|null $raw @@ -39,6 +41,7 @@ public function __construct( public Usage $usage, public Meta $meta, public Collection $messages, + public array $toolApprovalRequests = [], public array $additionalContent = [], public ?array $raw = null ) {} @@ -55,6 +58,7 @@ public function toArray(): array 'finish_reason' => $this->finishReason->value, 'tool_calls' => array_map(fn (ToolCall $toolCall): array => $toolCall->toArray(), $this->toolCalls), 'tool_results' => array_map(fn (ToolResult $toolResult): array => $toolResult->toArray(), $this->toolResults), + 'tool_approval_requests' => array_map(fn (ToolApprovalRequest $req): array => $req->toArray(), $this->toolApprovalRequests), 'usage' => $this->usage->toArray(), 'meta' => $this->meta->toArray(), 'messages' => $this->messages->map(fn (Message $message): array => $this->messageToArray($message))->toArray(), diff --git a/src/Text/ResponseBuilder.php b/src/Text/ResponseBuilder.php index 22462476b..2116c9b77 100644 --- a/src/Text/ResponseBuilder.php +++ b/src/Text/ResponseBuilder.php @@ -43,6 +43,7 @@ public function toResponse(): Response content: $finalStep->text, toolCalls: $finalStep->toolCalls, additionalContent: $additionalContent, + toolApprovalRequests: $finalStep->toolApprovalRequests, )); return new Response( @@ -54,6 +55,7 @@ public function toResponse(): Response usage: $this->calculateTotalUsage(), meta: $finalStep->meta, messages: $messages, + toolApprovalRequests: $finalStep->toolApprovalRequests, additionalContent: $finalStep->additionalContent, raw: $finalStep->raw, ); diff --git a/src/Text/Step.php b/src/Text/Step.php index c291534f3..8fe15c597 100644 --- a/src/Text/Step.php +++ b/src/Text/Step.php @@ -13,6 +13,7 @@ use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\Meta; use Prism\Prism\ValueObjects\ProviderToolCall; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\ToolResult; use Prism\Prism\ValueObjects\Usage; @@ -26,6 +27,7 @@ * @param ToolCall[] $toolCalls * @param ToolResult[] $toolResults * @param ProviderToolCall[] $providerToolCalls + * @param ToolApprovalRequest[] $toolApprovalRequests * @param Message[] $messages * @param SystemMessage[] $systemPrompts * @param array $additionalContent @@ -41,6 +43,7 @@ public function __construct( public Meta $meta, public array $messages, public array $systemPrompts, + public array $toolApprovalRequests = [], public array $additionalContent = [], public ?array $raw = null ) {} @@ -56,6 +59,7 @@ public function toArray(): array 'finish_reason' => $this->finishReason->value, 'tool_calls' => array_map(fn (ToolCall $toolCall): array => $toolCall->toArray(), $this->toolCalls), 'tool_results' => array_map(fn (ToolResult $toolResult): array => $toolResult->toArray(), $this->toolResults), + 'tool_approval_requests' => array_map(fn (ToolApprovalRequest $req): array => $req->toArray(), $this->toolApprovalRequests), 'provider_tool_calls' => array_map(fn (ProviderToolCall $providerToolCall): array => $providerToolCall->toArray(), $this->providerToolCalls), 'usage' => $this->usage->toArray(), 'meta' => $this->meta->toArray(), diff --git a/src/Tool.php b/src/Tool.php index 99f3f3371..676ab211b 100644 --- a/src/Tool.php +++ b/src/Tool.php @@ -39,14 +39,19 @@ class Tool /** @var array */ protected array $requiredParameters = []; - /** @var Closure():mixed|callable():mixed */ + /** @var Closure():mixed|callable():mixed|null */ protected $fn; /** @var null|false|Closure(Throwable,array):string */ protected null|false|Closure $failedHandler = null; + protected bool $clientExecuted = false; + protected bool $concurrent = false; + /** @var bool|Closure(array):bool */ + protected bool|Closure $requiresApproval = false; + public function __construct() { // @@ -69,6 +74,21 @@ public function for(string $description): self public function using(Closure|callable $fn): self { $this->fn = $fn; + $this->clientExecuted = false; + + return $this; + } + + /** + * Mark this tool as client-executed (no server-side handler). + * + * Client-executed tools are sent to the AI model but their execution + * is handled by the client application rather than the server. + */ + public function clientExecuted(): self + { + $this->clientExecuted = true; + $this->fn = null; return $this; } @@ -126,6 +146,42 @@ public function isConcurrent(): bool return $this->concurrent; } + /** + * Mark this tool as requiring user approval before execution. + * + * When a closure is provided, it receives the tool call arguments + * and should return true if approval is required. + * + * @param bool|Closure(array):bool $condition + */ + public function requiresApproval(bool|Closure $condition = true): self + { + $this->requiresApproval = $condition; + + return $this; + } + + /** + * Whether this tool has approval configured (static true or dynamic closure). + * Use this for early-exit checks without invoking the closure. + */ + public function hasApprovalConfigured(): bool + { + return $this->requiresApproval === true || $this->requiresApproval instanceof Closure; + } + + /** + * @param array $arguments + */ + public function needsApproval(array $arguments = []): bool + { + if ($this->requiresApproval instanceof Closure) { + return (bool) ($this->requiresApproval)($arguments); + } + + return $this->requiresApproval; + } + public function withParameter(Schema $parameter, bool $required = true): self { $this->parameters[$parameter->name()] = $parameter; @@ -246,6 +302,23 @@ public function hasParameters(): bool return (bool) count($this->parameters); } + public function isClientExecuted(): bool + { + return $this->clientExecuted; + } + + public function hasHandler(): bool + { + return $this->fn !== null; + } + + public function ensureRunnable(): void + { + if (! $this->hasHandler() && ! $this->clientExecuted) { + throw PrismException::toolMissingHandler($this->name); + } + } + /** * @return null|false|Closure(Throwable,array):string */ @@ -261,6 +334,10 @@ public function failedHandler(): null|false|Closure */ public function handle(...$args): string|ToolOutput|ToolError { + if ($this->fn === null) { + throw PrismException::toolHandlerNotDefined($this->name); + } + try { $value = call_user_func($this->fn, ...$args); diff --git a/src/ValueObjects/Messages/AssistantMessage.php b/src/ValueObjects/Messages/AssistantMessage.php index 9217f1704..bf4da9d06 100644 --- a/src/ValueObjects/Messages/AssistantMessage.php +++ b/src/ValueObjects/Messages/AssistantMessage.php @@ -7,6 +7,7 @@ use Illuminate\Contracts\Support\Arrayable; use Prism\Prism\Concerns\HasProviderOptions; use Prism\Prism\Contracts\Message; +use Prism\Prism\ValueObjects\ToolApprovalRequest; use Prism\Prism\ValueObjects\ToolCall; /** @@ -19,11 +20,13 @@ class AssistantMessage implements Arrayable, Message /** * @param ToolCall[] $toolCalls * @param array $additionalContent + * @param ToolApprovalRequest[] $toolApprovalRequests Approval requests for approval-required tools (not sent to LLM, for tracking) */ public function __construct( public readonly string $content, public readonly array $toolCalls = [], - public readonly array $additionalContent = [] + public readonly array $additionalContent = [], + public readonly array $toolApprovalRequests = [] ) {} /** @@ -37,6 +40,7 @@ public function toArray(): array 'content' => $this->content, 'tool_calls' => array_map(fn (ToolCall $toolCall): array => $toolCall->toArray(), $this->toolCalls), 'additional_content' => $this->additionalContent, + 'tool_approval_requests' => array_map(fn (ToolApprovalRequest $req): array => $req->toArray(), $this->toolApprovalRequests), ]; } } diff --git a/src/ValueObjects/Messages/ToolResultMessage.php b/src/ValueObjects/Messages/ToolResultMessage.php index 51a19f00e..fb98c5163 100644 --- a/src/ValueObjects/Messages/ToolResultMessage.php +++ b/src/ValueObjects/Messages/ToolResultMessage.php @@ -7,6 +7,7 @@ use Illuminate\Contracts\Support\Arrayable; use Prism\Prism\Concerns\HasProviderOptions; use Prism\Prism\Contracts\Message; +use Prism\Prism\ValueObjects\ToolApprovalResponse; use Prism\Prism\ValueObjects\ToolResult; /** @@ -18,11 +19,24 @@ class ToolResultMessage implements Arrayable, Message /** * @param ToolResult[] $toolResults + * @param ToolApprovalResponse[] $toolApprovalResponses Approval responses (from client) or consumed approvals (for tracking) */ public function __construct( - public readonly array $toolResults + public readonly array $toolResults = [], + public readonly array $toolApprovalResponses = [] ) {} + public function findByApprovalId(string $approvalId): ?ToolApprovalResponse + { + foreach ($this->toolApprovalResponses as $response) { + if ($response->approvalId === $approvalId) { + return $response; + } + } + + return null; + } + /** * @return array */ @@ -32,6 +46,7 @@ public function toArray(): array return [ 'type' => 'tool_result', 'tool_results' => array_map(fn (ToolResult $toolResult): array => $toolResult->toArray(), $this->toolResults), + 'tool_approval_responses' => array_map(fn (ToolApprovalResponse $response): array => $response->toArray(), $this->toolApprovalResponses), ]; } } diff --git a/src/ValueObjects/ToolApprovalRequest.php b/src/ValueObjects/ToolApprovalRequest.php new file mode 100644 index 000000000..91963aab2 --- /dev/null +++ b/src/ValueObjects/ToolApprovalRequest.php @@ -0,0 +1,33 @@ + + */ +readonly class ToolApprovalRequest implements Arrayable +{ + public function __construct( + public string $approvalId, + public string $toolCallId, + ) {} + + /** + * @return array + */ + #[\Override] + public function toArray(): array + { + return [ + 'approval_id' => $this->approvalId, + 'tool_call_id' => $this->toolCallId, + ]; + } +} diff --git a/src/ValueObjects/ToolApprovalResponse.php b/src/ValueObjects/ToolApprovalResponse.php new file mode 100644 index 000000000..d3756b4cf --- /dev/null +++ b/src/ValueObjects/ToolApprovalResponse.php @@ -0,0 +1,32 @@ + + */ +readonly class ToolApprovalResponse implements Arrayable +{ + public function __construct( + public string $approvalId, + public bool $approved, + public ?string $reason = null, + ) {} + + /** + * @return array + */ + #[\Override] + public function toArray(): array + { + return [ + 'approval_id' => $this->approvalId, + 'approved' => $this->approved, + 'reason' => $this->reason, + ]; + } +} diff --git a/tests/Concerns/CallsToolsConcurrentTest.php b/tests/Concerns/CallsToolsConcurrentTest.php index 3b3664f16..8500ca5e6 100644 --- a/tests/Concerns/CallsToolsConcurrentTest.php +++ b/tests/Concerns/CallsToolsConcurrentTest.php @@ -49,11 +49,13 @@ class TestToolCaller ]; $toolResults = []; + $hasPendingToolCalls = false; $events = iterator_to_array($this->caller->callToolsAndYieldEvents( [$tool1, $tool2], $toolCalls, 'msg123', - $toolResults + $toolResults, + $hasPendingToolCalls )); expect($toolResults)->toHaveCount(2); @@ -99,11 +101,13 @@ class TestToolCaller ]; $toolResults = []; + $hasPendingToolCalls = false; $events = iterator_to_array($this->caller->callToolsAndYieldEvents( [$tool1, $tool2, $tool3], $toolCalls, 'msg123', - $toolResults + $toolResults, + $hasPendingToolCalls )); // Verify results are in original order despite parallel execution @@ -161,11 +165,13 @@ class TestToolCaller ]; $toolResults = []; + $hasPendingToolCalls = false; $events = iterator_to_array($this->caller->callToolsAndYieldEvents( [$concurrentTool1, $sequentialTool, $concurrentTool2], $toolCalls, 'msg123', - $toolResults + $toolResults, + $hasPendingToolCalls )); // Verify all tools executed @@ -211,11 +217,13 @@ class TestToolCaller ]; $toolResults = []; + $hasPendingToolCalls = false; $events = iterator_to_array($this->caller->callToolsAndYieldEvents( [$tool1, $tool3], $toolCalls, 'msg123', - $toolResults + $toolResults, + $hasPendingToolCalls )); // All tool calls should have results (including error) @@ -260,10 +268,11 @@ class TestToolCaller new ToolCall('call1', 'concurrent', ['input' => 'test1']), new ToolCall('call2', 'sequential', ['input' => 'test2']), ]; - + $hasPendingToolCalls = false; $grouped = $this->caller->groupToolCallsByConcurrency( [$concurrentTool, $sequentialTool], - $toolCalls + $toolCalls, + $hasPendingToolCalls ); expect($grouped)->toHaveKeys(['concurrent', 'sequential']); diff --git a/tests/Concerns/CallsToolsTest.php b/tests/Concerns/CallsToolsTest.php index 4e9825902..c008b20e8 100644 --- a/tests/Concerns/CallsToolsTest.php +++ b/tests/Concerns/CallsToolsTest.php @@ -15,14 +15,14 @@ class CallsToolsTestHandler { use CallsTools; - public function execute(array $tools, array $toolCalls): array + public function execute(array $tools, array $toolCalls, bool &$hasPendingToolCalls = false): array { - return $this->callTools($tools, $toolCalls); + return $this->callTools($tools, $toolCalls, $hasPendingToolCalls); } - public function stream(array $tools, array $toolCalls, string $messageId, array &$toolResults): Generator + public function stream(array $tools, array $toolCalls, string $messageId, array &$toolResults, bool &$hasPendingToolCalls = false): Generator { - return $this->callToolsAndYieldEvents($tools, $toolCalls, $messageId, $toolResults); + return $this->callToolsAndYieldEvents($tools, $toolCalls, $messageId, $toolResults, $hasPendingToolCalls); } } @@ -319,3 +319,80 @@ public function stream(array $tools, array $toolCalls, string $messageId, array expect($events)->toBeEmpty() ->and($toolResults)->toBeEmpty(); }); + +it('executes server tools and skips client-executed tools in mixed scenario', function (): void { + // Server-executed tool (has handler) + $serverTool = (new Tool) + ->as('server_tool') + ->for('A server-executed tool') + ->withStringParameter('input', 'Input parameter') + ->using(fn (string $input): string => "Server processed: {$input}"); + + // Client-executed tool (no handler) + $clientTool = (new Tool) + ->as('client_tool') + ->for('A client-executed tool') + ->withStringParameter('action', 'Action to perform') + ->clientExecuted(); + + $toolCalls = [ + new ToolCall(id: 'call-server', name: 'server_tool', arguments: ['input' => 'test data']), + new ToolCall(id: 'call-client', name: 'client_tool', arguments: ['action' => 'click button']), + ]; + + $handler = new CallsToolsTestHandler; + $hasPendingToolCalls = false; + $results = $handler->execute([$serverTool, $clientTool], $toolCalls, $hasPendingToolCalls); + + // Server tool should have executed + expect($results)->toHaveCount(1) + ->and($results[0]->toolName)->toBe('server_tool') + ->and($results[0]->result)->toBe('Server processed: test data'); + + // Flag should indicate client-executed tools are pending + expect($hasPendingToolCalls)->toBeTrue(); +}); + +it('executes server tools and skips client-executed tools in mixed streaming scenario', function (): void { + // Server-executed tool (has handler) + $serverTool = (new Tool) + ->as('server_tool') + ->for('A server-executed tool') + ->withStringParameter('input', 'Input parameter') + ->using(fn (string $input): string => "Server processed: {$input}"); + + // Client-executed tool (no handler) + $clientTool = (new Tool) + ->as('client_tool') + ->for('A client-executed tool') + ->withStringParameter('action', 'Action to perform') + ->clientExecuted(); + + $toolCalls = [ + new ToolCall(id: 'call-client-1', name: 'client_tool', arguments: ['action' => 'scroll']), + new ToolCall(id: 'call-server', name: 'server_tool', arguments: ['input' => 'test data']), + new ToolCall(id: 'call-client-2', name: 'client_tool', arguments: ['action' => 'click']), + ]; + + $handler = new CallsToolsTestHandler; + $toolResults = []; + $hasPendingToolCalls = false; + $events = []; + + foreach ($handler->stream([$serverTool, $clientTool], $toolCalls, 'msg-123', $toolResults, $hasPendingToolCalls) as $event) { + $events[] = $event; + } + + // Only server tool should have result event + expect($events)->toHaveCount(1) + ->and($events[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[0]->toolResult->toolName)->toBe('server_tool') + ->and($events[0]->toolResult->result)->toBe('Server processed: test data'); + + // Only server tool results should be collected + expect($toolResults)->toHaveCount(1) + ->and($toolResults[0]->toolName)->toBe('server_tool'); + + // Flag should indicate client-executed tools are pending + expect($hasPendingToolCalls)->toBeTrue(); +}); diff --git a/tests/Fixtures/anthropic/stream-with-approval-tool-1.sse b/tests/Fixtures/anthropic/stream-with-approval-tool-1.sse new file mode 100644 index 000000000..dbf064d2d --- /dev/null +++ b/tests/Fixtures/anthropic/stream-with-approval-tool-1.sse @@ -0,0 +1,30 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_approval_tool_test","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":100,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I'll delete the file for you."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_delete_file_stream","name":"delete_file","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"path\": \"/tmp/test.txt\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":50}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/tests/Fixtures/anthropic/stream-with-client-executed-tool-1.sse b/tests/Fixtures/anthropic/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..e6fcba676 --- /dev/null +++ b/tests/Fixtures/anthropic/stream-with-client-executed-tool-1.sse @@ -0,0 +1,31 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_client_executed_test","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":100,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I'll use the client tool to help you."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_client_tool_stream","name":"client_tool","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"input\": \"test input\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":50}} + +event: message_stop +data: {"type":"message_stop"} + + diff --git a/tests/Fixtures/anthropic/structured-with-approval-phase2-1.json b/tests/Fixtures/anthropic/structured-with-approval-phase2-1.json new file mode 100644 index 000000000..d068e59ee --- /dev/null +++ b/tests/Fixtures/anthropic/structured-with-approval-phase2-1.json @@ -0,0 +1 @@ +{"model":"claude-sonnet-4-20250514","id":"msg_01StructuredApprovalPhase2","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01Phase2Result","name":"output_structured_data","input":{"result":"The file /tmp/test.txt has been deleted successfully."}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":180,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":20}} diff --git a/tests/Fixtures/anthropic/structured-with-approval-tool-1.json b/tests/Fixtures/anthropic/structured-with-approval-tool-1.json new file mode 100644 index 000000000..1fdc5de2c --- /dev/null +++ b/tests/Fixtures/anthropic/structured-with-approval-tool-1.json @@ -0,0 +1 @@ +{"model":"claude-sonnet-4-20250514","id":"msg_approval_structured","type":"message","role":"assistant","content":[{"type":"text","text":"I'll delete the file for you."},{"type":"tool_use","id":"toolu_delete_structured","name":"delete_file","input":{"path":"/tmp/test.txt"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":200,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":50}} diff --git a/tests/Fixtures/anthropic/structured-with-client-executed-tool-1.json b/tests/Fixtures/anthropic/structured-with-client-executed-tool-1.json new file mode 100644 index 000000000..980813079 --- /dev/null +++ b/tests/Fixtures/anthropic/structured-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"model":"claude-sonnet-4-20250514","id":"msg_client_executed_structured","type":"message","role":"assistant","content":[{"type":"text","text":"I'll use the client tool to help you with that request."},{"type":"tool_use","id":"toolu_client_structured","name":"client_tool","input":{"input":"test input"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":200,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":50}} + diff --git a/tests/Fixtures/anthropic/text-with-approval-phase2-1.json b/tests/Fixtures/anthropic/text-with-approval-phase2-1.json new file mode 100644 index 000000000..842ab7d40 --- /dev/null +++ b/tests/Fixtures/anthropic/text-with-approval-phase2-1.json @@ -0,0 +1 @@ +{"id":"msg_01ApprovalPhase2","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[{"type":"text","text":"The file has been deleted successfully."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":150,"output_tokens":10}} diff --git a/tests/Fixtures/anthropic/text-with-approval-tool-1.json b/tests/Fixtures/anthropic/text-with-approval-tool-1.json new file mode 100644 index 000000000..3752697c5 --- /dev/null +++ b/tests/Fixtures/anthropic/text-with-approval-tool-1.json @@ -0,0 +1 @@ +{"id":"msg_01ApprovalToolTest","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[{"type":"text","text":"I'll delete the file for you."},{"type":"tool_use","id":"toolu_delete_file_123","name":"delete_file","input":{"path":"/tmp/test.txt"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":50}} diff --git a/tests/Fixtures/anthropic/text-with-client-executed-tool-1.json b/tests/Fixtures/anthropic/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..9d409fb32 --- /dev/null +++ b/tests/Fixtures/anthropic/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"msg_01ClientExecutedTest","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[{"type":"text","text":"I'll use the client tool to help you with that."},{"type":"tool_use","id":"toolu_client_tool_123","name":"client_tool","input":{"input":"test input"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":50}} + diff --git a/tests/Fixtures/anthropic/text-with-mixed-tools-1.json b/tests/Fixtures/anthropic/text-with-mixed-tools-1.json new file mode 100644 index 000000000..ae9b147d5 --- /dev/null +++ b/tests/Fixtures/anthropic/text-with-mixed-tools-1.json @@ -0,0 +1 @@ +{"id":"msg_01MixedToolsTest","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[{"type":"text","text":"I'll use both tools to help you."},{"type":"tool_use","id":"toolu_server_tool_123","name":"server_tool","input":{"query":"weather in SF"}},{"type":"tool_use","id":"toolu_client_tool_456","name":"client_tool","input":{"action":"click submit"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":75}} diff --git a/tests/Fixtures/deepseek/stream-with-approval-tool-1.sse b/tests/Fixtures/deepseek/stream-with-approval-tool-1.sse new file mode 100644 index 000000000..9238ad5bf --- /dev/null +++ b/tests/Fixtures/deepseek/stream-with-approval-tool-1.sse @@ -0,0 +1,7 @@ +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"call_delete_file_stream","type":"function","function":{"name":"delete_file","arguments":""}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\": \"/tmp/test.txt\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] diff --git a/tests/Fixtures/deepseek/stream-with-client-executed-tool-1.sse b/tests/Fixtures/deepseek/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..5dc224e2c --- /dev/null +++ b/tests/Fixtures/deepseek/stream-with-client-executed-tool-1.sse @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"call_client_tool_stream","type":"function","function":{"name":"client_tool","arguments":""}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\": \"test input\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] + + diff --git a/tests/Fixtures/deepseek/text-with-approval-tool-1.json b/tests/Fixtures/deepseek/text-with-approval-tool-1.json new file mode 100644 index 000000000..e8ff557f3 --- /dev/null +++ b/tests/Fixtures/deepseek/text-with-approval-tool-1.json @@ -0,0 +1 @@ +{"id":"approval-tool-test","object":"chat.completion","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"call_delete_file_123","type":"function","function":{"name":"delete_file","arguments":"{\"path\":\"/tmp/test.txt\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150},"system_fingerprint":"fp_test"} diff --git a/tests/Fixtures/deepseek/text-with-client-executed-tool-1.json b/tests/Fixtures/deepseek/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..cc22dab36 --- /dev/null +++ b/tests/Fixtures/deepseek/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"client-executed-test","object":"chat.completion","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"call_client_tool_123","type":"function","function":{"name":"client_tool","arguments":"{\"input\":\"test input\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150},"system_fingerprint":"fp_test"} + diff --git a/tests/Fixtures/gemini/stream-with-approval-tool-1.json b/tests/Fixtures/gemini/stream-with-approval-tool-1.json new file mode 100644 index 000000000..3b83fcf51 --- /dev/null +++ b/tests/Fixtures/gemini/stream-with-approval-tool-1.json @@ -0,0 +1 @@ +data: {"candidates": [{"content": {"parts": [{"functionCall": {"name": "delete_file","args": {"path": "/tmp/test.txt"}}}],"role": "model"},"finishReason": "STOP","index": 0}],"usageMetadata": {"promptTokenCount": 100,"candidatesTokenCount": 50,"totalTokenCount": 150,"promptTokensDetails": [{"modality": "TEXT","tokenCount": 100}]},"modelVersion": "gemini-1.5-flash"} diff --git a/tests/Fixtures/gemini/stream-with-client-executed-tool-1.json b/tests/Fixtures/gemini/stream-with-client-executed-tool-1.json new file mode 100644 index 000000000..972a0996c --- /dev/null +++ b/tests/Fixtures/gemini/stream-with-client-executed-tool-1.json @@ -0,0 +1,3 @@ +data: {"candidates": [{"content": {"parts": [{"functionCall": {"name": "client_tool","args": {"input": "test input"}}}],"role": "model"},"finishReason": "STOP","index": 0}],"usageMetadata": {"promptTokenCount": 100,"candidatesTokenCount": 50,"totalTokenCount": 150,"promptTokensDetails": [{"modality": "TEXT","tokenCount": 100}]},"modelVersion": "gemini-1.5-flash"} + + diff --git a/tests/Fixtures/gemini/structured-with-approval-phase2-1.json b/tests/Fixtures/gemini/structured-with-approval-phase2-1.json new file mode 100644 index 000000000..90bf06cd4 --- /dev/null +++ b/tests/Fixtures/gemini/structured-with-approval-phase2-1.json @@ -0,0 +1,35 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "{\"result\":\"The file /tmp/test.txt has been deleted successfully.\"}" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.05 + } + ], + "usageMetadata": { + "promptTokenCount": 180, + "candidatesTokenCount": 20, + "totalTokenCount": 200, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 180 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 20 + } + ] + }, + "modelVersion": "gemini-2.0-flash", + "responseId": "structured-approval-phase2" +} diff --git a/tests/Fixtures/gemini/structured-with-approval-tool-1.json b/tests/Fixtures/gemini/structured-with-approval-tool-1.json new file mode 100644 index 000000000..9bf61a999 --- /dev/null +++ b/tests/Fixtures/gemini/structured-with-approval-tool-1.json @@ -0,0 +1,39 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "delete_file", + "args": { + "path": "/tmp/test.txt" + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.00003298009687569 + } + ], + "usageMetadata": { + "promptTokenCount": 200, + "candidatesTokenCount": 50, + "totalTokenCount": 250, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 200 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 50 + } + ] + }, + "modelVersion": "gemini-2.0-flash" +} diff --git a/tests/Fixtures/gemini/structured-with-client-executed-tool-1.json b/tests/Fixtures/gemini/structured-with-client-executed-tool-1.json new file mode 100644 index 000000000..c5a633d1c --- /dev/null +++ b/tests/Fixtures/gemini/structured-with-client-executed-tool-1.json @@ -0,0 +1,40 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "client_tool", + "args": { + "input": "test input" + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.00003298009687569 + } + ], + "usageMetadata": { + "promptTokenCount": 200, + "candidatesTokenCount": 50, + "totalTokenCount": 250, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 200 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 50 + } + ] + }, + "modelVersion": "gemini-2.0-flash" +} + diff --git a/tests/Fixtures/gemini/text-with-approval-tool-1.json b/tests/Fixtures/gemini/text-with-approval-tool-1.json new file mode 100644 index 000000000..317864568 --- /dev/null +++ b/tests/Fixtures/gemini/text-with-approval-tool-1.json @@ -0,0 +1,39 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "delete_file", + "args": { + "path": "/tmp/test.txt" + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.00003298009687569 + } + ], + "usageMetadata": { + "promptTokenCount": 100, + "candidatesTokenCount": 50, + "totalTokenCount": 150, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 100 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 50 + } + ] + }, + "modelVersion": "gemini-1.5-flash" +} diff --git a/tests/Fixtures/gemini/text-with-client-executed-tool-1.json b/tests/Fixtures/gemini/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..5fb57e5b9 --- /dev/null +++ b/tests/Fixtures/gemini/text-with-client-executed-tool-1.json @@ -0,0 +1,40 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "client_tool", + "args": { + "input": "test input" + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.00003298009687569 + } + ], + "usageMetadata": { + "promptTokenCount": 100, + "candidatesTokenCount": 50, + "totalTokenCount": 150, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 100 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 50 + } + ] + }, + "modelVersion": "gemini-1.5-flash" +} + diff --git a/tests/Fixtures/groq/stream-with-approval-tool-1.sse b/tests/Fixtures/groq/stream-with-approval-tool-1.sse new file mode 100644 index 000000000..614dee398 --- /dev/null +++ b/tests/Fixtures/groq/stream-with-approval-tool-1.sse @@ -0,0 +1,7 @@ +data: {"id":"chatcmpl-approval-tool-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_delete_file_stream","type":"function","function":{"name":"delete_file","arguments":""}}]},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_test"}} + +data: {"id":"chatcmpl-approval-tool-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\": \"/tmp/test.txt\"}"}}]},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_test"}} + +data: {"id":"chatcmpl-approval-tool-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.1,"prompt_tokens":100,"prompt_time":0.01,"completion_tokens":50,"completion_time":0.1,"total_tokens":150,"total_time":0.2},"x_groq":{"id":"req_test"}} + +data: [DONE] diff --git a/tests/Fixtures/groq/stream-with-client-executed-tool-1.sse b/tests/Fixtures/groq/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..c750092fc --- /dev/null +++ b/tests/Fixtures/groq/stream-with-client-executed-tool-1.sse @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_client_tool_stream","type":"function","function":{"name":"client_tool","arguments":""}}]},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_test"}} + +data: {"id":"chatcmpl-client-executed-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\": \"test input\"}"}}]},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_test"}} + +data: {"id":"chatcmpl-client-executed-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.1,"prompt_tokens":100,"prompt_time":0.01,"completion_tokens":50,"completion_time":0.1,"total_tokens":150,"total_time":0.2},"x_groq":{"id":"req_test"}} + +data: [DONE] + + diff --git a/tests/Fixtures/groq/text-with-approval-tool-1.json b/tests/Fixtures/groq/text-with-approval-tool-1.json new file mode 100644 index 000000000..4d430cf34 --- /dev/null +++ b/tests/Fixtures/groq/text-with-approval-tool-1.json @@ -0,0 +1 @@ +{"id":"chatcmpl-approval-tool","object":"chat.completion","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_delete_file","type":"function","function":{"name":"delete_file","arguments":"{\"path\": \"/tmp/test.txt\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.1,"prompt_tokens":100,"prompt_time":0.01,"completion_tokens":50,"completion_time":0.1,"total_tokens":150,"total_time":0.2},"system_fingerprint":"fp_test"} diff --git a/tests/Fixtures/groq/text-with-client-executed-tool-1.json b/tests/Fixtures/groq/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..eecec7bb8 --- /dev/null +++ b/tests/Fixtures/groq/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"chatcmpl-client-executed","object":"chat.completion","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_client_tool","type":"function","function":{"name":"client_tool","arguments":"{\"input\": \"test input\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.1,"prompt_tokens":100,"prompt_time":0.01,"completion_tokens":50,"completion_time":0.1,"total_tokens":150,"total_time":0.2},"system_fingerprint":"fp_test"} + diff --git a/tests/Fixtures/mistral/stream-with-approval-phase2-1.sse b/tests/Fixtures/mistral/stream-with-approval-phase2-1.sse new file mode 100644 index 000000000..77f334025 --- /dev/null +++ b/tests/Fixtures/mistral/stream-with-approval-phase2-1.sse @@ -0,0 +1,7 @@ +data: {"id":"approval-phase2","object":"chat.completion.chunk","created":1759185829,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + +data: {"id":"approval-phase2","object":"chat.completion.chunk","created":1759185829,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"content":"File deleted successfully."},"finish_reason":null}]} + +data: {"id":"approval-phase2","object":"chat.completion.chunk","created":1759185829,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"content":""},"finish_reason":"stop"}],"usage":{"prompt_tokens":150,"total_tokens":160,"completion_tokens":10}} + +data: [DONE] diff --git a/tests/Fixtures/mistral/stream-with-approval-tool-1.sse b/tests/Fixtures/mistral/stream-with-approval-tool-1.sse new file mode 100644 index 000000000..074d4eef6 --- /dev/null +++ b/tests/Fixtures/mistral/stream-with-approval-tool-1.sse @@ -0,0 +1,5 @@ +data: {"id":"approval-tool-test","object":"chat.completion.chunk","created":1759185828,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + +data: {"id":"approval-tool-test","object":"chat.completion.chunk","created":1759185828,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"tool_calls":[{"id":"delete_file_stream","function":{"name":"delete_file","arguments":"{\"path\": \"/tmp/test.txt\"}"},"index":0}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"total_tokens":150,"completion_tokens":50}} + +data: [DONE] diff --git a/tests/Fixtures/mistral/stream-with-client-executed-tool-1.sse b/tests/Fixtures/mistral/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..756442567 --- /dev/null +++ b/tests/Fixtures/mistral/stream-with-client-executed-tool-1.sse @@ -0,0 +1,6 @@ +data: {"id":"client-executed-test","object":"chat.completion.chunk","created":1759185828,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + +data: {"id":"client-executed-test","object":"chat.completion.chunk","created":1759185828,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"tool_calls":[{"id":"client_tool_stream","function":{"name":"client_tool","arguments":"{\"input\": \"test input\"}"},"index":0}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"total_tokens":150,"completion_tokens":50}} + +data: [DONE] + diff --git a/tests/Fixtures/mistral/text-with-approval-phase2-1.json b/tests/Fixtures/mistral/text-with-approval-phase2-1.json new file mode 100644 index 000000000..d17bba9fb --- /dev/null +++ b/tests/Fixtures/mistral/text-with-approval-phase2-1.json @@ -0,0 +1,21 @@ +{ + "id": "approval_phase2", + "object": "chat.completion", + "created": 1728462828, + "model": "mistral-large-latest", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The file has been deleted successfully." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 150, + "total_tokens": 160, + "completion_tokens": 10 + } +} diff --git a/tests/Fixtures/mistral/text-with-approval-tool-1.json b/tests/Fixtures/mistral/text-with-approval-tool-1.json new file mode 100644 index 000000000..b9fdb37fd --- /dev/null +++ b/tests/Fixtures/mistral/text-with-approval-tool-1.json @@ -0,0 +1,31 @@ +{ + "id": "approval_tool_test", + "object": "chat.completion", + "created": 1728462827, + "model": "mistral-large-latest", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "delete_file_123", + "type": "function", + "function": { + "name": "delete_file", + "arguments": "{\"path\": \"/tmp/test.txt\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "total_tokens": 150, + "completion_tokens": 50 + } +} diff --git a/tests/Fixtures/mistral/text-with-client-executed-tool-1.json b/tests/Fixtures/mistral/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..21a812489 --- /dev/null +++ b/tests/Fixtures/mistral/text-with-client-executed-tool-1.json @@ -0,0 +1,32 @@ +{ + "id": "client_executed_test", + "object": "chat.completion", + "created": 1728462827, + "model": "mistral-large-latest", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "client_tool_123", + "type": "function", + "function": { + "name": "client_tool", + "arguments": "{\"input\": \"test input\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "total_tokens": 150, + "completion_tokens": 50 + } +} + diff --git a/tests/Fixtures/ollama/stream-with-approval-tool-1.sse b/tests/Fixtures/ollama/stream-with-approval-tool-1.sse new file mode 100644 index 000000000..d736f9fc2 --- /dev/null +++ b/tests/Fixtures/ollama/stream-with-approval-tool-1.sse @@ -0,0 +1 @@ +{"model":"qwen2.5:14b","created_at":"2025-06-09T18:55:26.684517Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"delete_file","arguments":{"path":"/tmp/test.txt"}}}]},"done_reason":"stop","done":true,"total_duration":8210142000,"load_duration":22224542,"prompt_eval_count":100,"prompt_eval_duration":269880958,"eval_count":50,"eval_duration":7916594250} diff --git a/tests/Fixtures/ollama/stream-with-client-executed-tool-1.sse b/tests/Fixtures/ollama/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..044f3d26e --- /dev/null +++ b/tests/Fixtures/ollama/stream-with-client-executed-tool-1.sse @@ -0,0 +1,3 @@ +{"model":"qwen2.5:14b","created_at":"2025-06-09T18:55:26.684517Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"client_tool","arguments":{"input":"test input"}}}]},"done_reason":"stop","done":true,"total_duration":8210142000,"load_duration":22224542,"prompt_eval_count":100,"prompt_eval_duration":269880958,"eval_count":50,"eval_duration":7916594250} + + diff --git a/tests/Fixtures/ollama/text-with-approval-tool-1.json b/tests/Fixtures/ollama/text-with-approval-tool-1.json new file mode 100644 index 000000000..d736f9fc2 --- /dev/null +++ b/tests/Fixtures/ollama/text-with-approval-tool-1.json @@ -0,0 +1 @@ +{"model":"qwen2.5:14b","created_at":"2025-06-09T18:55:26.684517Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"delete_file","arguments":{"path":"/tmp/test.txt"}}}]},"done_reason":"stop","done":true,"total_duration":8210142000,"load_duration":22224542,"prompt_eval_count":100,"prompt_eval_duration":269880958,"eval_count":50,"eval_duration":7916594250} diff --git a/tests/Fixtures/ollama/text-with-client-executed-tool-1.json b/tests/Fixtures/ollama/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..d5d619328 --- /dev/null +++ b/tests/Fixtures/ollama/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"model":"qwen2.5:14b","created_at":"2025-06-09T18:55:26.684517Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"client_tool","arguments":{"input":"test input"}}}]},"done_reason":"stop","done":true,"total_duration":8210142000,"load_duration":22224542,"prompt_eval_count":100,"prompt_eval_duration":269880958,"eval_count":50,"eval_duration":7916594250} + diff --git a/tests/Fixtures/openai/stream-with-approval-phase2-1.json b/tests/Fixtures/openai/stream-with-approval-phase2-1.json new file mode 100644 index 000000000..e47b7a175 --- /dev/null +++ b/tests/Fixtures/openai/stream-with-approval-phase2-1.json @@ -0,0 +1,26 @@ +event: response.created +data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_approval_phase2","object":"response","created_at":1750705331,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.in_progress +data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_approval_phase2","object":"response","created_at":1750705331,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.output_item.added +data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"msg_approval_phase2","type":"message","status":"in_progress","content":[],"role":"assistant"}} + +event: response.content_part.added +data: {"type":"response.content_part.added","sequence_number":3,"item_id":"msg_approval_phase2","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":""}} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"msg_approval_phase2","output_index":0,"content_index":0,"delta":"File deleted successfully."} + +event: response.output_text.done +data: {"type":"response.output_text.done","sequence_number":5,"item_id":"msg_approval_phase2","output_index":0,"content_index":0,"text":"File deleted successfully."} + +event: response.content_part.done +data: {"type":"response.content_part.done","sequence_number":6,"item_id":"msg_approval_phase2","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":"File deleted successfully."}} + +event: response.output_item.done +data: {"type":"response.output_item.done","sequence_number":7,"output_index":0,"item":{"id":"msg_approval_phase2","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"File deleted successfully."}],"role":"assistant"}} + +event: response.completed +data: {"type":"response.completed","sequence_number":8,"response":{"id":"resp_approval_phase2","object":"response","created_at":1750705331,"status":"completed","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[{"id":"msg_approval_phase2","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"File deleted successfully."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":150,"input_tokens_details":{"cached_tokens":0},"output_tokens":10,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":160},"user":null,"metadata":{}}} diff --git a/tests/Fixtures/openai/stream-with-approval-tool-1.json b/tests/Fixtures/openai/stream-with-approval-tool-1.json new file mode 100644 index 000000000..c47529a22 --- /dev/null +++ b/tests/Fixtures/openai/stream-with-approval-tool-1.json @@ -0,0 +1,20 @@ +event: response.created +data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_approval_tool_stream","object":"response","created_at":1750705330,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"Delete a file. Requires user approval.","name":"delete_file","parameters":{"type":"object","properties":{"path":{"description":"File path to delete","type":"string"}},"required":["path"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.in_progress +data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_approval_tool_stream","object":"response","created_at":1750705330,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"Delete a file. Requires user approval.","name":"delete_file","parameters":{"type":"object","properties":{"path":{"description":"File path to delete","type":"string"}},"required":["path"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.output_item.added +data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"fc_approval_tool_stream","type":"function_call","status":"in_progress","arguments":"","call_id":"call_approval_tool_stream","name":"delete_file"}} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","sequence_number":3,"item_id":"fc_approval_tool_stream","output_index":0,"delta":"{\"path\":\"/tmp/test.txt\"}"} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","sequence_number":4,"item_id":"fc_approval_tool_stream","output_index":0,"arguments":"{\"path\":\"/tmp/test.txt\"}"} + +event: response.output_item.done +data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"id":"fc_approval_tool_stream","type":"function_call","status":"completed","arguments":"{\"path\":\"/tmp/test.txt\"}","call_id":"call_approval_tool_stream","name":"delete_file"}} + +event: response.completed +data: {"type":"response.completed","sequence_number":6,"response":{"id":"resp_approval_tool_stream","object":"response","created_at":1750705330,"status":"completed","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[{"id":"fc_approval_tool_stream","type":"function_call","status":"completed","arguments":"{\"path\":\"/tmp/test.txt\"}","call_id":"call_approval_tool_stream","name":"delete_file"}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"Delete a file. Requires user approval.","name":"delete_file","parameters":{"type":"object","properties":{"path":{"description":"File path to delete","type":"string"}},"required":["path"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":100,"input_tokens_details":{"cached_tokens":0},"output_tokens":50,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":150},"user":null,"metadata":{}}} diff --git a/tests/Fixtures/openai/stream-with-client-executed-tool-1.json b/tests/Fixtures/openai/stream-with-client-executed-tool-1.json new file mode 100644 index 000000000..272daa16c --- /dev/null +++ b/tests/Fixtures/openai/stream-with-client-executed-tool-1.json @@ -0,0 +1,22 @@ +event: response.created +data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_client_executed_stream","object":"response","created_at":1750705330,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"A tool that executes on the client","name":"client_tool","parameters":{"type":"object","properties":{"input":{"description":"Input parameter","type":"string"}},"required":["input"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.in_progress +data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_client_executed_stream","object":"response","created_at":1750705330,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"A tool that executes on the client","name":"client_tool","parameters":{"type":"object","properties":{"input":{"description":"Input parameter","type":"string"}},"required":["input"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.output_item.added +data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"fc_client_tool_stream","type":"function_call","status":"in_progress","arguments":"","call_id":"call_client_tool_stream","name":"client_tool"}} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","sequence_number":3,"item_id":"fc_client_tool_stream","output_index":0,"delta":"{\"input\":\"test input\"}"} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","sequence_number":4,"item_id":"fc_client_tool_stream","output_index":0,"arguments":"{\"input\":\"test input\"}"} + +event: response.output_item.done +data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"id":"fc_client_tool_stream","type":"function_call","status":"completed","arguments":"{\"input\":\"test input\"}","call_id":"call_client_tool_stream","name":"client_tool"}} + +event: response.completed +data: {"type":"response.completed","sequence_number":6,"response":{"id":"resp_client_executed_stream","object":"response","created_at":1750705330,"status":"completed","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[{"id":"fc_client_tool_stream","type":"function_call","status":"completed","arguments":"{\"input\":\"test input\"}","call_id":"call_client_tool_stream","name":"client_tool"}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"A tool that executes on the client","name":"client_tool","parameters":{"type":"object","properties":{"input":{"description":"Input parameter","type":"string"}},"required":["input"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":100,"input_tokens_details":{"cached_tokens":0},"output_tokens":50,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":150},"user":null,"metadata":{}}} + + diff --git a/tests/Fixtures/openai/structured-with-approval-phase2-1.json b/tests/Fixtures/openai/structured-with-approval-phase2-1.json new file mode 100644 index 000000000..43cb237f4 --- /dev/null +++ b/tests/Fixtures/openai/structured-with-approval-phase2-1.json @@ -0,0 +1,35 @@ +{ + "id": "resp_structured_approval_phase2", + "object": "response", + "created_at": 1762106102, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "msg_structured_approval_phase2", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "{\"result\":\"The file /tmp/test.txt has been deleted successfully.\"}" + } + ], + "role": "assistant" + } + ], + "usage": { + "input_tokens": 180, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 20, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 200 + }, + "service_tier": "default" +} diff --git a/tests/Fixtures/openai/structured-with-approval-tool-1.json b/tests/Fixtures/openai/structured-with-approval-tool-1.json new file mode 100644 index 000000000..d2b2f3204 --- /dev/null +++ b/tests/Fixtures/openai/structured-with-approval-tool-1.json @@ -0,0 +1,30 @@ +{ + "id": "resp_structured_approval_tool", + "object": "response", + "created_at": 1741989983, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "fc_delete_file_structured", + "type": "function_call", + "status": "completed", + "arguments": "{\"path\": \"/tmp/test.txt\"}", + "call_id": "call_delete_file_structured", + "name": "delete_file" + } + ], + "usage": { + "input_tokens": 200, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 50, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 250 + }, + "service_tier": "default", + "system_fingerprint": "fp_test" +} diff --git a/tests/Fixtures/openai/structured-with-client-executed-tool-1.json b/tests/Fixtures/openai/structured-with-client-executed-tool-1.json new file mode 100644 index 000000000..8aff9433b --- /dev/null +++ b/tests/Fixtures/openai/structured-with-client-executed-tool-1.json @@ -0,0 +1,31 @@ +{ + "id": "resp_structured_client_executed", + "object": "response", + "created_at": 1741989983, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "fc_client_tool_structured", + "type": "function_call", + "status": "completed", + "arguments": "{\"input\": \"test input\"}", + "call_id": "call_client_tool_structured", + "name": "client_tool" + } + ], + "usage": { + "input_tokens": 200, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 50, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 250 + }, + "service_tier": "default", + "system_fingerprint": "fp_test" +} + diff --git a/tests/Fixtures/openai/text-with-approval-phase2-1.json b/tests/Fixtures/openai/text-with-approval-phase2-1.json new file mode 100644 index 000000000..9f237d82b --- /dev/null +++ b/tests/Fixtures/openai/text-with-approval-phase2-1.json @@ -0,0 +1,34 @@ +{ + "id": "resp_approval_phase2", + "object": "response", + "created_at": 1741989984, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "msg_approval_phase2", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "The file has been deleted successfully." + } + ], + "role": "assistant" + } + ], + "usage": { + "input_tokens": 150, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 10, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 160 + }, + "service_tier": "default", + "system_fingerprint": "fp_test" +} diff --git a/tests/Fixtures/openai/text-with-approval-tool-1.json b/tests/Fixtures/openai/text-with-approval-tool-1.json new file mode 100644 index 000000000..a271ab2ff --- /dev/null +++ b/tests/Fixtures/openai/text-with-approval-tool-1.json @@ -0,0 +1,30 @@ +{ + "id": "resp_approval_tool_test", + "object": "response", + "created_at": 1741989983, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "fc_delete_file_123", + "type": "function_call", + "status": "completed", + "arguments": "{\"path\": \"/tmp/test.txt\"}", + "call_id": "call_delete_file_123", + "name": "delete_file" + } + ], + "usage": { + "input_tokens": 100, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 50, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 150 + }, + "service_tier": "default", + "system_fingerprint": "fp_test" +} diff --git a/tests/Fixtures/openai/text-with-client-executed-tool-1.json b/tests/Fixtures/openai/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..a62212d13 --- /dev/null +++ b/tests/Fixtures/openai/text-with-client-executed-tool-1.json @@ -0,0 +1,31 @@ +{ + "id": "resp_client_executed_test", + "object": "response", + "created_at": 1741989983, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "fc_client_tool_123", + "type": "function_call", + "status": "completed", + "arguments": "{\"input\": \"test input\"}", + "call_id": "call_client_tool_123", + "name": "client_tool" + } + ], + "usage": { + "input_tokens": 100, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 50, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 150 + }, + "service_tier": "default", + "system_fingerprint": "fp_test" +} + diff --git a/tests/Fixtures/openrouter/stream-with-approval-phase2-1.sse b/tests/Fixtures/openrouter/stream-with-approval-phase2-1.sse new file mode 100644 index 000000000..0fa18795f --- /dev/null +++ b/tests/Fixtures/openrouter/stream-with-approval-phase2-1.sse @@ -0,0 +1,7 @@ +data: {"id":"chatcmpl-approval-phase2","object":"chat.completion.chunk","created":1737243488,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-phase2","object":"chat.completion.chunk","created":1737243488,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"content":"File deleted successfully."},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-phase2","object":"chat.completion.chunk","created":1737243488,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":150,"completion_tokens":10,"total_tokens":160}} + +data: [DONE] diff --git a/tests/Fixtures/openrouter/stream-with-approval-tool-1.sse b/tests/Fixtures/openrouter/stream-with-approval-tool-1.sse new file mode 100644 index 000000000..633abedef --- /dev/null +++ b/tests/Fixtures/openrouter/stream-with-approval-tool-1.sse @@ -0,0 +1,7 @@ +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_delete_file_stream","type":"function","function":{"name":"delete_file","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\": \"/tmp/test.txt\"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] diff --git a/tests/Fixtures/openrouter/stream-with-client-executed-tool-1.sse b/tests/Fixtures/openrouter/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..e0a4d5e6c --- /dev/null +++ b/tests/Fixtures/openrouter/stream-with-client-executed-tool-1.sse @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_client_tool_stream","type":"function","function":{"name":"client_tool","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\": \"test input\"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] + + diff --git a/tests/Fixtures/openrouter/structured-with-approval-phase2-1.json b/tests/Fixtures/openrouter/structured-with-approval-phase2-1.json new file mode 100644 index 000000000..98cc9bb7e --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-approval-phase2-1.json @@ -0,0 +1 @@ +{"id":"gen-structured-approval-phase2","object":"chat.completion","created":1737243490,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"{\"result\":\"The file /tmp/test.txt has been deleted successfully.\"}"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":180,"completion_tokens":20,"total_tokens":200}} diff --git a/tests/Fixtures/openrouter/structured-with-approval-tool-1.json b/tests/Fixtures/openrouter/structured-with-approval-tool-1.json new file mode 100644 index 000000000..3aca633ff --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-approval-tool-1.json @@ -0,0 +1 @@ +{"id":"gen-structured-approval","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_delete_file","type":"function","function":{"name":"delete_file","arguments":"{\"path\":\"/tmp/test.txt\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} diff --git a/tests/Fixtures/openrouter/structured-with-client-executed-tool-1.json b/tests/Fixtures/openrouter/structured-with-client-executed-tool-1.json new file mode 100644 index 000000000..91e0c9975 --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-client-executed-tool-1.json @@ -0,0 +1 @@ +{"id":"gen-structured-client-exec","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_client_tool","type":"function","function":{"name":"client_tool","arguments":"{\"input\":\"test input\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} diff --git a/tests/Fixtures/openrouter/structured-with-multiple-tools-1.json b/tests/Fixtures/openrouter/structured-with-multiple-tools-1.json new file mode 100644 index 000000000..cc639c0aa --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-multiple-tools-1.json @@ -0,0 +1 @@ +{"id":"gen-structured-multi-1","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_weather_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Detroit\"}"}},{"id":"call_games_1","type":"function","function":{"name":"search_games","arguments":"{\"city\":\"Detroit\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":220,"completion_tokens":39,"total_tokens":259}} diff --git a/tests/Fixtures/openrouter/structured-with-multiple-tools-2.json b/tests/Fixtures/openrouter/structured-with-multiple-tools-2.json new file mode 100644 index 000000000..c1b01aa11 --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-multiple-tools-2.json @@ -0,0 +1 @@ +{"id":"gen-structured-multi-2","object":"chat.completion","created":1737243490,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"{\"game_time\":\"3pm\",\"weather_summary\":\"45°F and cold in Detroit\",\"recommendation\":\"Wear a warm coat to the game.\"}"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":287,"completion_tokens":37,"total_tokens":324}} diff --git a/tests/Fixtures/openrouter/structured-with-single-tool-1.json b/tests/Fixtures/openrouter/structured-with-single-tool-1.json new file mode 100644 index 000000000..3ef4cb670 --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-single-tool-1.json @@ -0,0 +1 @@ +{"id":"gen-structured-tool-1","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_weather_1","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"San Francisco, CA\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":120,"completion_tokens":18,"total_tokens":138}} diff --git a/tests/Fixtures/openrouter/structured-with-single-tool-2.json b/tests/Fixtures/openrouter/structured-with-single-tool-2.json new file mode 100644 index 000000000..a5dca5358 --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-single-tool-2.json @@ -0,0 +1 @@ +{"id":"gen-structured-tool-2","object":"chat.completion","created":1737243490,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"{\"summary\":\"The weather in San Francisco is 72°F and sunny.\",\"recommendation\":\"No coat needed — enjoy the sunshine!\"}"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":158,"completion_tokens":39,"total_tokens":197}} diff --git a/tests/Fixtures/openrouter/structured-with-tool-orchestration-1.json b/tests/Fixtures/openrouter/structured-with-tool-orchestration-1.json new file mode 100644 index 000000000..0574d5c0c --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-tool-orchestration-1.json @@ -0,0 +1 @@ +{"id":"gen-structured-orch-1","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_search_db","type":"function","function":{"name":"search_database","arguments":"{\"query\":\"AI safety\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":150,"completion_tokens":20,"total_tokens":170}} diff --git a/tests/Fixtures/openrouter/structured-with-tool-orchestration-2.json b/tests/Fixtures/openrouter/structured-with-tool-orchestration-2.json new file mode 100644 index 000000000..70c725a2f --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-tool-orchestration-2.json @@ -0,0 +1 @@ +{"id":"gen-structured-orch-2","object":"chat.completion","created":1737243490,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_fetch_ext","type":"function","function":{"name":"fetch_external","arguments":"{\"endpoint\":\"https://api.example.com/ai-safety\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":200,"completion_tokens":25,"total_tokens":225}} diff --git a/tests/Fixtures/openrouter/structured-with-tool-orchestration-3.json b/tests/Fixtures/openrouter/structured-with-tool-orchestration-3.json new file mode 100644 index 000000000..7bbc76cc9 --- /dev/null +++ b/tests/Fixtures/openrouter/structured-with-tool-orchestration-3.json @@ -0,0 +1 @@ +{"id":"gen-structured-orch-3","object":"chat.completion","created":1737243493,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"{\"findings\":\"AI safety research focuses on alignment, robustness, and interpretability.\",\"sources\":\"Internal database and external API\"}"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":280,"completion_tokens":40,"total_tokens":320}} diff --git a/tests/Fixtures/openrouter/structured-without-tool-calls-1.json b/tests/Fixtures/openrouter/structured-without-tool-calls-1.json new file mode 100644 index 000000000..23c63fa75 --- /dev/null +++ b/tests/Fixtures/openrouter/structured-without-tool-calls-1.json @@ -0,0 +1 @@ +{"id":"gen-structured-no-tools","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"{\"answer\":\"4\"}"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":80,"completion_tokens":10,"total_tokens":90}} diff --git a/tests/Fixtures/openrouter/text-with-approval-tool-1.json b/tests/Fixtures/openrouter/text-with-approval-tool-1.json new file mode 100644 index 000000000..38fb873bc --- /dev/null +++ b/tests/Fixtures/openrouter/text-with-approval-tool-1.json @@ -0,0 +1 @@ +{"id":"gen-approval-tool","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_delete_file","type":"function","function":{"name":"delete_file","arguments":"{\"path\":\"/tmp/test.txt\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} diff --git a/tests/Fixtures/openrouter/text-with-client-executed-tool-1.json b/tests/Fixtures/openrouter/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..5afde1944 --- /dev/null +++ b/tests/Fixtures/openrouter/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"gen-client-executed","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_client_tool","type":"function","function":{"name":"client_tool","arguments":"{\"input\":\"test input\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + diff --git a/tests/Fixtures/xai/stream-with-approval-tool-1.json b/tests/Fixtures/xai/stream-with-approval-tool-1.json new file mode 100644 index 000000000..332363381 --- /dev/null +++ b/tests/Fixtures/xai/stream-with-approval-tool-1.json @@ -0,0 +1,7 @@ +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_delete_file_stream","type":"function","function":{"name":"delete_file","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\":\"/tmp/test.txt\"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-approval-tool","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] diff --git a/tests/Fixtures/xai/stream-with-client-executed-tool-1.json b/tests/Fixtures/xai/stream-with-client-executed-tool-1.json new file mode 100644 index 000000000..63313a45e --- /dev/null +++ b/tests/Fixtures/xai/stream-with-client-executed-tool-1.json @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"client_tool","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\":\"test input\"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] + + diff --git a/tests/Fixtures/xai/text-with-approval-tool-1.json b/tests/Fixtures/xai/text-with-approval-tool-1.json new file mode 100644 index 000000000..f9013a0eb --- /dev/null +++ b/tests/Fixtures/xai/text-with-approval-tool-1.json @@ -0,0 +1,33 @@ +{ + "id": "approval-tool-test", + "object": "chat.completion", + "created": 1731129810, + "model": "grok-beta", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "0", + "function": { + "name": "delete_file", + "arguments": "{\"path\":\"/tmp/test.txt\"}" + }, + "type": "function" + } + ], + "refusal": null + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + }, + "system_fingerprint": "fp_test" +} diff --git a/tests/Fixtures/xai/text-with-client-executed-tool-1.json b/tests/Fixtures/xai/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..064707bc0 --- /dev/null +++ b/tests/Fixtures/xai/text-with-client-executed-tool-1.json @@ -0,0 +1,34 @@ +{ + "id": "client-executed-test", + "object": "chat.completion", + "created": 1731129810, + "model": "grok-beta", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "0", + "function": { + "name": "client_tool", + "arguments": "{\"input\":\"test input\"}" + }, + "type": "function" + } + ], + "refusal": null + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + }, + "system_fingerprint": "fp_test" +} + diff --git a/tests/Fixtures/z/generate-text-with-missing-meta-1.json b/tests/Fixtures/z/generate-text-with-missing-meta-1.json new file mode 100644 index 000000000..f95133b56 --- /dev/null +++ b/tests/Fixtures/z/generate-text-with-missing-meta-1.json @@ -0,0 +1,19 @@ +{ + "object": "chat.completion", + "created": 1737243487, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm an AI assistant. How can I help you today?" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 7, + "completion_tokens": 20, + "total_tokens": 27 + } +} diff --git a/tests/Fixtures/z/generate-text-with-missing-usage-1.json b/tests/Fixtures/z/generate-text-with-missing-usage-1.json new file mode 100644 index 000000000..90757eb1c --- /dev/null +++ b/tests/Fixtures/z/generate-text-with-missing-usage-1.json @@ -0,0 +1,16 @@ +{ + "id": "chatcmpl-no-usage", + "object": "chat.completion", + "created": 1737243487, + "model": "z-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm an AI assistant. How can I help you today?" + }, + "finish_reason": "stop" + } + ] +} diff --git a/tests/Fixtures/z/structured-missing-usage-1.json b/tests/Fixtures/z/structured-missing-usage-1.json new file mode 100644 index 000000000..7efa45206 --- /dev/null +++ b/tests/Fixtures/z/structured-missing-usage-1.json @@ -0,0 +1,16 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"message\":\"Hello.\",\"action\":\"ask_question\"}", + "role": "assistant" + } + } + ], + "created": 1765785136, + "id": "chatcmpl-123", + "model": "z-model", + "object": "chat.completion" +} diff --git a/tests/Fixtures/z/structured-with-approval-phase2-1.json b/tests/Fixtures/z/structured-with-approval-phase2-1.json new file mode 100644 index 000000000..c23f9e022 --- /dev/null +++ b/tests/Fixtures/z/structured-with-approval-phase2-1.json @@ -0,0 +1,21 @@ +{ + "id": "gen-structured-approval-phase2", + "object": "chat.completion", + "created": 1737243490, + "model": "z-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "{\"result\":\"The file /tmp/test.txt has been deleted successfully.\"}" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 180, + "completion_tokens": 20, + "total_tokens": 200 + } +} diff --git a/tests/Fixtures/z/structured-with-missing-meta-1.json b/tests/Fixtures/z/structured-with-missing-meta-1.json new file mode 100644 index 000000000..7ec6ba7f9 --- /dev/null +++ b/tests/Fixtures/z/structured-with-missing-meta-1.json @@ -0,0 +1,19 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"weather\":\"75º\",\"game_time\":\"3pm\"}", + "role": "assistant" + } + } + ], + "created": 1737243487, + "object": "chat.completion", + "usage": { + "prompt_tokens": 187, + "completion_tokens": 26, + "total_tokens": 213 + } +} diff --git a/tests/Fixtures/z/text-with-approval-phase2-1.json b/tests/Fixtures/z/text-with-approval-phase2-1.json new file mode 100644 index 000000000..f29275e40 --- /dev/null +++ b/tests/Fixtures/z/text-with-approval-phase2-1.json @@ -0,0 +1,21 @@ +{ + "id": "gen-approval-phase2", + "object": "chat.completion", + "created": 1737243490, + "model": "z-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The file /tmp/test.txt has been deleted successfully." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 180, + "completion_tokens": 20, + "total_tokens": 200 + } +} diff --git a/tests/Fixtures/z/text-with-approval-tool-1.json b/tests/Fixtures/z/text-with-approval-tool-1.json new file mode 100644 index 000000000..e507c5da4 --- /dev/null +++ b/tests/Fixtures/z/text-with-approval-tool-1.json @@ -0,0 +1,31 @@ +{ + "id": "gen-approval-tool", + "object": "chat.completion", + "created": 1737243487, + "model": "z-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_delete_file", + "type": "function", + "function": { + "name": "delete_file", + "arguments": "{\"path\":\"/tmp/test.txt\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + } +} diff --git a/tests/Fixtures/z/text-with-client-executed-tool-1.json b/tests/Fixtures/z/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..fdd9d1267 --- /dev/null +++ b/tests/Fixtures/z/text-with-client-executed-tool-1.json @@ -0,0 +1,31 @@ +{ + "id": "gen-client-executed", + "object": "chat.completion", + "created": 1737243487, + "model": "z-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_client_tool", + "type": "function", + "function": { + "name": "client_tool", + "arguments": "{\"input\":\"test input\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + } +} diff --git a/tests/Providers/Anthropic/AnthropicTextRequestTest.php b/tests/Providers/Anthropic/AnthropicTextRequestTest.php index b7286d0e0..7ac58891c 100644 --- a/tests/Providers/Anthropic/AnthropicTextRequestTest.php +++ b/tests/Providers/Anthropic/AnthropicTextRequestTest.php @@ -112,7 +112,8 @@ $tool = Tool::as('get_weather') ->for('Get current weather') - ->withStringParameter('location', 'The city name', true); + ->withStringParameter('location', 'The city name', true) + ->using(fn (string $location): string => "Weather for {$location}"); Prism::text() ->using(Provider::Anthropic, 'claude-3-5-haiku-latest') @@ -150,7 +151,8 @@ $tool = Tool::as('get_weather') ->for('Get current weather') - ->withStringParameter('location', 'The city name', true); + ->withStringParameter('location', 'The city name', true) + ->using(fn (string $location): string => "Weather for {$location}"); Prism::text() ->using(Provider::Anthropic, 'claude-3-5-haiku-latest') diff --git a/tests/Providers/Anthropic/AnthropicTextTest.php b/tests/Providers/Anthropic/AnthropicTextTest.php index 55ec9f1fb..131043428 100644 --- a/tests/Providers/Anthropic/AnthropicTextTest.php +++ b/tests/Providers/Anthropic/AnthropicTextTest.php @@ -9,6 +9,7 @@ use Illuminate\Support\Facades\Http; use Prism\Prism\Enums\Citations\CitationSourcePositionType; use Prism\Prism\Enums\Citations\CitationSourceType; +use Prism\Prism\Enums\FinishReason; use Prism\Prism\Enums\Provider; use Prism\Prism\Exceptions\PrismProviderOverloadedException; use Prism\Prism\Exceptions\PrismRateLimitedException; @@ -23,6 +24,9 @@ use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\ProviderRateLimit; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -582,6 +586,127 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/messages', 'anthropic/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes server tools and stops for client-executed tools in mixed scenario', function (): void { + FixtureResponse::fakeResponseSequence('v1/messages', 'anthropic/text-with-mixed-tools'); + + // Server-executed tool (has handler) + $serverTool = Tool::as('server_tool') + ->for('A server-side tool') + ->withStringParameter('query', 'Search query') + ->using(fn (string $query): string => "Result for: {$query}"); + + // Client-executed tool (no handler) + $clientTool = Tool::as('client_tool') + ->for('A client-side tool') + ->withStringParameter('action', 'Action to perform') + ->clientExecuted(); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$serverTool, $clientTool]) + ->withMaxSteps(3) + ->withPrompt('Use both tools') + ->asText(); + + // Should stop with ToolCalls finish reason (client tool pending) + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + + // Should have both tool calls in response + expect($response->toolCalls)->toHaveCount(2); + expect($response->toolCalls[0]->name)->toBe('server_tool'); + expect($response->toolCalls[1]->name)->toBe('client_tool'); + + // Server tool should have been executed (result in toolResults) + expect($response->toolResults)->toHaveCount(1); + expect($response->toolResults[0]->toolName)->toBe('server_tool'); + expect($response->toolResults[0]->result)->toBe('Result for: weather in SF'); + + // Only one step (stopped for client tool) + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('v1/messages', 'anthropic/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and continues to LLM response (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('v1/messages', 'anthropic/text-with-approval-phase2'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'toolu_delete_file_123', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_toolu_delete_file_123', toolCallId: 'toolu_delete_file_123'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_toolu_delete_file_123', approved: true), + ]), + ]) + ->asText(); + + expect($response->text)->toBe('The file has been deleted successfully.'); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); +}); + describe('exceptions', function (): void { it('throws a RateLimitException if the Anthropic responds with a 429', function (): void { Http::fake([ diff --git a/tests/Providers/Anthropic/StreamTest.php b/tests/Providers/Anthropic/StreamTest.php index 6927430ac..47678e3d1 100644 --- a/tests/Providers/Anthropic/StreamTest.php +++ b/tests/Providers/Anthropic/StreamTest.php @@ -26,6 +26,7 @@ use Prism\Prism\Streaming\Events\ThinkingCompleteEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; use Prism\Prism\Streaming\Events\ThinkingStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallDeltaEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; @@ -711,6 +712,84 @@ })->throws(PrismRequestTooLargeException::class); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('v1/messages', 'anthropic/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeStreamResponses('v1/messages', 'anthropic/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + describe('basic stream events', function (): void { it('can generate text with a basic stream', function (): void { FixtureResponse::fakeStreamResponses('v1/messages', 'anthropic/stream-basic-text'); diff --git a/tests/Providers/Anthropic/StructuredWithToolsTest.php b/tests/Providers/Anthropic/StructuredWithToolsTest.php index d47e3f25c..dd9aae92e 100644 --- a/tests/Providers/Anthropic/StructuredWithToolsTest.php +++ b/tests/Providers/Anthropic/StructuredWithToolsTest.php @@ -13,7 +13,13 @@ use Prism\Prism\Schema\ObjectSchema; use Prism\Prism\Schema\StringSchema; use Prism\Prism\Tool; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; +use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -200,6 +206,116 @@ expect($response->toolResults)->toBeArray(); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'anthropic/structured-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::structured() + ->using(Provider::Anthropic, 'claude-sonnet-4-0') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withProviderOptions(['use_tool_calling' => true]) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + + it('stops execution when approval-required tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'anthropic/structured-with-approval-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::Anthropic, 'claude-sonnet-4-0') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withProviderOptions(['use_tool_calling' => true]) + ->withPrompt('Delete /tmp/test.txt') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and returns structured output (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('*', 'anthropic/structured-with-approval-phase2'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::Anthropic, 'claude-sonnet-4-0') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withProviderOptions(['use_tool_calling' => true]) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'toolu_delete_structured', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_toolu_delete_structured', toolCallId: 'toolu_delete_structured'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_toolu_delete_structured', approved: true), + ]), + ]) + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKey('result') + ->and($response->structured['result'])->toContain('deleted'); + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + }); + it('includes strict field in tool definition when specified', function (): void { Prism::fake(); diff --git a/tests/Providers/DeepSeek/StreamTest.php b/tests/Providers/DeepSeek/StreamTest.php index f1b451cb4..d709135dc 100644 --- a/tests/Providers/DeepSeek/StreamTest.php +++ b/tests/Providers/DeepSeek/StreamTest.php @@ -17,6 +17,7 @@ use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Tests\Fixtures\FixtureResponse; @@ -136,6 +137,88 @@ }); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('chat/completions', 'deepseek/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeStreamResponses('chat/completions', 'deepseek/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('handles max_tokens parameter correctly', function (): void { FixtureResponse::fakeStreamResponses('chat/completions', 'deepseek/stream-max-tokens'); diff --git a/tests/Providers/DeepSeek/TextTest.php b/tests/Providers/DeepSeek/TextTest.php index 94c47c4d4..a5e3b0cd6 100644 --- a/tests/Providers/DeepSeek/TextTest.php +++ b/tests/Providers/DeepSeek/TextTest.php @@ -79,6 +79,53 @@ expect($response->finishReason)->toBe(FinishReason::Stop); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'deepseek/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'deepseek/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); +}); + it('can generate text using multiple tools and multiple steps', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'deepseek/generate-text-with-multiple-tools'); diff --git a/tests/Providers/Gemini/GeminiStreamTest.php b/tests/Providers/Gemini/GeminiStreamTest.php index 92e15dae3..40299f83d 100644 --- a/tests/Providers/Gemini/GeminiStreamTest.php +++ b/tests/Providers/Gemini/GeminiStreamTest.php @@ -16,6 +16,7 @@ use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Prism\Prism\ValueObjects\ProviderTool; @@ -232,6 +233,88 @@ }); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-1.5-flash') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-1.5-flash') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('yields ToolCall events before ToolResult events', function (): void { FixtureResponse::fakeResponseSequence('*', 'gemini/stream-with-tools'); diff --git a/tests/Providers/Gemini/GeminiTextTest.php b/tests/Providers/Gemini/GeminiTextTest.php index 354d86cb8..8d2055c70 100644 --- a/tests/Providers/Gemini/GeminiTextTest.php +++ b/tests/Providers/Gemini/GeminiTextTest.php @@ -166,6 +166,55 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/text-with-client-executed-tool'); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-1.5-flash') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/text-with-approval-tool'); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-1.5-flash') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('Image support with Gemini', function (): void { it('can send images from path', function (): void { FixtureResponse::fakeResponseSequence('*', 'gemini/image-detection'); diff --git a/tests/Providers/Gemini/StructuredWithToolsTest.php b/tests/Providers/Gemini/StructuredWithToolsTest.php index 36d94de36..a24b370ea 100644 --- a/tests/Providers/Gemini/StructuredWithToolsTest.php +++ b/tests/Providers/Gemini/StructuredWithToolsTest.php @@ -10,6 +10,12 @@ use Prism\Prism\Schema\ObjectSchema; use Prism\Prism\Schema\StringSchema; use Prism\Prism\Tool; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; +use Prism\Prism\ValueObjects\Messages\UserMessage; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -120,6 +126,113 @@ expect($finalStep->structured)->toBeArray(); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/structured-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::structured() + ->using(Provider::Gemini, 'gemini-2.0-flash') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + + it('stops execution when approval-required tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/structured-with-approval-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::Gemini, 'gemini-2.0-flash') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and returns structured output (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/structured-with-approval-phase2'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::Gemini, 'gemini-2.0-flash') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'delete_file', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_delete_file', toolCallId: 'delete_file'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_delete_file', approved: true), + ]), + ]) + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKey('result') + ->and($response->structured['result'])->toContain('deleted'); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); + it('returns structured output immediately when no tool calls needed', function (): void { FixtureResponse::fakeResponseSequence('*', 'gemini/structured-without-tool-calls'); diff --git a/tests/Providers/Groq/GroqTextTest.php b/tests/Providers/Groq/GroqTextTest.php index 82e924269..057a51b56 100644 --- a/tests/Providers/Groq/GroqTextTest.php +++ b/tests/Providers/Groq/GroqTextTest.php @@ -7,6 +7,7 @@ use Illuminate\Http\Client\Request; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Http; +use Prism\Prism\Enums\FinishReason; use Prism\Prism\Enums\Provider; use Prism\Prism\Enums\ToolChoice; use Prism\Prism\Exceptions\PrismRateLimitedException; @@ -118,6 +119,27 @@ ); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'groq/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('groq', 'llama-3.3-70b-versatile') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + it('handles specific tool choice', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'groq/generate-text-with-required-tool-call'); @@ -153,6 +175,30 @@ })->throwsNoExceptions(); }); +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'groq/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::Groq, 'llama-3.3-70b-versatile') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('Image support with grok', function (): void { it('can send images from path', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'groq/image-detection'); diff --git a/tests/Providers/Groq/StreamTest.php b/tests/Providers/Groq/StreamTest.php index 86f7783dd..99be0d86c 100644 --- a/tests/Providers/Groq/StreamTest.php +++ b/tests/Providers/Groq/StreamTest.php @@ -18,6 +18,7 @@ use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Prism\Prism\ValueObjects\Messages\UserMessage; @@ -124,6 +125,88 @@ expect($streamEndEvents)->toHaveCount(1); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('openai/v1/chat/completions', 'groq/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::Groq, 'llama-3.1-70b-versatile') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeStreamResponses('openai/v1/chat/completions', 'groq/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::Groq, 'llama-3.1-70b-versatile') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('handles maximum tool call depth exceeded', function (): void { FixtureResponse::fakeStreamResponses('openai/v1/chat/completions', 'groq/stream-with-tools'); diff --git a/tests/Providers/Mistral/MistralTextTest.php b/tests/Providers/Mistral/MistralTextTest.php index 90bf34405..161c5c20e 100644 --- a/tests/Providers/Mistral/MistralTextTest.php +++ b/tests/Providers/Mistral/MistralTextTest.php @@ -18,6 +18,9 @@ use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\ProviderRateLimit; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -118,6 +121,84 @@ ); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'mistral/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('mistral', 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'mistral/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('mistral', 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and continues to LLM response (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'mistral/text-with-approval-phase2'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('mistral', 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'delete_file_123', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_delete_file_123', toolCallId: 'delete_file_123'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_delete_file_123', approved: true), + ]), + ]) + ->generate(); + + expect($response->text)->toBe('The file has been deleted successfully.'); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); + it('handles specific tool choice', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'mistral/generate-text-with-required-tool-call'); diff --git a/tests/Providers/Mistral/StreamTest.php b/tests/Providers/Mistral/StreamTest.php index 55673c68b..2f42e8f61 100644 --- a/tests/Providers/Mistral/StreamTest.php +++ b/tests/Providers/Mistral/StreamTest.php @@ -17,14 +17,21 @@ use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; +use Prism\Prism\Streaming\Events\TextCompleteEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\TextStartEvent; use Prism\Prism\Streaming\Events\ThinkingCompleteEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; use Prism\Prism\Streaming\Events\ThinkingStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Messages\UserMessage; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -127,6 +134,166 @@ expect($streamEndEvents)->toHaveCount(1); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::Mistral, 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::Mistral, 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); + + it('continues streaming after approval when tool is approved (Phase 2)', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-with-approval-phase2'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $messages = [ + new UserMessage('Delete the file at /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'delete_file_stream', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_delete_file_stream', toolCallId: 'delete_file_stream'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_delete_file_stream', approved: true), + ]), + ]; + + $response = Prism::text() + ->using(Provider::Mistral, 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages($messages) + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + // StreamStartEvent comes first from resolveToolApprovals + expect($eventTypes[0])->toBe(StreamStartEvent::class); + + // ToolResultEvent comes right after (resolved approval) + expect($eventTypes[1])->toBe(ToolResultEvent::class); + expect($events[1]->toolResult->result)->toBe('Deleted: /tmp/test.txt'); + expect($events[1]->success)->toBeTrue(); + + // No second StreamStartEvent (already started) + $streamStartCount = count(array_filter($eventTypes, fn ($t): bool => $t === StreamStartEvent::class)); + expect($streamStartCount)->toBe(1); + + // Then the LLM continuation events follow + expect($eventTypes)->toContain(StepStartEvent::class); + expect($eventTypes)->toContain(TextStartEvent::class); + expect($eventTypes)->toContain(TextDeltaEvent::class); + expect($eventTypes)->toContain(TextCompleteEvent::class); + expect($eventTypes)->toContain(StepFinishEvent::class); + + // StepStartEvent comes after ToolResultEvent + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + expect($stepStartIndex)->toBeGreaterThan(1); + + // Verify text content + $text = ''; + foreach ($events as $event) { + if ($event instanceof TextDeltaEvent) { + $text .= $event->delta; + } + } + expect($text)->toBe('File deleted successfully.'); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::Stop); + }); +}); + it('handles maximum tool call depth exceeded', function (): void { FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-with-tools-1'); diff --git a/tests/Providers/Ollama/StreamTest.php b/tests/Providers/Ollama/StreamTest.php index cfd2721a1..b167fead2 100644 --- a/tests/Providers/Ollama/StreamTest.php +++ b/tests/Providers/Ollama/StreamTest.php @@ -17,6 +17,7 @@ use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Tests\Fixtures\FixtureResponse; @@ -126,6 +127,88 @@ expect($streamEndEvents)->toHaveCount(1); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('api/chat', 'ollama/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('ollama', 'qwen2.5:14b') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeStreamResponses('api/chat', 'ollama/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('ollama', 'qwen2.5:14b') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('throws a PrismRateLimitedException with a 429 response code', function (): void { Http::fake([ '*' => Http::response( diff --git a/tests/Providers/Ollama/TextTest.php b/tests/Providers/Ollama/TextTest.php index 63ddf9528..c56f97b8c 100644 --- a/tests/Providers/Ollama/TextTest.php +++ b/tests/Providers/Ollama/TextTest.php @@ -6,6 +6,7 @@ use Illuminate\Http\Client\Request; use Illuminate\Support\Facades\Http; +use Prism\Prism\Enums\FinishReason; use Prism\Prism\Enums\Provider; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; @@ -114,6 +115,53 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('api/chat', 'ollama/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('ollama', 'qwen2.5:14b') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('api/chat', 'ollama/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('ollama', 'qwen2.5:14b') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('Thinking parameter', function (): void { it('includes think parameter when thinking is enabled', function (): void { FixtureResponse::fakeResponseSequence('api/chat', 'ollama/text-with-thinking-enabled'); diff --git a/tests/Providers/OpenAI/StreamTest.php b/tests/Providers/OpenAI/StreamTest.php index 2354948a4..c02e3bf12 100644 --- a/tests/Providers/OpenAI/StreamTest.php +++ b/tests/Providers/OpenAI/StreamTest.php @@ -6,6 +6,7 @@ use Illuminate\Http\Client\Request; use Illuminate\Support\Facades\Http; +use Prism\Prism\Enums\FinishReason; use Prism\Prism\Enums\Provider; use Prism\Prism\Exceptions\PrismException; use Prism\Prism\Facades\Prism; @@ -20,10 +21,17 @@ use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\TextStartEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallDeltaEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; +use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\ProviderTool; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Prism\Prism\ValueObjects\Usage; use Tests\Fixtures\FixtureResponse; @@ -463,6 +471,166 @@ Http::assertSent(fn (Request $request): bool => $request->data()['parallel_tool_calls'] === false); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); + + it('continues streaming after approval when tool is approved (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-with-approval-phase2'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $messages = [ + new UserMessage('Delete the file at /tmp/test.txt'), + new AssistantMessage( + content: 'I will delete the file.', + toolCalls: [ + new ToolCall(id: 'fc_approval_tool_stream', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_fc_approval_tool_stream', toolCallId: 'fc_approval_tool_stream'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_fc_approval_tool_stream', approved: true), + ]), + ]; + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages($messages) + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + // StreamStartEvent comes first from resolveToolApprovals + expect($eventTypes[0])->toBe(StreamStartEvent::class); + + // ToolResultEvent comes right after (resolved approval) + expect($eventTypes[1])->toBe(ToolResultEvent::class); + expect($events[1]->toolResult->result)->toBe('Deleted: /tmp/test.txt'); + expect($events[1]->success)->toBeTrue(); + + // No second StreamStartEvent (already started) + $streamStartCount = count(array_filter($eventTypes, fn ($t): bool => $t === StreamStartEvent::class)); + expect($streamStartCount)->toBe(1); + + // Then the LLM continuation events follow + expect($eventTypes)->toContain(StepStartEvent::class); + expect($eventTypes)->toContain(TextStartEvent::class); + expect($eventTypes)->toContain(TextDeltaEvent::class); + expect($eventTypes)->toContain(TextCompleteEvent::class); + expect($eventTypes)->toContain(StepFinishEvent::class); + + // StepStartEvent comes after ToolResultEvent + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + expect($stepStartIndex)->toBeGreaterThan(1); + + // Verify text content + $text = ''; + foreach ($events as $event) { + if ($event instanceof TextDeltaEvent) { + $text .= $event->delta; + } + } + expect($text)->toBe('File deleted successfully.'); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::Stop); + }); +}); + it('emits usage information', function (): void { FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-basic-text-responses'); diff --git a/tests/Providers/OpenAI/StructuredWithToolsTest.php b/tests/Providers/OpenAI/StructuredWithToolsTest.php index 7afd8910f..9b52bfc1b 100644 --- a/tests/Providers/OpenAI/StructuredWithToolsTest.php +++ b/tests/Providers/OpenAI/StructuredWithToolsTest.php @@ -10,6 +10,12 @@ use Prism\Prism\Schema\ObjectSchema; use Prism\Prism\Schema\StringSchema; use Prism\Prism\Tool; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; +use Prism\Prism\ValueObjects\Messages\UserMessage; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; describe('Structured output with tools for OpenAI', function (): void { @@ -148,6 +154,113 @@ expect($response->steps)->toHaveCount(1); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/structured-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::structured() + ->using(Provider::OpenAI, 'gpt-4o') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + + it('stops execution when approval-required tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/structured-with-approval-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::OpenAI, 'gpt-4o') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and returns structured output (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/structured-with-approval-phase2'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::OpenAI, 'gpt-4o') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'fc_delete_file_structured', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_fc_delete_file_structured', toolCallId: 'fc_delete_file_structured'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_fc_delete_file_structured', approved: true), + ]), + ]) + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKey('result') + ->and($response->structured['result'])->toContain('deleted'); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); + it('handles tool orchestration correctly with multiple tool types', function (): void { FixtureResponse::fakeResponseSequence('v1/responses', 'openai/structured-with-tool-orchestration'); diff --git a/tests/Providers/OpenAI/TextTest.php b/tests/Providers/OpenAI/TextTest.php index 7055ec614..5b1a40020 100644 --- a/tests/Providers/OpenAI/TextTest.php +++ b/tests/Providers/OpenAI/TextTest.php @@ -21,6 +21,9 @@ use Prism\Prism\ValueObjects\Messages\UserMessage; use Prism\Prism\ValueObjects\ProviderTool; use Prism\Prism\ValueObjects\ProviderToolCall; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -331,6 +334,88 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and continues to LLM response (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/text-with-approval-phase2'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'fc_delete_file_123', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_fc_delete_file_123', toolCallId: 'fc_delete_file_123'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_fc_delete_file_123', approved: true), + ]), + ]) + ->asText(); + + expect($response->text)->toBe('The file has been deleted successfully.'); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); +}); + it('sets usage correctly with automatic caching', function (): void { FixtureResponse::fakeResponseSequence( 'v1/responses', diff --git a/tests/Providers/OpenRouter/StreamTest.php b/tests/Providers/OpenRouter/StreamTest.php index af61bc271..2dfc7670e 100644 --- a/tests/Providers/OpenRouter/StreamTest.php +++ b/tests/Providers/OpenRouter/StreamTest.php @@ -18,8 +18,15 @@ use Prism\Prism\Streaming\Events\TextStartEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; use Prism\Prism\Streaming\Events\ThinkingStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; +use Prism\Prism\ValueObjects\Messages\UserMessage; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -248,6 +255,166 @@ expect($streamEndEvents)->not->toBeEmpty(); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); + + it('continues streaming after approval when tool is approved (Phase 2)', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-with-approval-phase2'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $messages = [ + new UserMessage('Delete the file at /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call_delete_file_stream', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_call_delete_file_stream', toolCallId: 'call_delete_file_stream'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_call_delete_file_stream', approved: true), + ]), + ]; + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages($messages) + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + // StreamStartEvent comes first from resolveToolApprovals + expect($eventTypes[0])->toBe(StreamStartEvent::class); + + // ToolResultEvent comes right after (resolved approval) + expect($eventTypes[1])->toBe(ToolResultEvent::class); + expect($events[1]->toolResult->result)->toBe('Deleted: /tmp/test.txt'); + expect($events[1]->success)->toBeTrue(); + + // No second StreamStartEvent (already started) + $streamStartCount = count(array_filter($eventTypes, fn ($t): bool => $t === StreamStartEvent::class)); + expect($streamStartCount)->toBe(1); + + // Then the LLM continuation events follow + expect($eventTypes)->toContain(StepStartEvent::class); + expect($eventTypes)->toContain(TextStartEvent::class); + expect($eventTypes)->toContain(TextDeltaEvent::class); + expect($eventTypes)->toContain(TextCompleteEvent::class); + expect($eventTypes)->toContain(StepFinishEvent::class); + + // StepStartEvent comes after ToolResultEvent + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + expect($stepStartIndex)->toBeGreaterThan(1); + + // Verify text content + $text = ''; + foreach ($events as $event) { + if ($event instanceof TextDeltaEvent) { + $text .= $event->delta; + } + } + expect($text)->toBe('File deleted successfully.'); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::Stop); + }); +}); + it('can handle reasoning/thinking tokens in streaming', function (): void { FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-text-with-reasoning'); diff --git a/tests/Providers/OpenRouter/StructuredWithToolsTest.php b/tests/Providers/OpenRouter/StructuredWithToolsTest.php new file mode 100644 index 000000000..9752eb588 --- /dev/null +++ b/tests/Providers/OpenRouter/StructuredWithToolsTest.php @@ -0,0 +1,297 @@ +set('prism.providers.openrouter.api_key', env('OPENROUTER_API_KEY', 'test-api-key')); +}); + +describe('Structured output with tools for OpenRouter', function (): void { + it('can generate structured output with a single tool', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/structured-with-single-tool'); + + $schema = new ObjectSchema( + 'weather_analysis', + 'Analysis of weather conditions', + [ + new StringSchema('summary', 'A summary of the weather', true), + new StringSchema('recommendation', 'A recommendation based on weather', true), + ], + ['summary', 'recommendation'] + ); + + $weatherTool = (new Tool) + ->as('get_weather') + ->for('Get current weather for a location') + ->withStringParameter('location', 'The city and state') + ->using(fn (string $location): string => "Weather in {$location}: 72°F, sunny"); + + $response = Prism::structured() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withSchema($schema) + ->withTools([$weatherTool]) + ->withMaxSteps(3) + ->withPrompt('What is the weather in San Francisco and should I wear a coat?') + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKeys(['summary', 'recommendation']) + ->and($response->structured['summary'])->toBeString() + ->and($response->structured['recommendation'])->toBeString(); + + expect($response->toolCalls)->toBeArray(); + expect($response->toolResults)->toBeArray(); + + $finalStep = $response->steps->last(); + expect($finalStep->finishReason)->toBeIn([FinishReason::Stop, FinishReason::ToolCalls]); + }); + + it('can generate structured output with multiple tools and multi-step execution', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/structured-with-multiple-tools'); + + $schema = new ObjectSchema( + 'game_analysis', + 'Analysis of game time and weather', + [ + new StringSchema('game_time', 'The time of the game', true), + new StringSchema('weather_summary', 'Summary of weather conditions', true), + new StringSchema('recommendation', 'Recommendation on what to wear', true), + ], + ['game_time', 'weather_summary', 'recommendation'] + ); + + $tools = [ + (new Tool) + ->as('get_weather') + ->for('Get current weather for a location') + ->withStringParameter('city', 'The city name') + ->using(fn (string $city): string => "Weather in {$city}: 45°F and cold"), + (new Tool) + ->as('search_games') + ->for('Search for game times in a city') + ->withStringParameter('city', 'The city name') + ->using(fn (string $city): string => 'The Tigers game is at 3pm in Detroit'), + ]; + + $response = Prism::structured() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withSchema($schema) + ->withTools($tools) + ->withMaxSteps(5) + ->withPrompt('What time is the Tigers game today in Detroit and should I wear a coat? Please check both the game time and weather.') + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKeys(['game_time', 'weather_summary', 'recommendation']) + ->and($response->structured['game_time'])->toBeString() + ->and($response->structured['weather_summary'])->toBeString() + ->and($response->structured['recommendation'])->toBeString(); + + expect($response->toolCalls)->toBeArray(); + expect($response->toolResults)->toBeArray(); + + expect($response->steps)->not()->toBeEmpty(); + + $finalStep = $response->steps->last(); + expect($finalStep->structured)->toBeArray(); + }); + + it('returns structured output immediately when no tool calls needed', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/structured-without-tool-calls'); + + $schema = new ObjectSchema( + 'analysis', + 'Simple analysis', + [ + new StringSchema('answer', 'The answer', true), + ], + ['answer'] + ); + + $weatherTool = (new Tool) + ->as('get_weather') + ->for('Get weather for a location') + ->withStringParameter('location', 'The location') + ->using(fn (string $location): string => "Weather data for {$location}"); + + $response = Prism::structured() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withSchema($schema) + ->withTools([$weatherTool]) + ->withPrompt('What is 2 + 2?') + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKey('answer') + ->and($response->structured['answer'])->toBeString(); + + expect($response->toolCalls)->toBeArray(); + + expect($response->steps)->toHaveCount(1); + }); + + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/structured-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::structured() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + + it('stops execution when approval-required tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/structured-with-approval-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and returns structured output (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/structured-with-approval-phase2'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call_delete_file', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_call_delete_file', toolCallId: 'call_delete_file'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_call_delete_file', approved: true), + ]), + ]) + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKey('result') + ->and($response->structured['result'])->toContain('deleted'); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); + + it('handles tool orchestration correctly with multiple tool types', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/structured-with-tool-orchestration'); + + $schema = new ObjectSchema( + 'research_summary', + 'Summary of research findings', + [ + new StringSchema('findings', 'Key findings from research', true), + new StringSchema('sources', 'Sources consulted', true), + ], + ['findings', 'sources'] + ); + + $tools = [ + (new Tool) + ->as('search_database') + ->for('Search internal database') + ->withStringParameter('query', 'Search query') + ->using(fn (string $query): string => "Database results for: {$query}"), + (new Tool) + ->as('fetch_external') + ->for('Fetch data from external API') + ->withStringParameter('endpoint', 'API endpoint') + ->using(fn (string $endpoint): string => "External data from: {$endpoint}"), + ]; + + $response = Prism::structured() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withSchema($schema) + ->withTools($tools) + ->withMaxSteps(5) + ->withPrompt('Research the topic "AI safety" using both internal and external sources') + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKeys(['findings', 'sources']); + + expect($response->steps)->not()->toBeEmpty(); + expect($response->toolCalls)->toBeArray(); + expect($response->toolResults)->toBeArray(); + }); +}); diff --git a/tests/Providers/OpenRouter/TextTest.php b/tests/Providers/OpenRouter/TextTest.php index 8408374ec..6b68459c5 100644 --- a/tests/Providers/OpenRouter/TextTest.php +++ b/tests/Providers/OpenRouter/TextTest.php @@ -169,6 +169,53 @@ expect($response->finishReason)->toBe(FinishReason::Stop); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); +}); + it('forwards advanced provider options to openrouter', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/generate-text-with-a-prompt'); diff --git a/tests/Providers/XAI/StreamTest.php b/tests/Providers/XAI/StreamTest.php index 4e3ffcd1f..cde141b35 100644 --- a/tests/Providers/XAI/StreamTest.php +++ b/tests/Providers/XAI/StreamTest.php @@ -16,6 +16,7 @@ use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Tests\Fixtures\FixtureResponse; @@ -133,6 +134,88 @@ }); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'xai/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using('xai', 'grok-4') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + +describe('approval-required tools', function (): void { + it('stops streaming when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'xai/stream-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('xai', 'grok-4') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + expect($eventTypes[0])->toBe(StreamStartEvent::class); + expect($eventTypes[1])->toBe(StepStartEvent::class); + + $approvalIndex = array_search(ToolApprovalRequestEvent::class, $eventTypes); + expect($approvalIndex)->not->toBeFalse(); + + $approvalEvent = $events[$approvalIndex]; + expect($approvalEvent->toolCall->name)->toBe('delete_file'); + + $toolCallIndex = array_search(ToolCallEvent::class, $eventTypes); + expect($toolCallIndex)->not->toBeFalse(); + expect($toolCallIndex)->toBeLessThan($approvalIndex); + + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + expect($stepFinishIndex)->toBeGreaterThan($approvalIndex); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('handles max_tokens parameter correctly', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'xai/stream-basic-text-responses'); diff --git a/tests/Providers/XAI/XAITextTest.php b/tests/Providers/XAI/XAITextTest.php index 7d8f7a9bc..2c43f1eac 100644 --- a/tests/Providers/XAI/XAITextTest.php +++ b/tests/Providers/XAI/XAITextTest.php @@ -160,6 +160,53 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'xai/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::XAI, 'grok-beta') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'xai/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using('xai', 'grok-beta') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('Image support with XAI', function (): void { it('can send images from path', function (): void { FixtureResponse::fakeResponseSequence('chat/completions', 'xai/image-detection'); diff --git a/tests/Providers/Z/StructuredWithToolsTest.php b/tests/Providers/Z/StructuredWithToolsTest.php new file mode 100644 index 000000000..b2590c78d --- /dev/null +++ b/tests/Providers/Z/StructuredWithToolsTest.php @@ -0,0 +1,132 @@ +set('prism.providers.z.api_key', env('Z_API_KEY', 'zai-123')); +}); + +describe('Structured output with tools for Z', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/text-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::structured() + ->using(Provider::Z, 'z-model') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + + it('stops execution when approval-required tool is called', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/text-with-approval-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::Z, 'z-model') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete /tmp/test.txt') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + }); + + it('executes approved tool and returns structured output (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/structured-with-approval-phase2'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::structured() + ->using(Provider::Z, 'z-model') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call_delete_file', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_call_delete_file', toolCallId: 'call_delete_file'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_call_delete_file', approved: true), + ]), + ]) + ->asStructured(); + + expect($response->structured)->toBeArray() + ->and($response->structured)->toHaveKey('result') + ->and($response->structured['result'])->toContain('deleted'); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); +}); diff --git a/tests/Providers/Z/ZStructuredTest.php b/tests/Providers/Z/ZStructuredTest.php index 7a4f4f98c..ed4cf4da3 100644 --- a/tests/Providers/Z/ZStructuredTest.php +++ b/tests/Providers/Z/ZStructuredTest.php @@ -10,75 +10,130 @@ use Prism\Prism\Schema\EnumSchema; use Prism\Prism\Schema\ObjectSchema; use Prism\Prism\Schema\StringSchema; +use Prism\Prism\Structured\Response as StructuredResponse; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { config()->set('prism.providers.z.api_key', env('Z_API_KEY', 'zai-123')); }); -it('Z provider handles structured request', function (): void { - FixtureResponse::fakeResponseSequence('chat/completions', 'z/structured-basic-response'); - - $schema = new ObjectSchema( - name: 'InterviewResponse', - description: 'Structured response from AI interviewer', - properties: [ - new StringSchema( - name: 'message', - description: 'The AI interviewer response message', - nullable: false - ), - new EnumSchema( - name: 'action', - description: 'The next action to take in the interview', - options: ['ask_question', 'ask_followup', 'ask_clarification', 'complete_interview'], - nullable: false - ), - new EnumSchema( - name: 'status', - description: 'Current interview status', - options: ['waiting_for_answer', 'question_asked', 'followup_asked', 'completed'], - nullable: false - ), - new BooleanSchema( - name: 'is_question', - description: 'Whether this response contains a question', - nullable: false - ), - new StringSchema( - name: 'question_type', - description: 'Type of question being asked', - nullable: true - ), - new BooleanSchema( - name: 'move_to_next_question', - description: 'Whether to move to the next question after this response', - nullable: false - ), - ], - requiredFields: ['message', 'action', 'status', 'is_question', 'move_to_next_question'] - ); - - $response = Prism::structured() - ->using(Provider::Z, 'z-model') - ->withSchema($schema) - ->asStructured(); - - $text = <<<'JSON_STRUCTURED' +describe('Structured output for Z', function (): void { + it('returns structured output', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/structured-basic-response'); + + $schema = new ObjectSchema( + name: 'InterviewResponse', + description: 'Structured response from AI interviewer', + properties: [ + new StringSchema( + name: 'message', + description: 'The AI interviewer response message', + nullable: false + ), + new EnumSchema( + name: 'action', + description: 'The next action to take in the interview', + options: ['ask_question', 'ask_followup', 'ask_clarification', 'complete_interview'], + nullable: false + ), + new EnumSchema( + name: 'status', + description: 'Current interview status', + options: ['waiting_for_answer', 'question_asked', 'followup_asked', 'completed'], + nullable: false + ), + new BooleanSchema( + name: 'is_question', + description: 'Whether this response contains a question', + nullable: false + ), + new StringSchema( + name: 'question_type', + description: 'Type of question being asked', + nullable: true + ), + new BooleanSchema( + name: 'move_to_next_question', + description: 'Whether to move to the next question after this response', + nullable: false + ), + ], + requiredFields: ['message', 'action', 'status', 'is_question', 'move_to_next_question'] + ); + + $response = Prism::structured() + ->using(Provider::Z, 'z-model') + ->withSchema($schema) + ->asStructured(); + + expect($response)->toBeInstanceOf(StructuredResponse::class); + + $text = <<<'JSON_STRUCTURED' {"message":"That's a fantastic real-world application! Building a customer service chatbot with Laravel sounds like a great project that could really help streamline customer interactions. Let's shift gears a bit and talk about database optimization, which is crucial for backend performance. Have you had experience with database indexing, and can you share a situation where you needed to optimize database queries in a project?","action":"ask_question","status":"question_asked","is_question":true,"question_type":"database_optimization","move_to_next_question":true} JSON_STRUCTURED; - expect($response->text)->toBe($text) - ->and($response->structured)->toBe([ - 'message' => "That's a fantastic real-world application! Building a customer service chatbot with Laravel sounds like a great project that could really help streamline customer interactions. Let's shift gears a bit and talk about database optimization, which is crucial for backend performance. Have you had experience with database indexing, and can you share a situation where you needed to optimize database queries in a project?", - 'action' => 'ask_question', - 'status' => 'question_asked', - 'is_question' => true, - 'question_type' => 'database_optimization', - 'move_to_next_question' => true, - ]) - ->and($response->usage->promptTokens)->toBe(1309) - ->and($response->usage->completionTokens)->toBe(129) - ->and($response->meta->id)->toBe('chatcmpl-123') - ->and($response->meta->model)->toBe('z-model'); + expect($response->text)->toBe($text) + ->and($response->structured)->toBe([ + 'message' => "That's a fantastic real-world application! Building a customer service chatbot with Laravel sounds like a great project that could really help streamline customer interactions. Let's shift gears a bit and talk about database optimization, which is crucial for backend performance. Have you had experience with database indexing, and can you share a situation where you needed to optimize database queries in a project?", + 'action' => 'ask_question', + 'status' => 'question_asked', + 'is_question' => true, + 'question_type' => 'database_optimization', + 'move_to_next_question' => true, + ]) + ->and($response->usage->promptTokens)->toBe(1309) + ->and($response->usage->completionTokens)->toBe(129) + ->and($response->meta->id)->toBe('chatcmpl-123') + ->and($response->meta->model)->toBe('z-model'); + }); + + it('handles missing usage data in response', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/structured-missing-usage'); + + $schema = new ObjectSchema( + name: 'InterviewResponse', + description: 'Structured response', + properties: [ + new StringSchema('message', 'The message', nullable: false), + new StringSchema('action', 'The action', nullable: false), + ], + requiredFields: ['message', 'action'] + ); + + $response = Prism::structured() + ->using(Provider::Z, 'z-model') + ->withSchema($schema) + ->asStructured(); + + expect($response)->toBeInstanceOf(StructuredResponse::class); + expect($response->structured)->toBeArray(); + expect($response->usage->promptTokens)->toBe(0); + expect($response->usage->completionTokens)->toBe(0); + }); + + it('handles responses with missing id and model fields', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/structured-with-missing-meta'); + + $schema = new ObjectSchema( + name: 'Output', + description: 'Output', + properties: [ + new StringSchema('weather', 'Weather', nullable: false), + new StringSchema('game_time', 'Game time', nullable: false), + ], + requiredFields: ['weather', 'game_time'] + ); + + $response = Prism::structured() + ->using(Provider::Z, 'z-model') + ->withSchema($schema) + ->withPrompt('What time is the game?') + ->asStructured(); + + expect($response)->toBeInstanceOf(StructuredResponse::class); + expect($response->meta->id)->toBe(''); + expect($response->meta->model)->toBe('z-model'); + expect($response->structured['weather'])->toBe('75º'); + expect($response->structured['game_time'])->toBe('3pm'); + }); }); diff --git a/tests/Providers/Z/ZTextTest.php b/tests/Providers/Z/ZTextTest.php index 96889ee3e..a7fb7626b 100644 --- a/tests/Providers/Z/ZTextTest.php +++ b/tests/Providers/Z/ZTextTest.php @@ -16,7 +16,12 @@ use Prism\Prism\ValueObjects\Media\Document; use Prism\Prism\ValueObjects\Media\Image; use Prism\Prism\ValueObjects\Media\Video; +use Prism\Prism\ValueObjects\Messages\AssistantMessage; +use Prism\Prism\ValueObjects\Messages\ToolResultMessage; use Prism\Prism\ValueObjects\Messages\UserMessage; +use Prism\Prism\ValueObjects\ToolApprovalRequest; +use Prism\Prism\ValueObjects\ToolApprovalResponse; +use Prism\Prism\ValueObjects\ToolCall; use Tests\Fixtures\FixtureResponse; beforeEach(function (): void { @@ -63,6 +68,35 @@ ->and($response->steps[0]->finishReason)->toBe(FinishReason::Stop); }); + it('handles missing usage data in response', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/generate-text-with-missing-usage'); + + $response = Prism::text() + ->using(Provider::Z, 'z-model') + ->withPrompt('Who are you?') + ->asText(); + + expect($response)->toBeInstanceOf(TextResponse::class); + expect($response->usage->promptTokens)->toBe(0); + expect($response->usage->completionTokens)->toBe(0); + expect($response->text)->toBe("Hello! I'm an AI assistant. How can I help you today?"); + }); + + it('handles responses with missing id and model fields', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/generate-text-with-missing-meta'); + + $response = Prism::text() + ->using(Provider::Z, 'z-model') + ->withPrompt('Who are you?') + ->asText(); + + expect($response)->toBeInstanceOf(TextResponse::class); + expect($response->meta->id)->toBe(''); + expect($response->meta->model)->toBe('z-model'); + expect($response->text)->toContain("Hello! I'm an AI assistant"); + expect($response->finishReason)->toBe(FinishReason::Stop); + }); + it('can generate text using multiple tools and multiple steps', function (): void { FixtureResponse::fakeResponseSequence('chat/completions', 'z/generate-text-with-multiple-tools'); @@ -103,6 +137,16 @@ ->and($response->text)->toBe( "\nBased on the information I gathered:\n\n**Tigers Game Time:** The Tigers game today in Detroit is at 3:00 PM.\n\n**Weather and Coat Recommendation:** The weather will be 45° and cold. Yes, you should definitely wear a coat to the game! At 45 degrees, it will be quite chilly, especially if you'll be sitting outdoors for several hours. You might want to consider wearing a warm coat, and possibly dressing in layers with a hat and gloves for extra comfort during the game." ); + + expect($response->steps)->toHaveCount(2); + $secondStep = $response->steps[1]; + expect($secondStep->messages)->toHaveCount(3); + expect($secondStep->messages[0])->toBeInstanceOf(UserMessage::class); + expect($secondStep->messages[1])->toBeInstanceOf(AssistantMessage::class); + expect($secondStep->messages[1]->toolCalls)->toHaveCount(2); + expect($secondStep->messages[1]->toolCalls[0]->name)->toBe('search_games'); + expect($secondStep->messages[1]->toolCalls[1]->name)->toBe('get_weather'); + expect($secondStep->messages[2])->toBeInstanceOf(ToolResultMessage::class); }); }); @@ -263,3 +307,87 @@ })->throws(PrismRateLimitedException::class); }); + +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter') + ->clientExecuted(); + + $response = Prism::text() + ->using(Provider::Z, 'z-model') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + +describe('approval-required tools', function (): void { + it('stops execution when approval-required tool is called (Phase 1)', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/text-with-approval-tool'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::Z, 'z-model') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Delete the file at /tmp/test.txt') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('delete_file'); + expect($response->steps)->toHaveCount(1); + expect($response->steps[0]->toolApprovalRequests)->toHaveCount(1); + expect($response->steps[0]->toolApprovalRequests[0]->toolCallId)->toBe('call_delete_file'); + }); + + it('executes approved tool and continues (Phase 2)', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'z/text-with-approval-phase2'); + + $tool = Tool::as('delete_file') + ->for('Delete a file. Requires user approval.') + ->withStringParameter('path', 'File path to delete') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $response = Prism::text() + ->using(Provider::Z, 'z-model') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withMessages([ + new UserMessage('Delete the file at /tmp/test.txt'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call_delete_file', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'apr_call_delete_file', toolCallId: 'call_delete_file'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'apr_call_delete_file', approved: true), + ]), + ]) + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::Stop); + expect($response->text)->toContain('deleted'); + }); +}); diff --git a/tests/Streaming/Adapters/BroadcastAdapterTest.php b/tests/Streaming/Adapters/BroadcastAdapterTest.php index 85a45c842..434a34241 100644 --- a/tests/Streaming/Adapters/BroadcastAdapterTest.php +++ b/tests/Streaming/Adapters/BroadcastAdapterTest.php @@ -18,6 +18,7 @@ use Prism\Prism\Events\Broadcasting\ThinkingBroadcast; use Prism\Prism\Events\Broadcasting\ThinkingCompleteBroadcast; use Prism\Prism\Events\Broadcasting\ThinkingStartBroadcast; +use Prism\Prism\Events\Broadcasting\ToolApprovalRequestBroadcast; use Prism\Prism\Events\Broadcasting\ToolCallBroadcast; use Prism\Prism\Events\Broadcasting\ToolCallDeltaBroadcast; use Prism\Prism\Events\Broadcasting\ToolResultBroadcast; @@ -34,6 +35,7 @@ use Prism\Prism\Streaming\Events\ThinkingCompleteEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; use Prism\Prism\Streaming\Events\ThinkingStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallDeltaEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; @@ -235,6 +237,7 @@ function createBroadcastEventGenerator(array $events): Generator new ThinkingCompleteEvent('evt-7', 1640995206, 'reasoning-123'), new ToolCallEvent('evt-8', 1640995207, new ToolCall('tool-123', 'search', ['q' => 'test']), 'msg-456'), new ToolResultEvent('evt-9', 1640995208, new ToolResult('tool-123', 'search', ['q' => 'test'], ['result' => 'found']), 'msg-456', true), + new ToolApprovalRequestEvent('evt-8a', 1640995207, new ToolCall('approval-tool-1', 'approve_me', ['action' => 'delete']), 'msg-456', 'approval-req-1'), new ProviderToolEvent('evt-10', 1640995209, 'image_generation_call', 'completed', 'ig-789', ['result' => 'data']), new ErrorEvent('evt-11', 1640995210, 'test_error', 'Test error', true), new StreamEndEvent('evt-12', 1640995211, FinishReason::Stop), @@ -246,8 +249,8 @@ function createBroadcastEventGenerator(array $events): Generator // Should not throw any exceptions ($adapter)(createBroadcastEventGenerator($events)); - // Verify all events were processed and dispatched - expect(true)->toBeTrue(); // Test passes if no exceptions thrown + // Verify ToolApprovalRequestEvent is broadcast correctly + Event::assertDispatched(ToolApprovalRequestBroadcast::class); }); it('maintains event order when broadcasting', function (): void { diff --git a/tests/Streaming/Adapters/DataProtocolAdapterTest.php b/tests/Streaming/Adapters/DataProtocolAdapterTest.php index 0aa91bfa6..17b464b32 100644 --- a/tests/Streaming/Adapters/DataProtocolAdapterTest.php +++ b/tests/Streaming/Adapters/DataProtocolAdapterTest.php @@ -17,6 +17,7 @@ use Prism\Prism\Streaming\Events\ThinkingCompleteEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; use Prism\Prism\Streaming\Events\ThinkingStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Prism\Prism\Text\PendingRequest; @@ -96,6 +97,7 @@ function createThrowingGenerator(array $eventsBeforeError, Throwable $exception) new ThinkingEvent('evt-4', 1640995203, 'Thinking...', 'reasoning-123'), new ToolCallEvent('evt-5', 1640995204, new ToolCall('tool-123', 'search', ['q' => 'test']), 'msg-456'), new ToolResultEvent('evt-6', 1640995205, new ToolResult('tool-123', 'search', ['q' => 'test'], ['result' => 'found']), 'msg-456', true), + new ToolApprovalRequestEvent('evt-5a', 1640995204, new ToolCall('approval-1', 'approve_action', ['action' => 'confirm']), 'msg-456', 'approval-req-1'), new ErrorEvent('evt-7', 1640995206, 'test_error', 'Test error', true), new StreamEndEvent('evt-8', 1640995207, FinishReason::Stop), ]; @@ -530,6 +532,144 @@ function createThrowingGenerator(array $eventsBeforeError, Throwable $exception) } }); +it('formats tool-approval-request events for Data Protocol', function (): void { + $events = [ + new ToolApprovalRequestEvent( + 'evt-1', + 1640995200, + new ToolCall('tool-approval-123', 'delete_file', ['path' => '/tmp/test.txt']), + 'msg-456', + 'approval-ulid-456', + ), + ]; + + $adapter = new DataProtocolAdapter; + $response = ($adapter)(createDataEventGenerator($events)); + $callback = $response->getCallback(); + + $outputBuffer = fopen('php://memory', 'r+'); + ob_start(function ($buffer) use ($outputBuffer): string { + fwrite($outputBuffer, $buffer); + + return ''; + }); + + try { + $callback(); + ob_end_flush(); + + rewind($outputBuffer); + $capturedOutput = stream_get_contents($outputBuffer); + + expect($capturedOutput)->toContain('data: {"type":"tool-approval-request"'); + expect($capturedOutput)->toContain('"approvalId":"approval-ulid-456"'); + expect($capturedOutput)->toContain('"toolCallId":"tool-approval-123"'); + + $lines = explode("\n", trim($capturedOutput)); + $dataLines = array_filter($lines, fn (string $line): bool => str_starts_with($line, 'data: ') && $line !== 'data: [DONE]'); + $approvalLine = array_values($dataLines)[0]; + $json = json_decode(substr($approvalLine, 6), true); + + expect($json['type'])->toBe('tool-approval-request'); + expect($json['approvalId'])->toBe('approval-ulid-456'); + expect($json['toolCallId'])->toBe('tool-approval-123'); + } finally { + fclose($outputBuffer); + } +}); + +it('uses responseMessageId in start event when set', function (): void { + $events = [ + new StreamStartEvent('evt-1', 1640995200, 'claude-3', 'anthropic'), + new TextDeltaEvent('evt-2', 1640995201, 'Hello', 'msg-456'), + new StreamEndEvent('evt-3', 1640995202, FinishReason::Stop), + ]; + + $adapter = new DataProtocolAdapter('existing-msg-id'); + $response = ($adapter)(createDataEventGenerator($events)); + $callback = $response->getCallback(); + + $outputBuffer = fopen('php://memory', 'r+'); + ob_start(function ($buffer) use ($outputBuffer): string { + fwrite($outputBuffer, $buffer); + + return ''; + }); + + try { + $callback(); + ob_end_flush(); + + rewind($outputBuffer); + $capturedOutput = stream_get_contents($outputBuffer); + + expect($capturedOutput)->toContain('"messageId":"existing-msg-id"'); + expect($capturedOutput)->not->toContain('"messageId":"evt-1"'); + } finally { + fclose($outputBuffer); + } +}); + +it('uses provider-generated messageId in start event when responseMessageId is not set', function (): void { + $events = [ + new StreamStartEvent('evt-1', 1640995200, 'claude-3', 'anthropic'), + new StreamEndEvent('evt-2', 1640995201, FinishReason::Stop), + ]; + + $adapter = new DataProtocolAdapter; + $response = ($adapter)(createDataEventGenerator($events)); + $callback = $response->getCallback(); + + $outputBuffer = fopen('php://memory', 'r+'); + ob_start(function ($buffer) use ($outputBuffer): string { + fwrite($outputBuffer, $buffer); + + return ''; + }); + + try { + $callback(); + ob_end_flush(); + + rewind($outputBuffer); + $capturedOutput = stream_get_contents($outputBuffer); + + expect($capturedOutput)->toContain('"messageId":"evt-1"'); + } finally { + fclose($outputBuffer); + } +}); + +it('uses provider-generated messageId when responseMessageId is null', function (): void { + $events = [ + new StreamStartEvent('evt-1', 1640995200, 'claude-3', 'anthropic'), + new StreamEndEvent('evt-2', 1640995201, FinishReason::Stop), + ]; + + $adapter = new DataProtocolAdapter; + $response = ($adapter)(createDataEventGenerator($events)); + $callback = $response->getCallback(); + + $outputBuffer = fopen('php://memory', 'r+'); + ob_start(function ($buffer) use ($outputBuffer): string { + fwrite($outputBuffer, $buffer); + + return ''; + }); + + try { + $callback(); + ob_end_flush(); + + rewind($outputBuffer); + $capturedOutput = stream_get_contents($outputBuffer); + + expect($capturedOutput)->toContain('"messageId":"evt-1"'); + } finally { + fclose($outputBuffer); + } +}); + it('formats artifact events using data- prefix convention', function (): void { $artifact = new Artifact( data: 'iVBORw0KGgo=', diff --git a/tests/Streaming/StreamCollectorTest.php b/tests/Streaming/StreamCollectorTest.php index 9bbb08f1d..7bca669c1 100644 --- a/tests/Streaming/StreamCollectorTest.php +++ b/tests/Streaming/StreamCollectorTest.php @@ -9,6 +9,7 @@ use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\TextStartEvent; +use Prism\Prism\Streaming\Events\ToolApprovalRequestEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; use Prism\Prism\Streaming\Events\ToolResultEvent; use Prism\Prism\Streaming\StreamCollector; @@ -290,6 +291,115 @@ function ($request, Collection $collected) use (&$messages): void { expect($messages->first()->toolCalls)->toHaveCount(1); }); +it('collects tool approval requests into assistant message', function (): void { + $messages = null; + + $toolCall = new ToolCall('call-delete-1', 'delete_file', ['path' => '/tmp/foo.txt']); + + $events = [ + new TextStartEvent('evt-1', 1640995200, 'msg-1'), + new ToolCallEvent('evt-2', 1640995201, $toolCall, 'msg-1'), + new ToolApprovalRequestEvent('evt-3', 1640995202, $toolCall, 'msg-1', 'approval-delete-1'), + new StreamEndEvent('evt-4', 1640995203, FinishReason::ToolCalls), + ]; + + $collector = new StreamCollector( + createCollectorEventGenerator($events), + null, + function ($request, Collection $collected) use (&$messages): void { + $messages = $collected; + } + ); + + iterator_to_array($collector->collect()); + + expect($messages)->toHaveCount(1); + expect($messages->first())->toBeInstanceOf(AssistantMessage::class); + expect($messages->first()->toolCalls)->toHaveCount(1); + expect($messages->first()->toolApprovalRequests)->toHaveCount(1); + expect($messages->first()->toolApprovalRequests[0]->approvalId)->toBe('approval-delete-1'); + expect($messages->first()->toolApprovalRequests[0]->toolCallId)->toBe('call-delete-1'); +}); + +it('Phase 2: collects ToolResultEvents from approval resolution before LLM continuation events', function (): void { + $messages = null; + + $toolResult1 = new ToolResult('call-1', 'delete_file', ['path' => '/tmp/a.txt'], 'Deleted: /tmp/a.txt'); + $toolResult2 = new ToolResult('call-2', 'delete_file', ['path' => '/tmp/b.txt'], 'Deleted: /tmp/b.txt'); + + // Simulates Phase 2 stream: ToolResultEvents from approval resolution first, then LLM continuation + $events = [ + new ToolResultEvent('evt-1', 1640995200, $toolResult1, 'msg-1', true), + new ToolResultEvent('evt-2', 1640995201, $toolResult2, 'msg-1', true), + new StreamStartEvent('evt-3', 1640995202, 'gpt-4', 'openai'), + new TextStartEvent('evt-4', 1640995203, 'msg-2'), + new TextDeltaEvent('evt-5', 1640995204, 'Files deleted successfully.', 'msg-2'), + new StreamEndEvent('evt-6', 1640995205, FinishReason::Stop), + ]; + + $collector = new StreamCollector( + createCollectorEventGenerator($events), + null, + function ($request, Collection $collected) use (&$messages): void { + $messages = $collected; + } + ); + + $yieldedEvents = iterator_to_array($collector->collect()); + + // Events pass through in order - ToolResultEvents before LLM events + expect($yieldedEvents)->toHaveCount(6) + ->and($yieldedEvents[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($yieldedEvents[1])->toBeInstanceOf(ToolResultEvent::class) + ->and($yieldedEvents[2])->toBeInstanceOf(StreamStartEvent::class) + ->and($yieldedEvents[5])->toBeInstanceOf(StreamEndEvent::class); + + // ToolResultMessage built from Phase 2 events before the assistant response + expect($messages)->toHaveCount(2) + ->and($messages[0])->toBeInstanceOf(ToolResultMessage::class) + ->and($messages[0]->toolResults)->toHaveCount(2) + ->and($messages[0]->toolResults[0]->result)->toBe('Deleted: /tmp/a.txt') + ->and($messages[0]->toolResults[1]->result)->toBe('Deleted: /tmp/b.txt') + ->and($messages[1])->toBeInstanceOf(AssistantMessage::class) + ->and($messages[1]->content)->toBe('Files deleted successfully.'); +}); + +it('collects tool approval requests after server-executed tool results', function (): void { + $messages = null; + + $searchCall = new ToolCall('call-search-1', 'search', ['q' => 'test']); + $deleteCall = new ToolCall('call-delete-1', 'delete_file', ['path' => '/tmp/foo.txt']); + $searchResult = new ToolResult('call-search-1', 'search', ['q' => 'test'], ['result' => 'found']); + + $events = [ + new TextStartEvent('evt-1', 1640995200, 'msg-1'), + new ToolCallEvent('evt-2', 1640995201, $searchCall, 'msg-1'), + new ToolCallEvent('evt-3', 1640995202, $deleteCall, 'msg-1'), + new ToolResultEvent('evt-4', 1640995203, $searchResult, 'msg-1', true), + new ToolApprovalRequestEvent('evt-5', 1640995204, $deleteCall, 'msg-1', 'approval-delete-1'), + new StreamEndEvent('evt-6', 1640995205, FinishReason::ToolCalls), + ]; + + $collector = new StreamCollector( + createCollectorEventGenerator($events), + null, + function ($request, Collection $collected) use (&$messages): void { + $messages = $collected; + } + ); + + iterator_to_array($collector->collect()); + + expect($messages)->toHaveCount(2); + expect($messages[0])->toBeInstanceOf(AssistantMessage::class); + expect($messages[0]->toolCalls)->toHaveCount(2); + expect($messages[0]->toolApprovalRequests)->toHaveCount(1); + expect($messages[0]->toolApprovalRequests[0]->toolCallId)->toBe('call-delete-1'); + expect($messages[1])->toBeInstanceOf(ToolResultMessage::class); + expect($messages[1]->toolResults)->toHaveCount(1); + expect($messages[1]->toolResults[0]->toolCallId)->toBe('call-search-1'); +}); + it('handles multiple tool results in single message', function (): void { $messages = null; diff --git a/tests/Structured/PendingRequestTest.php b/tests/Structured/PendingRequestTest.php index 316f7df78..5d3c5b996 100644 --- a/tests/Structured/PendingRequestTest.php +++ b/tests/Structured/PendingRequestTest.php @@ -6,6 +6,7 @@ use Prism\Prism\Enums\Provider; use Prism\Prism\Enums\StructuredMode; use Prism\Prism\Exceptions\PrismException; +use Prism\Prism\Facades\Tool; use Prism\Prism\Schema\StringSchema; use Prism\Prism\Structured\PendingRequest; use Prism\Prism\Structured\Request; @@ -26,6 +27,18 @@ ->toThrow(PrismException::class, 'A schema is required for structured output'); }); +test('it throws exception when tool has no handler and is not client-executed', function (): void { + $tool = Tool::as('broken_tool') + ->for('A tool that forgot using()'); + + $this->pendingRequest + ->using(Provider::OpenAI, 'gpt-4') + ->withSchema(new StringSchema('test', 'test description')) + ->withTools([$tool]) + ->withPrompt('test') + ->toRequest(); +})->throws(PrismException::class, 'Tool (broken_tool) has no handler defined. Use using() to set a handler or clientExecuted() to mark it as client-executed.'); + test('it cannot have both prompt and messages', function (): void { $this->pendingRequest ->using(Provider::OpenAI, 'gpt-4') diff --git a/tests/Text/PendingRequestTest.php b/tests/Text/PendingRequestTest.php index 0b4c4fb13..de450772f 100644 --- a/tests/Text/PendingRequestTest.php +++ b/tests/Text/PendingRequestTest.php @@ -374,6 +374,17 @@ enum TestModel: string ); }); +test('it throws exception when tool has no handler and is not client-executed', function (): void { + $tool = Tool::as('broken_tool') + ->for('A tool that forgot using()'); + + $this->pendingRequest + ->using(Provider::OpenAI, 'gpt-4') + ->withTools([$tool]) + ->withPrompt('test') + ->toRequest(); +})->throws(PrismException::class, 'Tool (broken_tool) has no handler defined. Use using() to set a handler or clientExecuted() to mark it as client-executed.'); + test('it throws exception when using both prompt and messages', function (): void { $this->pendingRequest ->using(Provider::OpenAI, 'gpt-4') diff --git a/tests/ToolApprovalTest.php b/tests/ToolApprovalTest.php new file mode 100644 index 000000000..09b831fc7 --- /dev/null +++ b/tests/ToolApprovalTest.php @@ -0,0 +1,1351 @@ +callTools($tools, $toolCalls, $hasPendingToolCalls); + } + + public function stream(array $tools, array $toolCalls, string $messageId, array &$toolResults, bool &$hasPendingToolCalls = false): Generator + { + return $this->callToolsAndYieldEvents($tools, $toolCalls, $messageId, $toolResults, $hasPendingToolCalls); + } + + public function resolve(Request $request): void + { + $this->resolveToolApprovals($request); + } + + public function resolveStream(Request $request, string $messageId, ?StreamState $state = null): Generator + { + return $this->resolveToolApprovalsAndYieldEvents($request, $messageId, $state); + } +} + +function getResolvedToolResults(Request $request): array +{ + foreach (array_reverse($request->messages()) as $message) { + if ($message instanceof ToolResultMessage) { + return $message->toolResults; + } + } + + return []; +} + +function createTextRequest(array $messages = [], array $tools = []): Request +{ + return new Request( + model: 'test-model', + providerKey: 'test', + systemPrompts: [], + prompt: null, + messages: $messages, + maxSteps: 5, + maxTokens: null, + temperature: null, + topP: null, + tools: $tools, + clientOptions: [], + clientRetry: [0], + toolChoice: ToolChoice::Auto, + ); +} + +describe('Tool::requiresApproval()', function (): void { + it('defaults to not requiring approval', function (): void { + $tool = (new Tool) + ->as('test') + ->for('Test tool') + ->using(fn (): string => 'result'); + + expect($tool->needsApproval())->toBeFalse(); + }); + + it('can be marked as requiring approval with static true', function (): void { + $tool = (new Tool) + ->as('test') + ->for('Test tool') + ->using(fn (): string => 'result') + ->requiresApproval(); + + expect($tool->needsApproval())->toBeTrue(); + }); + + it('can be marked as requiring approval with explicit true', function (): void { + $tool = (new Tool) + ->as('test') + ->for('Test tool') + ->using(fn (): string => 'result') + ->requiresApproval(true); + + expect($tool->needsApproval())->toBeTrue(); + }); + + it('can be set to not require approval with false', function (): void { + $tool = (new Tool) + ->as('test') + ->for('Test tool') + ->using(fn (): string => 'result') + ->requiresApproval(false); + + expect($tool->needsApproval())->toBeFalse(); + }); + + it('supports dynamic approval via closure', function (): void { + $tool = (new Tool) + ->as('transfer') + ->for('Transfer money') + ->withNumberParameter('amount', 'Amount') + ->using(fn (float $amount): string => "Transferred {$amount}") + ->requiresApproval(fn (array $args): bool => $args['amount'] > 1000); + + expect($tool->needsApproval(['amount' => 500]))->toBeFalse(); + expect($tool->needsApproval(['amount' => 1500]))->toBeTrue(); + }); + + it('hasApprovalConfigured returns true for static or closure without invoking closure', function (): void { + $staticTool = (new Tool)->as('a')->for('A')->using(fn (): string => '')->requiresApproval(); + expect($staticTool->hasApprovalConfigured())->toBeTrue(); + + $closureTool = (new Tool)->as('b')->for('B')->using(fn (): string => '') + ->requiresApproval(fn (array $args): bool => $args['x'] > 0); + expect($closureTool->hasApprovalConfigured())->toBeTrue(); + + $disabledTool = (new Tool)->as('c')->for('C')->using(fn (): string => '')->requiresApproval(false); + expect($disabledTool->hasApprovalConfigured())->toBeFalse(); + }); +}); + +describe('Phase 1: filterServerExecutedToolCalls with approval tools', function (): void { + it('skips approval-required tools and sets pending flag', function (): void { + $normalTool = (new Tool) + ->as('normal_tool') + ->for('A normal tool') + ->using(fn (): string => 'result'); + + $approvalTool = (new Tool) + ->as('approval_tool') + ->for('Needs approval') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $toolCalls = [ + new ToolCall(id: 'call-1', name: 'normal_tool', arguments: []), + new ToolCall(id: 'call-2', name: 'approval_tool', arguments: []), + ]; + + $handler = new ToolApprovalTestHandler; + $hasPendingToolCalls = false; + $results = $handler->execute([$normalTool, $approvalTool], $toolCalls, $hasPendingToolCalls); + + expect($results)->toHaveCount(1) + ->and($results[0]->toolName)->toBe('normal_tool') + ->and($hasPendingToolCalls)->toBeTrue(); + }); + + it('emits ToolApprovalRequestEvent in streaming for approval-required tools', function (): void { + $approvalTool = (new Tool) + ->as('dangerous_tool') + ->for('Dangerous operation') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $toolCalls = [ + new ToolCall(id: 'call-1', name: 'dangerous_tool', arguments: ['action' => 'delete']), + ]; + + $handler = new ToolApprovalTestHandler; + $toolResults = []; + $hasPendingToolCalls = false; + $events = []; + + foreach ($handler->stream([$approvalTool], $toolCalls, 'msg-123', $toolResults, $hasPendingToolCalls) as $event) { + $events[] = $event; + } + + expect($events)->toHaveCount(1) + ->and($events[0])->toBeInstanceOf(ToolApprovalRequestEvent::class) + ->and($events[0]->toolCall->id)->toBe('call-1') + ->and($events[0]->toolCall->name)->toBe('dangerous_tool') + ->and($events[0]->messageId)->toBe('msg-123') + ->and($hasPendingToolCalls)->toBeTrue() + ->and($toolResults)->toBeEmpty(); + }); + + it('yields ToolApprovalRequestEvent after server-executed tool results in streaming', function (): void { + $normalTool = (new Tool) + ->as('normal') + ->for('Normal tool') + ->using(fn (): string => 'normal result'); + + $approvalTool = (new Tool) + ->as('approval') + ->for('Approval tool') + ->using(fn (): string => 'should not run') + ->requiresApproval(); + + $toolCalls = [ + new ToolCall(id: 'call-1', name: 'normal', arguments: []), + new ToolCall(id: 'call-2', name: 'approval', arguments: []), + ]; + + $handler = new ToolApprovalTestHandler; + $toolResults = []; + $hasPendingToolCalls = false; + $events = []; + + foreach ($handler->stream([$normalTool, $approvalTool], $toolCalls, 'msg-123', $toolResults, $hasPendingToolCalls) as $event) { + $events[] = $event; + } + + expect($events)->toHaveCount(2) + ->and($events[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[0]->toolResult->toolCallId)->toBe('call-1') + ->and($events[0]->toolResult->result)->toBe('normal result') + ->and($events[1])->toBeInstanceOf(ToolApprovalRequestEvent::class) + ->and($events[1]->toolCall->id)->toBe('call-2') + ->and($hasPendingToolCalls)->toBeTrue() + ->and($toolResults)->toHaveCount(1); + }); + + it('handles mixed tools: normal, client-executed, and approval-required', function (): void { + $normalTool = (new Tool) + ->as('normal') + ->for('Normal tool') + ->using(fn (): string => 'normal result'); + + $clientTool = (new Tool) + ->as('client') + ->for('Client tool') + ->clientExecuted(); + + $approvalTool = (new Tool) + ->as('approval') + ->for('Approval tool') + ->using(fn (): string => 'should not run') + ->requiresApproval(); + + $toolCalls = [ + new ToolCall(id: 'call-1', name: 'normal', arguments: []), + new ToolCall(id: 'call-2', name: 'client', arguments: []), + new ToolCall(id: 'call-3', name: 'approval', arguments: []), + ]; + + $handler = new ToolApprovalTestHandler; + $hasPendingToolCalls = false; + $results = $handler->execute([$normalTool, $clientTool, $approvalTool], $toolCalls, $hasPendingToolCalls); + + expect($results)->toHaveCount(1) + ->and($results[0]->toolName)->toBe('normal') + ->and($hasPendingToolCalls)->toBeTrue(); + }); + + it('skips tool with dynamic approval only when closure returns true', function (): void { + $tool = (new Tool) + ->as('transfer') + ->for('Transfer money') + ->withNumberParameter('amount', 'Amount') + ->using(fn (float $amount): string => "Transferred {$amount}") + ->requiresApproval(fn (array $args): bool => $args['amount'] > 1000); + + $smallTransfer = new ToolCall(id: 'call-1', name: 'transfer', arguments: ['amount' => 500]); + $largeTransfer = new ToolCall(id: 'call-2', name: 'transfer', arguments: ['amount' => 2000]); + + $handler = new ToolApprovalTestHandler; + + $hasPending = false; + $results = $handler->execute([$tool], [$smallTransfer], $hasPending); + expect($results)->toHaveCount(1) + ->and($results[0]->result)->toBe('Transferred 500') + ->and($hasPending)->toBeFalse(); + + $hasPending = false; + $results = $handler->execute([$tool], [$largeTransfer], $hasPending); + expect($results)->toBeEmpty() + ->and($hasPending)->toBeTrue(); + }); +}); + +describe('Phase 2: resolveToolApprovals', function (): void { + it('returns empty when no ToolResultMessage with toolApprovalResponses exists', function (): void { + $tool = (new Tool) + ->as('test') + ->for('Test') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $request = createTextRequest( + messages: [new UserMessage('hello')], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + expect(getResolvedToolResults($request))->toBeEmpty(); + }); + + it('executes approved tools', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete the file'), + new AssistantMessage( + content: 'I will delete the file.', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->toolName)->toBe('delete_file') + ->and($results[0]->result)->toBe('Deleted: /tmp/test.txt') + ->and($results[0]->toolCallId)->toBe('call-1'); + + $messages = $request->messages(); + $lastMessage = end($messages); + expect($lastMessage)->toBeInstanceOf(ToolResultMessage::class); + }); + + it('creates denial result for denied tools', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete the file'), + new AssistantMessage( + content: 'I will delete the file.', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: false, reason: 'Too dangerous'), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->toolName)->toBe('delete_file') + ->and($results[0]->result)->toBe('Too dangerous') + ->and($results[0]->toolCallId)->toBe('call-1'); + }); + + it('adds denial when no approval response provided for approval-required tool', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete the file'), + new AssistantMessage( + content: 'I will delete the file.', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], []), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->result)->toBe('No approval response provided') + ->and($results[0]->toolCallId)->toBe('call-1'); + }); + + it('replaces ToolResultMessage with toolApprovalResponses with merged ToolResultMessage', function (): void { + $tool = (new Tool) + ->as('test_tool') + ->for('Test') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('test'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'test_tool', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $toolResultMessages = collect($request->messages())->filter(fn ($m): bool => $m instanceof ToolResultMessage); + expect($toolResultMessages)->toHaveCount(1); + + $toolResultMessage = $toolResultMessages->first(); + expect($toolResultMessage->toolResults)->toHaveCount(1) + ->and($toolResultMessage->toolResults[0]->result)->toBe('result') + ->and($toolResultMessage->toolApprovalResponses)->toHaveCount(1) + ->and($toolResultMessage->toolApprovalResponses[0]->approvalId)->toBe('approval-1') + ->and($toolResultMessage->toolApprovalResponses[0]->approved)->toBeTrue(); + }); + + it('yields ToolResultEvent for each resolved tool in streaming', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/a.txt']), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/b.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + new ToolApprovalRequest(approvalId: 'approval-2', toolCallId: 'call-2'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + new ToolApprovalResponse(approvalId: 'approval-2', approved: false, reason: 'Keep this file'), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $events = []; + + foreach ($handler->resolveStream($request, 'msg-456') as $event) { + $events[] = $event; + } + + expect($events)->toHaveCount(2) + ->and($events[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[0]->toolResult->toolCallId)->toBe('call-1') + ->and($events[0]->toolResult->result)->toBe('Deleted: /tmp/a.txt') + ->and($events[0]->success)->toBeTrue() + ->and($events[1])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[1]->toolResult->toolCallId)->toBe('call-2') + ->and($events[1]->toolResult->result)->toBe('Keep this file') + ->and($events[1]->success)->toBeFalse(); + }); + + it('yields ToolResultEvents for approved tools before request is updated for LLM continuation', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete files'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/a.txt']), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/b.txt']), + new ToolCall(id: 'call-3', name: 'delete_file', arguments: ['path' => '/tmp/c.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + new ToolApprovalRequest(approvalId: 'approval-2', toolCallId: 'call-2'), + new ToolApprovalRequest(approvalId: 'approval-3', toolCallId: 'call-3'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + new ToolApprovalResponse(approvalId: 'approval-2', approved: true), + new ToolApprovalResponse(approvalId: 'approval-3', approved: false, reason: 'Skip this'), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $events = iterator_to_array($handler->resolveStream($request, 'msg-789')); + + // All ToolResultEvents must be yielded before generator completes (i.e. before LLM would continue) + expect($events)->toHaveCount(3) + ->and($events[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[0]->toolResult->toolCallId)->toBe('call-1') + ->and($events[0]->toolResult->result)->toBe('Deleted: /tmp/a.txt') + ->and($events[1])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[1]->toolResult->toolCallId)->toBe('call-2') + ->and($events[1]->toolResult->result)->toBe('Deleted: /tmp/b.txt') + ->and($events[2])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[2]->toolResult->toolCallId)->toBe('call-3') + ->and($events[2]->toolResult->result)->toBe('Skip this') + ->and($events[2]->success)->toBeFalse(); + + // Request is only updated with merged ToolResultMessage after all events are yielded + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(3) + ->and($results[0]->result)->toBe('Deleted: /tmp/a.txt') + ->and($results[1]->result)->toBe('Deleted: /tmp/b.txt') + ->and($results[2]->result)->toBe('Skip this'); + }); + + it('yields ToolResultEvents for approved tools in order before generator completes', function (): void { + $approvalTool = (new Tool) + ->as('transfer') + ->for('Transfer money') + ->withNumberParameter('amount', 'Amount') + ->using(fn (float $amount): string => "Transferred {$amount}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('transfer money'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'transfer', arguments: ['amount' => 100]), + new ToolCall(id: 'call-2', name: 'transfer', arguments: ['amount' => 500]), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + new ToolApprovalRequest(approvalId: 'approval-2', toolCallId: 'call-2'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + new ToolApprovalResponse(approvalId: 'approval-2', approved: true), + ]), + ], + tools: [$approvalTool], + ); + + $handler = new ToolApprovalTestHandler; + $events = iterator_to_array($handler->resolveStream($request, 'msg-stream')); + + // All ToolResultEvents must appear before the generator is exhausted (simulating "before LLM continues") + expect($events)->toHaveCount(2) + ->and($events[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[0]->toolResult->result)->toBe('Transferred 100') + ->and($events[0]->success)->toBeTrue() + ->and($events[1])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[1]->toolResult->result)->toBe('Transferred 500') + ->and($events[1]->success)->toBeTrue(); + }); + + it('merges resolved tool results with existing client-executed results', function (): void { + $approvalTool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('search and delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'search', arguments: []), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-2', toolCallId: 'call-2'), + ], + ), + new ToolResultMessage( + toolResults: [ + new ToolResult(toolCallId: 'call-1', toolName: 'search', args: [], result: 'client search result'), + ], + toolApprovalResponses: [ + new ToolApprovalResponse(approvalId: 'approval-2', approved: true), + ], + ), + ], + tools: [ + (new Tool)->as('search')->for('Search')->using(fn (): string => '')->clientExecuted(), + $approvalTool, + ], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $toolResultMessage = collect($request->messages())->first(fn ($m): bool => $m instanceof ToolResultMessage); + expect($toolResultMessage->toolResults)->toHaveCount(2) + ->and($toolResultMessage->toolResults[0]->toolCallId)->toBe('call-1') + ->and($toolResultMessage->toolResults[0]->result)->toBe('client search result') + ->and($toolResultMessage->toolResults[1]->toolCallId)->toBe('call-2') + ->and($toolResultMessage->toolResults[1]->result)->toBe('Deleted: /tmp/test.txt'); + }); + + it('skips approval-required tools that already have a result in the tool message', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/a.txt']), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/b.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + new ToolApprovalRequest(approvalId: 'approval-2', toolCallId: 'call-2'), + ], + ), + new ToolResultMessage( + toolResults: [ + new ToolResult(toolCallId: 'call-1', toolName: 'delete_file', args: ['path' => '/tmp/a.txt'], result: 'Already executed'), + ], + toolApprovalResponses: [ + new ToolApprovalResponse(approvalId: 'approval-2', approved: false, reason: 'User denied'), + ], + ), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(2) + ->and($results[0]->toolCallId)->toBe('call-1') + ->and($results[0]->result)->toBe('Already executed') + ->and($results[1]->toolCallId)->toBe('call-2') + ->and($results[1]->result)->toBe('User denied'); + }); + + it('only resolves tool calls with approval responses, skipping others', function (): void { + $normalTool = (new Tool) + ->as('search') + ->for('Search the web') + ->using(fn (): string => 'search results'); + + $approvalTool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('search and delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'search', arguments: []), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/test.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-2', toolCallId: 'call-2'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-2', approved: true), + ]), + ], + tools: [$normalTool, $approvalTool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->toolName)->toBe('delete_file') + ->and($results[0]->result)->toBe('Deleted: /tmp/test.txt'); + }); + + it('adds denial for last assistant when no approval message follows it', function (): void { + $tool = (new Tool) + ->as('test_tool') + ->for('Test') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $messages = [ + new UserMessage('first request'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-old', name: 'test_tool', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-old', toolCallId: 'call-old'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-old', approved: true), + ]), + new ToolResultMessage([ + new ToolResult(toolCallId: 'call-old', toolName: 'test_tool', args: [], result: 'done'), + ]), + new UserMessage('second request'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-new', name: 'test_tool', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-new', toolCallId: 'call-new'), + ], + ), + ]; + + $request = createTextRequest( + messages: $messages, + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->toolCallId)->toBe('call-new') + ->and($results[0]->result)->toBe('No approval response provided'); + }); + + it('resolves approval message that comes after the last assistant message', function (): void { + $tool = (new Tool) + ->as('test_tool') + ->for('Test') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('first request'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-old', name: 'test_tool', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-old', toolCallId: 'call-old'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-old', approved: true), + ]), + new ToolResultMessage([ + new ToolResult(toolCallId: 'call-old', toolName: 'test_tool', args: [], result: 'done'), + ]), + new UserMessage('second request'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-new', name: 'test_tool', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-new', toolCallId: 'call-new'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-new', approved: true), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->toolCallId)->toBe('call-new') + ->and($results[0]->result)->toBe('result'); + }); + + it('handles denial without explicit reason using default message', function (): void { + $tool = (new Tool) + ->as('test_tool') + ->for('Test') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('test'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'test_tool', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: false), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->result)->toBe('User denied tool execution'); + }); + + it('yields StreamStartEvent before first ToolResultEvent when StreamState is provided', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/a.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + ]), + ], + tools: [$tool], + ); + + $state = new StreamState; + $handler = new ToolApprovalTestHandler; + $events = iterator_to_array($handler->resolveStream($request, 'msg-start', $state)); + + expect($events)->toHaveCount(2) + ->and($events[0])->toBeInstanceOf(StreamStartEvent::class) + ->and($events[0]->model)->toBe('test-model') + ->and($events[0]->provider)->toBe('test') + ->and($events[1])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[1]->toolResult->toolCallId)->toBe('call-1'); + + expect($state->hasStreamStarted())->toBeTrue(); + }); + + it('does not yield StreamStartEvent when stream has already started', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/a.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + ]), + ], + tools: [$tool], + ); + + $state = new StreamState; + $state->markStreamStarted(); + + $handler = new ToolApprovalTestHandler; + $events = iterator_to_array($handler->resolveStream($request, 'msg-nostart', $state)); + + expect($events)->toHaveCount(1) + ->and($events[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[0]->toolResult->toolCallId)->toBe('call-1'); + }); + + it('does not yield StreamStartEvent when no tool results are produced', function (): void { + $tool = (new Tool) + ->as('search') + ->for('Search') + ->withStringParameter('query', 'Query') + ->using(fn (string $query): string => "Results for: {$query}"); + + $request = createTextRequest( + messages: [ + new UserMessage('search'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'search', arguments: ['query' => 'test']), + ], + ), + new ToolResultMessage([ + new ToolResult(toolCallId: 'call-1', toolName: 'search', args: ['query' => 'test'], result: 'Results'), + ]), + ], + tools: [$tool], + ); + + $state = new StreamState; + $handler = new ToolApprovalTestHandler; + $events = iterator_to_array($handler->resolveStream($request, 'msg-none', $state)); + + expect($events)->toHaveCount(0); + expect($state->hasStreamStarted())->toBeFalse(); + }); + + it('yields StreamStartEvent only once even with multiple tool results', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/a.txt']), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/b.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + new ToolApprovalRequest(approvalId: 'approval-2', toolCallId: 'call-2'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + new ToolApprovalResponse(approvalId: 'approval-2', approved: false, reason: 'Keep this'), + ]), + ], + tools: [$tool], + ); + + $state = new StreamState; + $handler = new ToolApprovalTestHandler; + $events = iterator_to_array($handler->resolveStream($request, 'msg-multi', $state)); + + expect($events)->toHaveCount(3) + ->and($events[0])->toBeInstanceOf(StreamStartEvent::class) + ->and($events[1])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[1]->toolResult->toolCallId)->toBe('call-1') + ->and($events[2])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[2]->toolResult->toolCallId)->toBe('call-2'); + + expect($state->hasStreamStarted())->toBeTrue(); + }); + + it('does not yield StreamStartEvent when no StreamState is provided', function (): void { + $tool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('delete'), + new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete_file', arguments: ['path' => '/tmp/a.txt']), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $events = iterator_to_array($handler->resolveStream($request, 'msg-nostate')); + + expect($events)->toHaveCount(1) + ->and($events[0])->toBeInstanceOf(ToolResultEvent::class) + ->and($events[0]->toolResult->toolCallId)->toBe('call-1'); + }); + + it('returns empty when message history has no AssistantMessage', function (): void { + $tool = (new Tool) + ->as('test_tool') + ->for('Test') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('hello'), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + ]), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + expect(getResolvedToolResults($request))->toBeEmpty(); + }); + + it('returns empty when ToolResultMessage appears before AssistantMessage', function (): void { + $tool = (new Tool) + ->as('test_tool') + ->for('Test') + ->using(fn (): string => 'result') + ->requiresApproval(); + + $request = createTextRequest( + messages: [ + new UserMessage('hello'), + new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'approval-1', approved: true), + ]), + new AssistantMessage( + content: 'I will do it.', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'test_tool', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ), + ], + tools: [$tool], + ); + + $handler = new ToolApprovalTestHandler; + $handler->resolve($request); + + $results = getResolvedToolResults($request); + expect($results)->toHaveCount(1) + ->and($results[0]->result)->toBe('No approval response provided'); + }); +}); + +describe('Phase 1: concurrent tools with approval', function (): void { + it('executes concurrent tools and skips approval-required tools', function (): void { + $concurrentTool = (new Tool) + ->as('fast_lookup') + ->for('Fast lookup') + ->withStringParameter('query', 'Query') + ->using(fn (string $query): string => "Found: {$query}") + ->concurrent(); + + $approvalTool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $anotherConcurrentTool = (new Tool) + ->as('quick_check') + ->for('Quick check') + ->withStringParameter('item', 'Item') + ->using(fn (string $item): string => "Checked: {$item}") + ->concurrent(); + + $toolCalls = [ + new ToolCall(id: 'call-1', name: 'fast_lookup', arguments: ['query' => 'test']), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/file.txt']), + new ToolCall(id: 'call-3', name: 'quick_check', arguments: ['item' => 'status']), + ]; + + $handler = new ToolApprovalTestHandler; + $hasPendingToolCalls = false; + $results = $handler->execute( + [$concurrentTool, $approvalTool, $anotherConcurrentTool], + $toolCalls, + $hasPendingToolCalls, + ); + + expect($results)->toHaveCount(2) + ->and($results[0]->toolName)->toBe('fast_lookup') + ->and($results[0]->result)->toBe('Found: test') + ->and($results[1]->toolName)->toBe('quick_check') + ->and($results[1]->result)->toBe('Checked: status') + ->and($hasPendingToolCalls)->toBeTrue(); + }); + + it('executes concurrent tools and skips approval-required tools in streaming', function (): void { + $concurrentTool = (new Tool) + ->as('fast_lookup') + ->for('Fast lookup') + ->withStringParameter('query', 'Query') + ->using(fn (string $query): string => "Found: {$query}") + ->concurrent(); + + $approvalTool = (new Tool) + ->as('delete_file') + ->for('Delete a file') + ->withStringParameter('path', 'File path') + ->using(fn (string $path): string => "Deleted: {$path}") + ->requiresApproval(); + + $toolCalls = [ + new ToolCall(id: 'call-1', name: 'fast_lookup', arguments: ['query' => 'test']), + new ToolCall(id: 'call-2', name: 'delete_file', arguments: ['path' => '/tmp/file.txt']), + ]; + + $handler = new ToolApprovalTestHandler; + $toolResults = []; + $hasPendingToolCalls = false; + $events = []; + + foreach ($handler->stream([$concurrentTool, $approvalTool], $toolCalls, 'msg-123', $toolResults, $hasPendingToolCalls) as $event) { + $events[] = $event; + } + + $toolResultEvents = array_filter($events, fn ($e): bool => $e instanceof ToolResultEvent); + $approvalEvents = array_filter($events, fn ($e): bool => $e instanceof ToolApprovalRequestEvent); + + expect($toolResults)->toHaveCount(1) + ->and($toolResults[0]->toolName)->toBe('fast_lookup') + ->and(array_values($toolResultEvents))->toHaveCount(1) + ->and(array_values($approvalEvents))->toHaveCount(1) + ->and($hasPendingToolCalls)->toBeTrue(); + }); +}); + +describe('ToolApprovalRequest value object', function (): void { + it('serializes to array correctly', function (): void { + $request = new ToolApprovalRequest( + approvalId: 'approval-1', + toolCallId: 'call-1', + ); + + expect($request->toArray())->toBe([ + 'approval_id' => 'approval-1', + 'tool_call_id' => 'call-1', + ]); + }); +}); + +describe('ToolApprovalResponse value object', function (): void { + it('serializes to array correctly', function (): void { + $response = new ToolApprovalResponse( + approvalId: 'call-123', + approved: true, + reason: 'User confirmed', + ); + + expect($response->toArray())->toBe([ + 'approval_id' => 'call-123', + 'approved' => true, + 'reason' => 'User confirmed', + ]); + }); +}); + +describe('ToolResultMessage with toolApprovalResponses', function (): void { + it('finds approval responses by approval ID', function (): void { + $message = new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'call-1', approved: true), + new ToolApprovalResponse(approvalId: 'call-2', approved: false), + ]); + + $found = $message->findByApprovalId('call-1'); + expect($found)->not->toBeNull() + ->and($found->approved)->toBeTrue(); + + $found2 = $message->findByApprovalId('call-2'); + expect($found2)->not->toBeNull() + ->and($found2->approved)->toBeFalse(); + + expect($message->findByApprovalId('nonexistent'))->toBeNull(); + }); + + it('serializes tool approval responses in toArray', function (): void { + $message = new ToolResultMessage([], [ + new ToolApprovalResponse(approvalId: 'call-1', approved: true), + ]); + + $array = $message->toArray(); + expect($array['type'])->toBe('tool_result') + ->and($array['tool_approval_responses'])->toHaveCount(1) + ->and($array['tool_approval_responses'][0]['approval_id'])->toBe('call-1') + ->and($array['tool_approval_responses'][0]['approved'])->toBeTrue(); + }); + + it('serializes both tool results and approval responses in toArray', function (): void { + $message = new ToolResultMessage( + toolResults: [ + new ToolResult(toolCallId: 'call-1', toolName: 'delete', args: [], result: 'done'), + ], + toolApprovalResponses: [ + new ToolApprovalResponse(approvalId: 'call-1', approved: true), + ], + ); + + $array = $message->toArray(); + expect($array['tool_results'])->toHaveCount(1) + ->and($array['tool_approval_responses'])->toHaveCount(1) + ->and($array['tool_approval_responses'][0]['approval_id'])->toBe('call-1') + ->and($array['tool_approval_responses'][0]['approved'])->toBeTrue(); + }); +}); + +describe('AssistantMessage with toolApprovalRequests', function (): void { + it('serializes tool approval requests in toArray', function (): void { + $message = new AssistantMessage( + content: '', + toolCalls: [ + new ToolCall(id: 'call-1', name: 'delete', arguments: []), + ], + additionalContent: [], + toolApprovalRequests: [ + new ToolApprovalRequest(approvalId: 'approval-1', toolCallId: 'call-1'), + ], + ); + + $array = $message->toArray(); + expect($array['tool_approval_requests'])->toHaveCount(1) + ->and($array['tool_approval_requests'][0])->toBe([ + 'approval_id' => 'approval-1', + 'tool_call_id' => 'call-1', + ]); + }); +}); + +describe('ToolApprovalRequestEvent', function (): void { + it('has correct event type', function (): void { + $event = new ToolApprovalRequestEvent( + id: 'evt-1', + timestamp: 1234567890, + toolCall: new ToolCall(id: 'call-1', name: 'test_tool', arguments: ['key' => 'value']), + messageId: 'msg-1', + approvalId: 'approval-1', + ); + + expect($event->type())->toBe(StreamEventType::ToolApprovalRequest); + + $array = $event->toArray(); + expect($array['approval_id'])->toBe('approval-1') + ->and($array['tool_name'])->toBe('test_tool') + ->and($array['tool_id'])->toBe('call-1') + ->and($array['arguments'])->toBe(['key' => 'value']) + ->and($array['message_id'])->toBe('msg-1'); + }); +}); diff --git a/tests/ToolTest.php b/tests/ToolTest.php index f7865ec8b..61abf4a67 100644 --- a/tests/ToolTest.php +++ b/tests/ToolTest.php @@ -181,3 +181,15 @@ public function __invoke(string $query): string $searchTool->handle('What time is the event?'); }); + +it('can throw a prism exception when handle is called on a tool without a handler', function (): void { + $tool = (new Tool) + ->as('client_tool') + ->for('A tool without a handler') + ->withParameter(new StringSchema('query', 'the search query')); + + $this->expectException(PrismException::class); + $this->expectExceptionMessage('Tool (client_tool) has no handler defined'); + + $tool->handle('test'); +});