Skip to content

Commit e2ef357

Browse files
GuikingoneGuikingone
committed
feat(platform): add Speech
1 parent cc7b3f8 commit e2ef357

35 files changed

+1391
-3
lines changed

‎.phpactor.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"$schema": "/phpactor.schema.json",
3+
"language_server_phpstan.enabled": true
4+
}

‎demo/tests/Blog/Command/StreamCommandTest.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,13 @@
1616
use Symfony\AI\Agent\AgentInterface;
1717
use Symfony\AI\Platform\Message\MessageBag;
1818
use Symfony\AI\Platform\Metadata\Metadata;
19+
use Symfony\AI\Platform\Result\DeferredResult;
20+
use Symfony\AI\Platform\Result\InMemoryRawResult;
1921
use Symfony\AI\Platform\Result\RawResultInterface;
2022
use Symfony\AI\Platform\Result\ResultInterface;
23+
use Symfony\AI\Platform\Result\TextResult;
24+
use Symfony\AI\Platform\Speech\Speech;
25+
use Symfony\AI\Platform\Test\PlainConverter;
2126
use Symfony\Component\Console\Input\ArrayInput;
2227
use Symfony\Component\Console\Output\BufferedOutput;
2328
use Symfony\Component\Console\Style\SymfonyStyle;
@@ -52,6 +57,15 @@ public function getRawResult(): ?RawResultInterface
5257
public function setRawResult(RawResultInterface $rawResult): void
5358
{
5459
}
60+
61+
public function addSpeech(Speech $speech): void
62+
{
63+
}
64+
65+
public function getSpeech(string $identifier): Speech
66+
{
67+
return new Speech([], new DeferredResult(new PlainConverter(new TextResult('foo')), new InMemoryRawResult()), 'bar');
68+
}
5569
});
5670

5771
$input = new ArrayInput([]);

‎docs/components/platform.rst‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,64 @@ This allows fast and isolated testing of AI-powered features without relying on
532532

533533
This requires `cURL` and the `ext-curl` extension to be installed.
534534

535+
Speech support
536+
~~~~~~~~~~~~~~
537+
538+
Using speech to send messages / receive answers as audio is a common use case when integrating agents and/or chats.
539+
540+
Speech support can be enable using ``Symfony\AI\Platform\Speech\SpeechProviderListener``::
541+
542+
use Symfony\AI\Agent\Agent;
543+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
544+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
545+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
546+
use Symfony\AI\Platform\Message\Message;
547+
use Symfony\AI\Platform\Message\MessageBag;
548+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
549+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
550+
use Symfony\Component\EventDispatcher\EventDispatcher;
551+
552+
$eventDispatcher = new EventDispatcher();
553+
$eventDispatcher->addSubscriber(new SpeechProviderListener([
554+
new ElevenLabsSpeechProvider(PlatformFactory::create(
555+
apiKey: $elevenLabsApiKey,
556+
httpClient: http_client(),
557+
speechConfiguration: new SpeechConfiguration(
558+
ttsModel: 'eleven_multilingual_v2',
559+
ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
560+
sttModel: 'eleven_multilingual_v2'
561+
)),
562+
),
563+
], []));
564+
565+
$platform = OpenAiPlatformFactory::create($openAiApiKey, httpClient: HttpClient::create(), eventDispatcher: $eventDispatcher);
566+
567+
$agent = new Agent($platform, 'gpt-4o');
568+
$answer = $agent->call(new MessageBag(
569+
Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
570+
));
571+
572+
echo $answer->getSpeech('elevenlabs')->asBinary();
573+
574+
When using the bundle, the configuration allows to configure models and voices::
575+
576+
ai:
577+
platform:
578+
elevenlabs:
579+
api_key: '%env(ELEVEN_LABS_API_KEY)%'
580+
581+
speech:
582+
elevenlabs:
583+
platform: 'ai.platform.elevenlabs'
584+
tts_model: 'eleven_multilingual_v2'
585+
tts_voice: '%env(ELEVEN_LABS_VOICE_IDENTIFIER)%'
586+
tts_extra_options:
587+
foo: bar
588+
589+
.. note::
590+
591+
Please be aware that enabling speech support requires to define corresponding platforms.
592+
535593
Code Examples
536594
~~~~~~~~~~~~~
537595

‎examples/speech/README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Speech Examples
2+
3+
Speech is mainly used to transform text to audio and vice versa, it can also be used to create an audio to audio pipeline.
4+
5+
To run the examples, you can use additional tools like (mpg123)[https://www.mpg123.de/]:
6+
7+
```bash
8+
php speech/agent-eleven-labs-speech-tts.php | mpg123 -
9+
php speech/agent-eleven-labs-speech-sts.php | mpg123 -
10+
```

‎examples/speech/agent-eleven-labs-speech-sts.php‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechListener;
14+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
15+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
16+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
17+
use Symfony\AI\Platform\Message\Content\Audio;
18+
use Symfony\AI\Platform\Message\Message;
19+
use Symfony\AI\Platform\Message\MessageBag;
20+
use Symfony\AI\Platform\Speech\SpeechAwarePlatform;
21+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
22+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
23+
use Symfony\Component\EventDispatcher\EventDispatcher;
24+
25+
require_once dirname(__DIR__).'/bootstrap.php';
26+
27+
$speechAwarePlatform = new SpeechAwarePlatform(
28+
PlatformFactory::create(
29+
apiKey: env('ELEVEN_LABS_API_KEY'),
30+
httpClient: http_client(),
31+
),
32+
new SpeechConfiguration(
33+
ttsModel: 'eleven_multilingual_v2',
34+
ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
35+
sttModel: 'eleven_multilingual_v2'
36+
),
37+
);
38+
39+
$eventDispatcher = new EventDispatcher();
40+
$eventDispatcher->addSubscriber(new SpeechProviderListener([
41+
new ElevenLabsSpeechProvider($speechAwarePlatform),
42+
], [
43+
new ElevenLabsSpeechListener($speechAwarePlatform),
44+
]));
45+
46+
$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
47+
48+
$agent = new Agent($platform, 'gpt-4o');
49+
$answer = $agent->call(new MessageBag(
50+
Message::ofUser(Audio::fromFile(dirname(__DIR__, 2).'/fixtures/audio.mp3'))
51+
));
52+
53+
echo $answer->getSpeech('elevenlabs')->asBinary();

‎examples/speech/agent-eleven-labs-speech-stt.php‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechListener;
14+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
15+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
16+
use Symfony\AI\Platform\Message\Content\Audio;
17+
use Symfony\AI\Platform\Message\Message;
18+
use Symfony\AI\Platform\Message\MessageBag;
19+
use Symfony\AI\Platform\Speech\SpeechAwarePlatform;
20+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
21+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
22+
use Symfony\Component\EventDispatcher\EventDispatcher;
23+
24+
require_once dirname(__DIR__).'/bootstrap.php';
25+
26+
$eventDispatcher = new EventDispatcher();
27+
$eventDispatcher->addSubscriber(new SpeechProviderListener([], [
28+
new ElevenLabsSpeechListener(new SpeechAwarePlatform(
29+
PlatformFactory::create(
30+
apiKey: env('ELEVEN_LABS_API_KEY'),
31+
httpClient: http_client(),
32+
),
33+
speechConfiguration: new SpeechConfiguration(
34+
ttsModel: 'eleven_multilingual_v2',
35+
ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
36+
sttModel: 'eleven_multilingual_v2'
37+
)),
38+
),
39+
]));
40+
41+
$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
42+
43+
$agent = new Agent($platform, 'gpt-4o');
44+
$answer = $agent->call(new MessageBag(
45+
Message::ofUser(Audio::fromFile(dirname(__DIR__, 2).'/fixtures/audio.mp3'))
46+
));
47+
48+
echo $answer->getContent();

‎examples/speech/agent-eleven-labs-speech-tts.php‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
14+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
15+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
16+
use Symfony\AI\Platform\Message\Message;
17+
use Symfony\AI\Platform\Message\MessageBag;
18+
use Symfony\AI\Platform\Speech\SpeechAwarePlatform;
19+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
20+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
21+
use Symfony\Component\EventDispatcher\EventDispatcher;
22+
23+
require_once dirname(__DIR__).'/bootstrap.php';
24+
25+
$eventDispatcher = new EventDispatcher();
26+
$eventDispatcher->addSubscriber(new SpeechProviderListener([
27+
new ElevenLabsSpeechProvider(new SpeechAwarePlatform(
28+
PlatformFactory::create(
29+
apiKey: env('ELEVEN_LABS_API_KEY'),
30+
httpClient: http_client(),
31+
),
32+
speechConfiguration: new SpeechConfiguration(
33+
ttsModel: 'eleven_multilingual_v2',
34+
ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
35+
sttModel: 'eleven_multilingual_v2'
36+
)),
37+
),
38+
], []));
39+
40+
$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
41+
42+
$agent = new Agent($platform, 'gpt-4o');
43+
$answer = $agent->call(new MessageBag(
44+
Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
45+
));
46+
47+
echo $answer->getSpeech('elevenlabs')->asBinary();

‎src/agent/src/Output.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
use Symfony\AI\Platform\Message\MessageBag;
1515
use Symfony\AI\Platform\Result\ResultInterface;
16+
use Symfony\AI\Platform\Speech\Speech;
1617

1718
/**
1819
* @author Christopher Hertel <mail@christopher-hertel.de>
@@ -27,6 +28,7 @@ public function __construct(
2728
private ResultInterface $result,
2829
private readonly MessageBag $messageBag,
2930
private readonly array $options = [],
31+
private ?Speech $speech = null,
3032
) {
3133
}
3234

@@ -57,4 +59,14 @@ public function getOptions(): array
5759
{
5860
return $this->options;
5961
}
62+
63+
public function setSpeech(?Speech $speech): void
64+
{
65+
$this->speech = $speech;
66+
}
67+
68+
public function getSpeech(): ?Speech
69+
{
70+
return $this->speech;
71+
}
6072
}

‎src/ai-bundle/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,4 @@ CHANGELOG
3434
- Token usage metadata in agent results including prompt, completion, total, cached, and thinking tokens
3535
- Rate limit information tracking for supported platforms
3636
* Add support for configuring chats and message stores
37+
* Add support for configuring speeches

‎src/ai-bundle/config/options.php‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1086,6 +1086,23 @@
10861086
->end()
10871087
->end()
10881088
->end()
1089+
->arrayNode('speech')
1090+
->useAttributeAsKey('name')
1091+
->arrayPrototype()
1092+
->children()
1093+
->stringNode('platform')->isRequired()->end()
1094+
->stringNode('tts_model')->end()
1095+
->stringNode('tts_voice')->end()
1096+
->arrayNode('tts_extra_options')
1097+
->scalarPrototype()->end()
1098+
->end()
1099+
->stringNode('stt_model')->end()
1100+
->arrayNode('stt_extra_options')
1101+
->scalarPrototype()->end()
1102+
->end()
1103+
->end()
1104+
->end()
1105+
->end()
10891106
->arrayNode('vectorizer')
10901107
->info('Vectorizers for converting strings to Vector objects and transforming TextDocument arrays to VectorDocument arrays')
10911108
->useAttributeAsKey('name')

0 commit comments

Comments
 (0)