diff --git a/backend/index.yaml b/backend/index.yaml index 3420d85496b7..c3e590fc8a3c 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -1194,6 +1194,7 @@ tags: - image-generation - video-generation + - sound-generation - diffusion-models license: apache-2.0 alias: "diffusers" diff --git a/backend/python/diffusers/audio_utils.py b/backend/python/diffusers/audio_utils.py new file mode 100644 index 000000000000..ffc4e9112631 --- /dev/null +++ b/backend/python/diffusers/audio_utils.py @@ -0,0 +1,24 @@ +import array +import sys +import wave + + +def write_pcm_wav(destination, samples, sampling_rate): + """Write normalized floating-point audio samples as mono 16-bit PCM.""" + pcm = array.array( + "h", + ( + max(-32768, min(32767, round(float(sample) * 32768))) + for sample in samples + ), + ) + if pcm.itemsize != 2: + raise RuntimeError("16-bit PCM requires two-byte signed integers") + if sys.byteorder != "little": + pcm.byteswap() + + with wave.open(destination, "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(sampling_rate) + output.writeframes(pcm.tobytes()) diff --git a/backend/python/diffusers/backend.py b/backend/python/diffusers/backend.py index 539ce54448bf..79da7f044ea4 100755 --- a/backend/python/diffusers/backend.py +++ b/backend/python/diffusers/backend.py @@ -26,6 +26,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) from grpc_auth import get_auth_interceptors from model_utils import resolve_model_reference +from audio_utils import write_pcm_wav # Import dynamic loader for pipeline discovery @@ -883,6 +884,49 @@ def GenerateImage(self, request, context): return backend_pb2.Result(message="Media generated", success=True) + def SoundGeneration(self, request, context): + if not request.dst: + return backend_pb2.Result(success=False, message="request.dst is required") + + prompt = request.text or request.caption + if not prompt: + return backend_pb2.Result(success=False, message="request.text is required") + + try: + generation_options = dict(self.options) + if "num_inference_steps" in generation_options: + generation_options["num_inference_steps"] = int( + generation_options["num_inference_steps"] + ) + generation_options["prompt"] = prompt + if request.HasField("duration"): + generation_options["audio_length_in_s"] = request.duration + if request.HasField("temperature"): + generation_options["guidance_scale"] = request.temperature + + generated = self.pipe(**generation_options) + if not hasattr(generated, "audios") or len(generated.audios) == 0: + return backend_pb2.Result( + success=False, + message="The diffusers pipeline returned no audio", + ) + + samples = generated.audios[0] + if hasattr(samples, "reshape"): + samples = samples.reshape(-1) + if hasattr(samples, "tolist"): + samples = samples.tolist() + + sampling_rate = getattr( + getattr(getattr(self.pipe, "vae", None), "config", None), + "sampling_rate", + 16000, + ) + write_pcm_wav(request.dst, samples, sampling_rate) + return backend_pb2.Result(success=True, message="Sound generated successfully") + except Exception as err: + return backend_pb2.Result(success=False, message=f"SoundGeneration error: {err}") + def UpscaleImage(self, request, context): try: if not request.src: diff --git a/backend/python/diffusers/test.py b/backend/python/diffusers/test.py index eff293ee6e10..d8a178dc1677 100644 --- a/backend/python/diffusers/test.py +++ b/backend/python/diffusers/test.py @@ -4,6 +4,9 @@ import unittest import subprocess import time +import os +import tempfile +import wave from unittest.mock import patch, MagicMock # Import dynamic loader for testing (these don't need gRPC) @@ -373,3 +376,64 @@ def test_options_merged_into_pipeline_kwargs(self): finally: os.unlink(src_file.name) os.unlink(dst_file.name) + + +class TestWritePcmWav(unittest.TestCase): + def test_writes_clipped_float_samples_as_mono_pcm(self): + from audio_utils import write_pcm_wav + + destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + destination.close() + + try: + write_pcm_wav(destination.name, [0.0, 0.5, -0.5, 2.0], 16000) + + with wave.open(destination.name, "rb") as generated: + self.assertEqual(generated.getframerate(), 16000) + self.assertEqual(generated.getnchannels(), 1) + self.assertEqual(generated.getsampwidth(), 2) + self.assertEqual(generated.getnframes(), 4) + self.assertEqual( + generated.readframes(4), + b"\x00\x00\x00@\x00\xc0\xff\x7f", + ) + finally: + os.unlink(destination.name) + + +@unittest.skipUnless(GRPC_AVAILABLE, "gRPC modules not available") +class TestSoundGeneration(unittest.TestCase): + def test_maps_request_options_and_writes_pipeline_audio(self): + from backend import BackendServicer + + service = BackendServicer.__new__(BackendServicer) + service.options = {"num_inference_steps": 200.0} + service.pipe = MagicMock() + service.pipe.return_value.audios = [[0.0, 0.5, -0.5]] + service.pipe.vae.config.sampling_rate = 16000 + + destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + destination.close() + + try: + request = backend_pb2.SoundGenerationRequest( + text="ocean waves", + dst=destination.name, + duration=2.5, + temperature=0, + ) + + result = service.SoundGeneration(request, context=None) + + self.assertTrue(result.success, result.message) + service.pipe.assert_called_once_with( + num_inference_steps=200, + prompt="ocean waves", + audio_length_in_s=2.5, + guidance_scale=0, + ) + with wave.open(destination.name, "rb") as generated: + self.assertEqual(generated.getframerate(), 16000) + self.assertEqual(generated.getnframes(), 3) + finally: + os.unlink(destination.name) diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 50ace9b3634f..630d76222a1b 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -369,10 +369,10 @@ var BackendCapabilities = map[string]BackendCapability{ // --- Image/video generation backends --- "diffusers": { - GRPCMethods: []GRPCMethod{MethodGenerateImage, MethodUpscaleImage, MethodGenerateVideo}, - PossibleUsecases: []string{UsecaseImage, UsecaseVideo}, + GRPCMethods: []GRPCMethod{MethodGenerateImage, MethodUpscaleImage, MethodGenerateVideo, MethodSoundGeneration}, + PossibleUsecases: []string{UsecaseImage, UsecaseVideo, UsecaseSoundGeneration}, DefaultUsecases: []string{UsecaseImage}, - Description: "HuggingFace diffusers — Stable Diffusion, Flux, video generation", + Description: "HuggingFace diffusers — image, video, and sound generation", }, "longcat-video": { GRPCMethods: []GRPCMethod{MethodGenerateVideo}, diff --git a/core/config/backend_capabilities_test.go b/core/config/backend_capabilities_test.go index b18c410693ba..8329a28115bc 100644 --- a/core/config/backend_capabilities_test.go +++ b/core/config/backend_capabilities_test.go @@ -57,6 +57,12 @@ var _ = Describe("BackendCapabilities", func() { }) var _ = Describe("GetBackendCapability", func() { + It("advertises diffusers sound generation", func() { + capability := GetBackendCapability("diffusers") + Expect(capability.GRPCMethods).To(ContainElement(MethodSoundGeneration)) + Expect(capability.PossibleUsecases).To(ContainElement(UsecaseSoundGeneration)) + }) + It("returns the capability for a known backend", func() { cap := GetBackendCapability("llama-cpp") Expect(cap).NotTo(BeNil()) diff --git a/docs/content/features/text-to-audio.md b/docs/content/features/text-to-audio.md index 8ff355a73f1f..f4b7a84f70bc 100644 --- a/docs/content/features/text-to-audio.md +++ b/docs/content/features/text-to-audio.md @@ -302,6 +302,31 @@ The `/v1/sound-generation` endpoint is compatible with the [ElevenLabs sound gen Error responses: `400` for a missing or invalid model or request parameters, and `500` for a backend error during sound generation. +### AudioLDM 2 + +[AudioLDM 2](https://github.com/haoheliu/AudioLDM2) generates sound effects, +music, and speech from a text description. Install the gallery model: + +```bash +local-ai models install audioldm2 +``` + +Generate a WAV file through the sound-generation endpoint: + +```bash +curl http://localhost:8080/v1/sound-generation \ + -H "Content-Type: application/json" \ + -d '{ + "model_id": "audioldm2", + "text": "Waves breaking on a rocky beach during a distant thunderstorm", + "duration_seconds": 10 + }' --output storm.wav +``` + +AudioLDM 2 uses the `AudioLDM2Pipeline` from the diffusers backend. The +`duration_seconds` field maps to the pipeline's `audio_length_in_s` option, and +`prompt_influence` maps to `guidance_scale`. + #### Configuration You can configure ACE-Step models with various options: diff --git a/gallery/audioldm2.yaml b/gallery/audioldm2.yaml new file mode 100644 index 000000000000..05571b0a9d50 --- /dev/null +++ b/gallery/audioldm2.yaml @@ -0,0 +1,15 @@ +--- +name: "audioldm2" + +config_file: | + backend: diffusers + known_usecases: + - sound_generation + parameters: + model: cvssp/audioldm2 + diffusers: + pipeline_type: AudioLDM2Pipeline + cuda: true + options: + - num_inference_steps:200 + - torch_dtype:fp16 diff --git a/gallery/index.yaml b/gallery/index.yaml index 08549238953f..03fb611bfe17 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,20 @@ --- +- name: audioldm2 + url: github:mudler/LocalAI/gallery/audioldm2.yaml@master + urls: + - https://huggingface.co/cvssp/audioldm2 + - https://github.com/haoheliu/AudioLDM2 + description: | + AudioLDM 2 generates sound effects, music, and speech from natural-language + descriptions through the diffusers backend and LocalAI sound-generation API. + license: cc-by-nc-sa-4.0 + tags: + - audio + - sound-generation + - text-to-audio + - diffusers + - gpu + last_checked: "2026-08-13" - &ornith-1-0-9b name: "ornith-1.0-9b-q4" variants: