From 057c6a2ba7f1891fcdc067d76efa29f1b7ca2769 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Tue, 18 Aug 2026 16:03:17 +0000 Subject: [PATCH 01/11] chore(release): add portable binary installation - add checksummed release installers with source fallback - introduce 0.1.0 package versioning and runtime notices - align server profiles and CUDA compatibility --- CMakeLists.txt | 49 ++++++++++-------- CMakePresets.json | 20 +++++--- README.md | 31 +++++++++--- THIRD_PARTY_NOTICES.md | 19 +++++-- VERSION | 2 +- docker/Dockerfile | 8 +-- docs/build.md | 9 ++-- docs/development/cublas-shim.md | 17 ++++--- docs/development/windows-build.md | 2 +- docs/install.md | 46 ++++++++++------- docs/server.md | 10 ++-- ggml-patches/0005-skinny-q8-gemm.patch | 12 ++++- ggml-patches/README.md | 4 +- kernels/cublas_shim.cu | 6 +-- kernels/ver_cublas.map | 2 +- scripts/configure.sh | 2 +- scripts/install.ps1 | 37 ++++++++++---- scripts/install.sh | 70 +++++++++++++++++++++----- scripts/windows/build.ps1 | 7 ++- src/asr/CMakeLists.txt | 6 +++ tests/install/install_ps1_test.py | 14 +++--- tests/install/install_sh_test.py | 53 ++++++++++++++++--- 22 files changed, 300 insertions(+), 126 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d216765..fb93442 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,15 +8,17 @@ cmake_minimum_required(VERSION 3.26) # nemo_speech_tts_version) stay in sync without a hardcoded literal. file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" _nemo_speech_version_lines) foreach(_line ${_nemo_speech_version_lines}) - if(_line MATCHES "^NEMO_SPEECH_VERSION:[ \t]*([0-9]+\\.[0-9]+\\.[0-9]+)") - set(NEMO_SPEECH_VERSION "${CMAKE_MATCH_1}") + if(_line MATCHES + "^NEMO_SPEECH_VERSION:[ \t]*([0-9]+\\.[0-9]+\\.[0-9]+)([-+][0-9A-Za-z.-]+)?[ \t]*$") + set(NEMO_SPEECH_VERSION "${CMAKE_MATCH_1}${CMAKE_MATCH_2}") + set(NEMO_SPEECH_PROJECT_VERSION "${CMAKE_MATCH_1}") endif() endforeach() -if(NOT NEMO_SPEECH_VERSION) +if(NOT NEMO_SPEECH_VERSION OR NOT NEMO_SPEECH_PROJECT_VERSION) message(FATAL_ERROR "could not parse NEMO_SPEECH_VERSION from ${CMAKE_CURRENT_SOURCE_DIR}/VERSION") endif() -project(nemo_speech VERSION ${NEMO_SPEECH_VERSION} LANGUAGES C CXX) +project(nemo_speech VERSION ${NEMO_SPEECH_PROJECT_VERSION} LANGUAGES C CXX) add_compile_definitions(NEMO_SPEECH_VERSION_STR="${NEMO_SPEECH_VERSION}") include(GNUInstallDirs) @@ -153,15 +155,14 @@ if(NEMO_SPEECH_BUILD_DIAR AND NOT NEMO_SPEECH_BUILD_ASR) "without the transcribe CLI/API surface") endif() -# Drop-in cuBLAS shim (native GEMM, no cuBLASLt). When ON, builds -# libcublas.so.13 from kernels/cublas_shim.cu. The shipping container image -# substitutes it for real cuBLAS to drop ~564 MB; put it on LD_LIBRARY_PATH -# ahead of the system cuBLAS to reproduce that GEMM path natively. +# Drop-in cuBLAS shim (native GEMM, no cuBLASLt). The shipping container image +# substitutes it for real cuBLAS to reduce its runtime closure; put it on +# LD_LIBRARY_PATH ahead of the system cuBLAS to reproduce that GEMM path. # Disabled by default for source builds, which link the CUDA toolkit's cuBLAS. # Container builds enable the shim explicitly to reduce the runtime image size. # It is only built when GGML_CUDA is also ON (see the target below), and is a # no-op for Metal, Vulkan, and CPU builds. -option(NEMO_SPEECH_CUBLAS_SHIM "Build the in-tree drop-in cuBLAS shim (libcublas.so.13, native GEMM, no cuBLASLt)" OFF) +option(NEMO_SPEECH_CUBLAS_SHIM "Build the in-tree drop-in cuBLAS shim (native GEMM, no cuBLASLt)" OFF) # Whether the linked ggml has the project ASR patches applied (ggml-patches/: # the fused rel-pos attention op and the F16 depthwise-conv kernel). The ASR @@ -279,18 +280,20 @@ if(GGML_METAL) LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() -# In-tree cuBLAS shim: a drop-in libcublas.so.13 backed by native GEMM kernels -# (no cuBLASLt). ggml-cuda links real cuBLAS at build time, but the shim shares -# its SONAME, so binaries resolve it instead when it is first on LD_LIBRARY_PATH. -# The container images ship it in place of real cuBLAS (~564 MB saved); this -# target reproduces that for native builds. +# In-tree cuBLAS shim backed by native GEMM kernels (no cuBLASLt). ggml-cuda +# links real cuBLAS at build time, but the shim shares its major-version SONAME, +# so binaries resolve it instead when it is first on LD_LIBRARY_PATH. # The cuBLAS shim is a Linux-only container size optimization. It relies on ELF -# SONAME versioning (SOVERSION 13 -> libcublas.so.13) plus a GNU-ld -# --version-script, neither of which exists with MSVC/link.exe. On Windows, -# ggml-cuda links the real cuBLAS DLL from the CUDA toolkit, so the shim is -# neither needed nor buildable - skip it. +# SONAME and symbol versioning plus a GNU-ld --version-script, neither of which +# exists with MSVC/link.exe. On Windows, ggml-cuda links the real cuBLAS DLL +# from the CUDA toolkit, so the shim is neither needed nor buildable - skip it. if(GGML_CUDA AND NEMO_SPEECH_CUBLAS_SHIM AND NOT WIN32) enable_language(CUDA) + if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.0) + set(NEMO_SPEECH_CUBLAS_SOVERSION 12) + else() + set(NEMO_SPEECH_CUBLAS_SOVERSION 13) + endif() add_library(nemo_speech_cublas_shim SHARED kernels/cublas_shim.cu) if(CMAKE_CUDA_ARCHITECTURES) set(NEMO_SPEECH_CUBLAS_SHIM_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}") @@ -301,11 +304,15 @@ if(GGML_CUDA AND NEMO_SPEECH_CUBLAS_SHIM AND NOT WIN32) endif() set_target_properties(nemo_speech_cublas_shim PROPERTIES OUTPUT_NAME cublas - SOVERSION 13 + SOVERSION "${NEMO_SPEECH_CUBLAS_SOVERSION}" CUDA_ARCHITECTURES "${NEMO_SPEECH_CUBLAS_SHIM_ARCHITECTURES}" CUDA_SEPARABLE_COMPILATION ON) + configure_file( + kernels/ver_cublas.map + "${CMAKE_CURRENT_BINARY_DIR}/ver_cublas.map" + @ONLY) target_link_options(nemo_speech_cublas_shim PRIVATE - "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/kernels/ver_cublas.map") + "LINKER:--version-script=${CMAKE_CURRENT_BINARY_DIR}/ver_cublas.map") elseif(GGML_CUDA AND NEMO_SPEECH_CUBLAS_SHIM AND WIN32) message(STATUS "NEMO_SPEECH_CUBLAS_SHIM: skipped on Windows; linking the CUDA " @@ -500,7 +507,7 @@ configure_package_config_file( write_basic_package_version_file( ${CMAKE_CURRENT_BINARY_DIR}/NeMoSpeechConfigVersion.cmake VERSION ${PROJECT_VERSION} - COMPATIBILITY SameMajorVersion) + COMPATIBILITY SameMinorVersion) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/NeMoSpeechConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/NeMoSpeechConfigVersion.cmake diff --git a/CMakePresets.json b/CMakePresets.json index 59a0b9a..9a11ad6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -206,49 +206,53 @@ { "name": "cpu-server", "inherits": "cpu-asr", - "displayName": "CPU ASR and TTS HTTP server and playground", + "displayName": "CPU speech CLI, HTTP server, and playground", "cacheVariables": { "NEMO_SPEECH_BUILD_TTS": "ON", + "NEMO_SPEECH_BUILD_NMT": "ON", "NEMO_SPEECH_BUILD_HTTP": "ON", "NEMO_SPEECH_BUILD_GRPC": "OFF", "NEMO_SPEECH_WITH_GRPC": "OFF", - "NEMO_SPEECH_WITH_NMT": "OFF" + "NEMO_SPEECH_WITH_NMT": "ON" } }, { "name": "cuda-server", "inherits": "cuda-asr", - "displayName": "CUDA ASR and TTS HTTP server and playground", + "displayName": "CUDA speech CLI, HTTP server, and playground", "cacheVariables": { "NEMO_SPEECH_BUILD_TTS": "ON", + "NEMO_SPEECH_BUILD_NMT": "ON", "NEMO_SPEECH_BUILD_HTTP": "ON", "NEMO_SPEECH_BUILD_GRPC": "OFF", "NEMO_SPEECH_WITH_GRPC": "OFF", - "NEMO_SPEECH_WITH_NMT": "OFF" + "NEMO_SPEECH_WITH_NMT": "ON" } }, { "name": "metal-server", "inherits": "metal-asr", - "displayName": "Metal ASR and TTS HTTP server and playground", + "displayName": "Metal speech CLI, HTTP server, and playground", "cacheVariables": { "NEMO_SPEECH_BUILD_TTS": "ON", + "NEMO_SPEECH_BUILD_NMT": "ON", "NEMO_SPEECH_BUILD_HTTP": "ON", "NEMO_SPEECH_BUILD_GRPC": "OFF", "NEMO_SPEECH_WITH_GRPC": "OFF", - "NEMO_SPEECH_WITH_NMT": "OFF" + "NEMO_SPEECH_WITH_NMT": "ON" } }, { "name": "vulkan-server", "inherits": "vulkan-asr", - "displayName": "Vulkan ASR and TTS HTTP server and playground", + "displayName": "Vulkan speech CLI, HTTP server, and playground", "cacheVariables": { "NEMO_SPEECH_BUILD_TTS": "ON", + "NEMO_SPEECH_BUILD_NMT": "ON", "NEMO_SPEECH_BUILD_HTTP": "ON", "NEMO_SPEECH_BUILD_GRPC": "OFF", "NEMO_SPEECH_WITH_GRPC": "OFF", - "NEMO_SPEECH_WITH_NMT": "OFF" + "NEMO_SPEECH_WITH_NMT": "ON" } }, { diff --git a/README.md b/README.md index b19d661..078aefe 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,24 @@ # NeMo-Speech.cpp -A lightweight native C++ runtime for NVIDIA Nemotron Speech models built on ggml. Runs speech models in realtime and in batch mode across platforms/backends. +A lightweight native C++ runtime for running NVIDIA speech and voice models locally, with broad hardware support. It supports speech recognition, speaker diarization, translation, and speech synthesis in realtime and batch mode. + +It builds on speech models and tooling from [NVIDIA NeMo Speech](https://github.com/NVIDIA-NeMo/Speech). Native inference is powered by ggml. + +NVIDIA's official local speech inference solution, with day-0 support for our latest speech models. + +## Models and applications + +| Application | Supported models | +|---|---| +| Speech recognition | [Nemotron 3.5 ASR Streaming 0.6B](https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b), [Nemotron Speech Streaming 0.6B](https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b), [Parakeet TDT 0.6B v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3), and [Parakeet CTC 1.1B](https://huggingface.co/nvidia/parakeet-ctc-1.1b) | +| Speaker diarization | [Streaming Sortformer 4-speaker v2](https://huggingface.co/nvidia/diar_streaming_sortformer_4spk-v2), standalone or combined with ASR | +| Text and speech translation | [Riva Translate 4B Instruct v2](https://huggingface.co/nvidia/Riva-Translate-4B-Instruct-v2), with composed ASR-to-NMT-to-TTS speech translation | +| Speech synthesis | [MagpieTTS Multilingual 357M](https://huggingface.co/nvidia/magpie_tts_multilingual_357m) with [NeMo NanoCodec](https://huggingface.co/nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps) | +| Speech processing | [Silero VAD](https://github.com/snakers4/silero-vad), punctuation and capitalization, endpointing, text normalization, and subtitles | ## Contents +- [Models and applications](#models-and-applications) - [Installation](#installation) - [Quick start](#quick-start) - [Command line](#command-line) @@ -16,18 +31,18 @@ A lightweight native C++ runtime for NVIDIA Nemotron Speech models built on ggml ## Installation -From a source checkout, install the CLI, HTTP API, and browser playground for -the detected platform and backend: +Install the `nemo-speech` CLI for the detected platform and backend: ```bash -scripts/install.sh --source +curl -fsSL https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.sh | sh export PATH="$HOME/.local/bin:$PATH" # current shell; future shells are updated ``` -The source build requires Git, CMake 3.26 or newer, Ninja, a C++17 compiler, -and the toolkit for the selected GPU backend. See -[Installation](docs/install.md) for platform-specific prerequisites, options, -and the native release-archive flow. +The installer prefers a verified native release and falls back to a source +build when an artifact is unavailable. A source build requires Git, CMake 3.26 +or newer, Ninja, a C++17 compiler, and the selected GPU toolkit. See +[Installation](docs/install.md) for platform-specific prerequisites and +options. ## Quick start diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 659ee7c..4b07117 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -129,16 +129,29 @@ Debian/Ubuntu shared library under project notice set is installed under `/opt/nemo-speech/share/licenses/nemo-speech/`. +### NVIDIA CUDA Runtime + +- Source: [NVIDIA CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) +- Copyright (c) NVIDIA CORPORATION & AFFILIATES +- License: NVIDIA Software License Agreement and CUDA Supplement + +CUDA release archives include the redistributable CUDA runtime library. The +applicable agreement is installed with the archive under +`share/licenses/nemo-speech/nvidia/cuda-runtime/`. + ## Other incorporated third-party code and data ### SentencePiece -- Source: [`google/sentencepiece`](https://github.com/google/sentencepiece) +- Source: [`google/sentencepiece`](https://github.com/google/sentencepiece), + revision `17d7580d6407802f85855d2cc9190634e2c95624` - Copyright 2018 Google Inc. - License: Apache License 2.0 -Default Windows ASR builds link SentencePiece statically and include its license -notice. +Default Windows ASR and Linux release builds statically link the SentencePiece +runtime and its bundled Abseil, protobuf-lite, and Darts-clone components. Their +Apache 2.0 and BSD license texts are installed under +`share/licenses/nemo-speech/third_party/sentencepiece/`. ### whisper.cpp sample audio diff --git a/VERSION b/VERSION index 1f73b04..5e37c63 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -NEMO_SPEECH_VERSION: 1.0.0 +NEMO_SPEECH_VERSION: 0.1.0 diff --git a/docker/Dockerfile b/docker/Dockerfile index c1b799f..47cfb0a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -165,8 +165,8 @@ RUN if [ "${ENABLE_GGML_PATCHES}" = "ON" ]; then bash /work/scripts/apply-ggml-p # NCCL disabled: single-GPU inference does not use multi-GPU collectives, # and the linked libnccl is otherwise dead runtime weight. -# NEMO_SPEECH_CUBLAS_SHIM=ON builds the drop-in libcublas.so.13 (native GEMM, -# no cuBLASLt) as a normal build target. +# NEMO_SPEECH_CUBLAS_SHIM=ON builds the drop-in libcublas.so. +# (native GEMM, no cuBLASLt) as a normal build target. RUN cmake -G Ninja -S /work -B /work/build \ -DCMAKE_BUILD_TYPE=Release \ -DGGML_CUDA=${ENABLE_CUDA} \ @@ -302,7 +302,7 @@ RUN set -eux; \ dep="$1"; \ # Skip the CUDA driver (toolkit-injected at run time) and real cuBLAS/ # cuBLASLt: the in-tree shim in /opt/nemo-speech/lib provides - # libcublas.so.13, and nothing else needs cuBLASLt. + # libcublas.so., and nothing else needs cuBLASLt. case "$dep" in \ */libcuda.so*|*/libcublas.so*|*/libcublasLt.so*) return 0;; \ esac; \ @@ -330,7 +330,7 @@ RUN set -eux; \ fi; \ done; \ # The cuBLAS shim (CMake target -DNEMO_SPEECH_CUBLAS_SHIM=ON, staged via - # /out/lib) is the only libcublas.so.13 in the image and is first on + # /out/lib) is the only libcublas.so. in the image and is first on # LD_LIBRARY_PATH; real cuBLAS/cuBLASLt were skipped by copy_dep above. for b in "$root"/opt/nemo-speech/bin/*; do [ -f "$b" ] && strip --strip-unneeded "$b" || true; done; \ chmod 0755 "$root"/opt/nemo-speech/bin/*; \ diff --git a/docs/build.md b/docs/build.md index c39ded6..1436aee 100644 --- a/docs/build.md +++ b/docs/build.md @@ -82,13 +82,14 @@ landmarks are: | `cuda-speech` | CUDA ASR, diarization, NMT, and TTS | | `metal-nmt` | Metal-enabled NMT build | | `vulkan-diar` | Vulkan standalone diarization build | -| `-server` | ASR, diarization, TTS, HTTP API, realtime WebSocket, and playground | +| `-server` | ASR, diarization, NMT, TTS, HTTP API, realtime WebSocket, and playground | | `cuda-full` | CUDA server plus normalization, Flashlight, and language frontends | | `developer` | CPU speech components plus HTTP, gRPC, examples, tests, and tools | -The `-server` presets are what the source installer uses. They do not -pull in NMT, protobuf, or gRPC. Use `cuda-full`, `developer`, or explicit CMake options when -those features and the separate `riva_server` executable are needed. +The `-server` presets are used by the source installer and release +builders. They include NMT but not protobuf or gRPC. Use `cuda-full`, +`developer`, or explicit CMake options when the Riva-compatible adapters and +separate `riva_server` executable are needed. The preset selects which components and ggml backend are compiled. diff --git a/docs/development/cublas-shim.md b/docs/development/cublas-shim.md index d93e379..560ef5d 100644 --- a/docs/development/cublas-shim.md +++ b/docs/development/cublas-shim.md @@ -3,13 +3,13 @@ The minimal runtime image ships **no NVIDIA cuBLAS**. ggml-cuda's non-quantized GEMMs (FastConformer attention, subsampling convs, CTC head; the q8/RNNT weight matmuls already use ggml's quantized kernels) are served instead -by an in-tree drop-in `libcublas.so.13`. +by an in-tree drop-in `libcublas`. ## The shim -`kernels/cublas_shim.cu` (with the symbol map `kernels/ver_cublas.map`) is a -drop-in `libcublas.so.13`: shape-specialized CUDA GEMM/GEMV kernels, including -WMMA tensor-core paths, but **no cuBLASLt**. It inherits +`kernels/cublas_shim.cu` (with the generated symbol map from +`kernels/ver_cublas.map`) is a drop-in cuBLAS library: shape-specialized CUDA +GEMM/GEMV kernels, including WMMA tensor-core paths, but **no cuBLASLt**. It inherits `CMAKE_CUDA_ARCHITECTURES` when set and falls back to JIT-portable `compute_80` PTX for ad-hoc builds. Dropping real cuBLAS + cuBLASLt is the bulk of the container size. The shim is built as a separate library from ggml. @@ -17,9 +17,10 @@ container size. The shim is built as a separate library from ggml. It's an optional CMake target, **`NEMO_SPEECH_CUBLAS_SHIM` (default `OFF`)**, built when explicitly enabled together with `GGML_CUDA` (Linux only, auto-skipped on Windows; a no-op for Metal, Vulkan, and CPU builds). Normal -source builds therefore link the CUDA toolkit's cuBLAS and cuBLASLt. The -Dockerfile enables the shim explicitly and skips those libraries in the runtime -image's library closure. +source builds therefore link the CUDA toolkit's cuBLAS and cuBLASLt. Container +and release-archive builds enable the shim explicitly and skip those libraries +in their runtime closure. The generated SONAME and symbol version match the +CUDA toolkit major used for the build. To build and exercise the container GEMM path outside the container, enable the shim and put its output directory first on the loader path: @@ -37,4 +38,4 @@ The heavier project-specific CUDA kernels (fused rel-pos attention, skinny-Q8 GE NVFP4 quantization, BF16 FastConformer epilogues, fused LayerNorm, and F16 depthwise conv2d) live as ggml patches rather than in `kernels/` - see [ggml patches](ggml-patches.md). `kernels/` holds only the cuBLAS shim and its -version map. +version-map template. diff --git a/docs/development/windows-build.md b/docs/development/windows-build.md index 7430030..fb9cf99 100644 --- a/docs/development/windows-build.md +++ b/docs/development/windows-build.md @@ -97,7 +97,7 @@ powershell -ExecutionPolicy Bypass -File scripts\windows\build.ps1 -Backend vulk # CPU-only, no server powershell -ExecutionPolicy Bypass -File scripts\windows\build.ps1 -Backend cpu -# CPU ASR + TTS + HTTP API, realtime WebSocket, and playground +# CPU ASR + NMT + TTS + HTTP API, realtime WebSocket, and playground powershell -ExecutionPolicy Bypass -File scripts\windows\build.ps1 -Backend cpu -Profile server # Full runtime profile (add -HttpTls for TLS) diff --git a/docs/install.md b/docs/install.md index 3be6d36..5da5d37 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,38 +1,46 @@ # Install NeMo-Speech.cpp -The current public installation path builds and installs the backend-matched -ASR, diarization, and TTS CLI, HTTP API, realtime WebSocket endpoint, and browser -playground from a source checkout. Models are distributed separately and are -never downloaded when the server starts. Ready-to-run GGUFs are available from -the linked Hugging Face repositories in the [ASR](asr/models.md) and +The installer selects a backend-matched native release containing the ASR, +diarization, translation, and TTS CLI, HTTP API, realtime WebSocket endpoint, +browser playground, SDK, and notices. It builds from source when a matching +archive is unavailable. Models are distributed separately and are never +downloaded when the server starts. Ready-to-run GGUFs are available from the +linked Hugging Face repositories in the [ASR](asr/models.md) and [TTS](tts/models.md) model guides. -The installers also contain the native release flow. Once a public release URL -is configured, they prefer release archives containing the CLI, runtime -libraries, headers, CMake package files, and license notices, with source as the -fallback. - ## Linux and macOS Inspect [`scripts/install.sh`](../scripts/install.sh), then run: ```bash -scripts/install.sh --source +curl -fsSL https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.sh | sh export PATH="$HOME/.local/bin:$PATH" # current shell; future shells are updated nemo-speech --version ``` +With no version argument, the installer reads the current release identifier +from the repository's `VERSION` file, including prerelease identifiers. +Native Linux archives require glibc 2.31 or newer (Ubuntu 20.04 or equivalent). + The installer selects CUDA when `nvidia-smi` is available, Metal on Apple -Silicon, and CPU otherwise. Override that decision for a source build: +Silicon, and CPU otherwise. Override the backend or force a source build: ```bash -scripts/install.sh --source --backend cpu +curl -fsSL https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.sh | + sh -s -- --backend cpu +curl -fsSL https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.sh | + sh -s -- --source ``` +On Linux aarch64, the CUDA release is selected by platform and driver: +`cuda12` for Jetson Orin and `cuda13` for Jetson Thor or DGX Spark. Set +`NEMO_SPEECH_CUDA_SERIES=12` or `13` only when automatic detection is not +available. + It installs without `sudo` and links the CLI into `~/.local/bin`. Run `--help` to see prefix, backend, PATH, and dry-run options. Downloaded archives are -verified against their published SHA-256 files; an archive with an invalid or -mismatched checksum always fails rather than falling back to source. +verified against their published SHA-256 files; a present archive with an +invalid or mismatched checksum always fails rather than falling back to source. The source fallback requires Git, CMake 3.26 or newer, Ninja, a C++17 compiler, and the toolkit for the selected GPU backend. It clones only the submodules @@ -47,7 +55,7 @@ Inspect [`scripts/install.ps1`](../scripts/install.ps1), then run from PowerShell: ```powershell -.\scripts\install.ps1 -Source +irm https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.ps1 | iex nemo-speech --version ``` @@ -71,8 +79,8 @@ Select the components to install: |---|---| | `core` | ASR, diarization, and TTS | | `asr` | ASR and diarization | -| `server` (default) | `core` plus the HTTP API and playground | -| `full` | `server` plus NMT, gRPC, Flashlight, and JA/ZH tokenizers | +| `server` (default) | `core` plus NMT, the HTTP API, and playground | +| `full` | `server` plus gRPC, Flashlight, and JA/ZH tokenizers | Use `-Grpc`, `-Nmt`, `-Flashlight`, `-TtsJa`, `-TtsZh`, `-Http`, or `-HttpTls` to customize a profile. Binary installation is limited to `server`; other @@ -110,6 +118,8 @@ nemo-speech----.tar.gz nemo-speech--windows--.zip ``` +Linux aarch64 CUDA archives use `cuda12` or `cuda13` as the backend suffix. + To uninstall on Linux or macOS, remove the prefix printed during installation and `~/.local/bin/nemo-speech`; remove the two-line NeMo-Speech.cpp PATH entry from the shell startup file if the installer added it. On Windows, diff --git a/docs/server.md b/docs/server.md index 2b2b764..2baa2bc 100644 --- a/docs/server.md +++ b/docs/server.md @@ -6,11 +6,11 @@ The project provides two server executables over the same core C++ engines: playground. It loads configured models once into an `EngineRegistry`. - `riva_server` hosts the Riva-compatible gRPC services. -They are separate processes and do not share loaded model instances. The -source installer and `*-server` presets build the HTTP executable for ASR and -diarization plus TTS without the gRPC dependency chain. `cuda-full`, `developer`, -or explicit component options add NMT, optional language frontends, and -`riva_server` (presets: [build guide](build.md)). +They are separate processes and do not share loaded model instances. The source +installer and `*-server` presets build the HTTP executable for ASR, diarization, +NMT, and TTS without the gRPC dependency chain. `cuda-full`, `developer`, or +explicit component options add optional language frontends and `riva_server` +(presets: [build guide](build.md)). ```bash nemo-speech serve \ diff --git a/ggml-patches/0005-skinny-q8-gemm.patch b/ggml-patches/0005-skinny-q8-gemm.patch index 53339a5..2b7de6b 100644 --- a/ggml-patches/0005-skinny-q8-gemm.patch +++ b/ggml-patches/0005-skinny-q8-gemm.patch @@ -3,7 +3,7 @@ new file mode 100644 index 00000000..60147ff1 --- /dev/null +++ b/src/ggml-cuda/skinny-q8.cu -@@ -0,0 +1,608 @@ +@@ -0,0 +1,616 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// Skinny-N Q8_0 GEMM for streaming ASR encoders (nemo-speech). @@ -408,6 +408,12 @@ index 00000000..60147ff1 + if (planar_q8 && src0 != root) { + return false; + } ++ ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const bool skinny_q8_available = ampere_mma_available(cc); ++ if (!planar_q8 && !skinny_q8_available) { ++ return false; ++ } + // The repack cache is keyed by src0->data and assumes the bytes are + // immutable from the outside: only accept long-lived weight buffers, never + // transient compute-pool tensors (whose addresses get reused). @@ -435,7 +441,9 @@ index 00000000..60147ff1 + // because stock MMQ cannot interpret tensor-wide planes; skq8_gemm + // tiles arbitrary N in 64-column chunks. + GGML_ASSERT(tensor_ok && call_ok && "planar Q8 weight used in unsupported mul_mat shape"); -+ return total_n > MMVQ_MAX_BATCH_SIZE; ++ GGML_ASSERT((skinny_q8_available || total_n <= MMVQ_MAX_BATCH_SIZE) && ++ "wide planar Q8 requires an SM80+ CUDA kernel; use block Q8 on older GPUs"); ++ return skinny_q8_available && total_n > MMVQ_MAX_BATCH_SIZE; + } + + const bool repacked = [&]() { diff --git a/ggml-patches/README.md b/ggml-patches/README.md index 5dc6cb7..f3b7dca 100644 --- a/ggml-patches/README.md +++ b/ggml-patches/README.md @@ -110,7 +110,9 @@ stock comparison therefore requires both a pristine ggml checkout and use with `GGML_SKINNY_Q8_INPLACE=0` under a multi-stream scheduler). Accepts serialized tensor-planar Q8 weights (`GGML_TENSOR_FLAG_Q8_PLANAR`, see 0006) without a runtime repack. Kill - switch: `GGML_SKINNY_Q8=0`. The repack is in-place by default (reuses the + switch: `GGML_SKINNY_Q8=0`. Turing and older GPUs retain stock block-Q8 + matmul; wide planar Q8 fails explicitly because its tensor-wide layout has + no stock fallback. The repack is in-place by default (reuses the weight buffer, saving the ~1.07 GB cudaMalloc duplicate on parakeet-xxl), which is correct and fast for the streaming-ASR encoder runtime. Two caveats for the llama.cpp NMT decoder, which the NMT pipeline handles by diff --git a/kernels/cublas_shim.cu b/kernels/cublas_shim.cu index 3084a20..a8e8bff 100644 --- a/kernels/cublas_shim.cu +++ b/kernels/cublas_shim.cu @@ -1,6 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Drop-in libcublas.so.13 replacement for nemo-speech. +// Drop-in libcublas replacement for nemo-speech. // // ggml-cuda calls a small set of cuBLAS GEMM entry points for the matmuls that // are not quantized (FastConformer attention scores/context, the subsampling @@ -9,8 +9,8 @@ // specialized for the shapes used here (including WMMA tensor-core paths, but // no cuBLASLt), so the runtime needs neither real cuBLAS nor cuBLASLt. // -// Built as `libcublas.so.13` (SONAME + symbols versioned `libcublas.so.13`); -// it is the image's only libcublas. ggml is not modified. +// Its SONAME and symbol version match the CUDA toolkit used for the build; +// it is the release archive's only libcublas. ggml is not modified. // // cublas_v2.h is deliberately not included: it tags these functions // __host__ __device__ under nvcc. The cuBLAS enums/handles are passed at fixed diff --git a/kernels/ver_cublas.map b/kernels/ver_cublas.map index 396a3b2..76f1682 100644 --- a/kernels/ver_cublas.map +++ b/kernels/ver_cublas.map @@ -1,7 +1,7 @@ /* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ -libcublas.so.13 { +libcublas.so.@NEMO_SPEECH_CUBLAS_SOVERSION@ { global: cublasCreate_v2; cublasDestroy_v2; diff --git a/scripts/configure.sh b/scripts/configure.sh index dfc7d7b..d2bfe75 100755 --- a/scripts/configure.sh +++ b/scripts/configure.sh @@ -81,7 +81,7 @@ need_flashlight=OFF need_ja=OFF need_zh=OFF case "$PRESET" in - *-nmt|*-speech|cuda-full|developer) need_nmt=ON ;; + *-nmt|*-speech|*-server|cuda-full|developer) need_nmt=ON ;; esac case "$PRESET" in cuda-full|developer) need_grpc=ON ;; diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 6399d6a..a4b07c5 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -26,12 +26,17 @@ $ErrorActionPreference = "Stop" $releaseBase = if ($env:NEMO_SPEECH_RELEASE_BASE_URL) { $env:NEMO_SPEECH_RELEASE_BASE_URL.TrimEnd('/') } else { - "" + "https://github.com/NVIDIA/NeMo-Speech.cpp/releases" } $sourceUrl = if ($env:NEMO_SPEECH_SOURCE_URL) { $env:NEMO_SPEECH_SOURCE_URL } else { - (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + "https://github.com/NVIDIA/NeMo-Speech.cpp.git" +} +$versionUrl = if ($env:NEMO_SPEECH_VERSION_URL) { + $env:NEMO_SPEECH_VERSION_URL +} else { + "https://raw.githubusercontent.com/NVIDIA/NeMo-Speech.cpp/main/VERSION" } if ($Source -and $BinaryOnly) { throw "-Source and -BinaryOnly are mutually exclusive" } $profileIncludesTts = $Profile -ne 'asr' @@ -42,6 +47,20 @@ if (($TtsJa -or $TtsZh) -and -not $profileIncludesTts) { throw 'The Japanese and Mandarin tokenizers require TTS. Use a profile that includes TTS.' } +function Invoke-DownloadWithRetry { + param([string]$Uri, [string]$OutFile) + + for ($attempt = 1; $attempt -le 3; $attempt++) { + try { + Invoke-WebRequest -Uri $Uri -OutFile $OutFile + return + } catch { + if ($attempt -eq 3) { throw } + Start-Sleep -Seconds $attempt + } + } +} + function Assert-SourcePrerequisites { param([string]$SelectedBackend, [string]$Architecture) @@ -143,13 +162,11 @@ if ($Version -eq "latest") { Write-Host "No release endpoint is configured; building from the current source branch." } else { try { - $response = Invoke-WebRequest -Uri "$releaseBase/latest" -MaximumRedirection 10 - $location = if ($response.BaseResponse.RequestMessage) { - $response.BaseResponse.RequestMessage.RequestUri.AbsoluteUri - } else { - $response.BaseResponse.ResponseUri.AbsoluteUri + $manifest = (Invoke-WebRequest -Uri $versionUrl).Content + if ($manifest -notmatch '(?m)^NEMO_SPEECH_VERSION:\s*([^\s]+)\s*$') { + throw "VERSION does not contain NEMO_SPEECH_VERSION" } - $Version = ($location.TrimEnd('/') -split '/')[-1] + $Version = $Matches[1] } catch { if ($BinaryOnly) { throw "Could not resolve the latest release. $($_.Exception.Message)" } $binaryCandidate = $false @@ -209,8 +226,8 @@ try { $binaryReady = $false if (-not $Source -and $binaryCandidate) { try { - Invoke-WebRequest -Uri $url -OutFile $archivePath - Invoke-WebRequest -Uri "$url.sha256" -OutFile "$archivePath.sha256" + Invoke-DownloadWithRetry -Uri $url -OutFile $archivePath + Invoke-DownloadWithRetry -Uri "$url.sha256" -OutFile "$archivePath.sha256" $binaryReady = $true } catch { if ($BinaryOnly) { throw "Release artifact or checksum is unavailable. $($_.Exception.Message)" } diff --git a/scripts/install.sh b/scripts/install.sh index 739023f..6f89230 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 set -eu -release_base=${NEMO_SPEECH_RELEASE_BASE_URL:-} -source_url=${NEMO_SPEECH_SOURCE_URL:-.} +release_base=${NEMO_SPEECH_RELEASE_BASE_URL:-https://github.com/NVIDIA/NeMo-Speech.cpp/releases} +source_url=${NEMO_SPEECH_SOURCE_URL:-https://github.com/NVIDIA/NeMo-Speech.cpp.git} +version_url=${NEMO_SPEECH_VERSION_URL:-https://raw.githubusercontent.com/NVIDIA/NeMo-Speech.cpp/main/VERSION} version=latest channel=stable prefix= @@ -64,7 +65,8 @@ Usage: install.sh [options] -h, --help Set NEMO_SPEECH_RELEASE_BASE_URL to use a release mirror or local test server. -Set NEMO_SPEECH_SOURCE_URL or NEMO_SPEECH_SOURCE_REF to override the source. +Set NEMO_SPEECH_SOURCE_URL or NEMO_SPEECH_SOURCE_REF to override GitHub. +Set NEMO_SPEECH_VERSION_URL to override the current-version manifest. EOF } @@ -98,6 +100,10 @@ case "$machine" in arm64|aarch64) arch=aarch64 ;; *) echo "unsupported architecture: $machine" >&2; exit 1 ;; esac +device_model= +if [ -r /proc/device-tree/model ]; then + device_model=$(tr -d '\000' &2 @@ -113,11 +119,49 @@ if [ "$backend" = auto ]; then backend=metal elif command -v nvidia-smi >/dev/null 2>&1; then backend=cuda + elif [ "$os" = linux ] && [ "$arch" = aarch64 ]; then + case "$device_model" in + *jetson*|*thor*|*dgx*spark*|*gb10*) backend=cuda ;; + *) backend=cpu ;; + esac else backend=cpu fi fi +artifact_backend=$backend +if [ "$os" = linux ] && [ "$arch" = aarch64 ] && [ "$backend" = cuda ]; then + cuda_series=${NEMO_SPEECH_CUDA_SERIES:-} + if [ -z "$cuda_series" ]; then + case "$device_model" in + *orin*) cuda_series=12 ;; + *thor*|*dgx*spark*|*gb10*) cuda_series=13 ;; + esac + fi + if [ -z "$cuda_series" ] && command -v nvidia-smi >/dev/null 2>&1; then + driver_version=$(nvidia-smi --query-gpu=driver_version \ + --format=csv,noheader 2>/dev/null | sed -n '1p' || true) + driver_major=${driver_version%%.*} + case "$driver_major" in + ''|*[!0-9]*) ;; + *) [ "$driver_major" -ge 580 ] && cuda_series=13 || cuda_series=12 ;; + esac + fi + if [ -z "$cuda_series" ] && command -v nvcc >/dev/null 2>&1; then + cuda_series=$(nvcc --version 2>/dev/null | + sed -n 's/.*release \([0-9][0-9]*\)\..*/\1/p' | head -n 1) + fi + cuda_series=${cuda_series:-12} + case "$cuda_series" in + 12|13) ;; + *) + echo "NEMO_SPEECH_CUDA_SERIES must be 12 or 13; found '$cuda_series'." >&2 + exit 2 + ;; + esac + artifact_backend=cuda$cuda_series +fi + binary_candidate=1 [ -n "$release_base" ] || binary_candidate=0 if [ "$install_mode" = binary ] && [ "$binary_candidate" -eq 0 ]; then @@ -134,10 +178,10 @@ if [ "$version" = latest ]; then echo "No release endpoint is configured; building from the current source branch." else require_command curl "curl is required to resolve and download releases" - if effective=$(curl -fsSL -o /dev/null -w '%{url_effective}' "$release_base/latest"); then - version=${effective##*/} - if [ -z "$version" ]; then binary_candidate=0; fi - else + version_manifest=$(curl -fsSL "$version_url" 2>/dev/null || true) + version=$(printf '%s\n' "$version_manifest" | + sed -n 's/^NEMO_SPEECH_VERSION:[[:space:]]*//p' | head -n 1) + if [ -z "$version" ]; then binary_candidate=0 fi if [ "$binary_candidate" -eq 0 ]; then @@ -180,11 +224,11 @@ if [ -z "$prefix" ]; then fi fi bin_dir=$HOME/.local/bin -archive=nemo-speech-$release_version-$os-$arch-$backend.tar.gz +archive=nemo-speech-$release_version-$os-$arch-$artifact_backend.tar.gz url=$release_base/download/$tag/$archive checksum_url=$url.sha256 -echo "NeMo-Speech.cpp $release_version ($os/$arch, $backend)" +echo "NeMo-Speech.cpp $release_version ($os/$arch, $artifact_backend)" if [ "$install_mode" != source ] && [ "$binary_candidate" -eq 1 ]; then echo "Artifact: $url" fi @@ -194,8 +238,8 @@ fi echo "Prefix: $prefix" [ "$dry_run" -eq 0 ] || exit 0 -install_identity="$release_version $os $arch $backend" -source_identity="$install_identity source:$source_ref profile:speech-server" +install_identity="$release_version $os $arch $artifact_backend" +source_identity="$release_version $os $arch $backend source:$source_ref profile:speech-server" install_metadata=$prefix/.nemo-speech-install if [ "$install_mode" != source ] && [ "$binary_candidate" -eq 1 ] && [ -x "$prefix/bin/nemo-speech" ] && [ -f "$install_metadata" ] && @@ -302,6 +346,7 @@ else fi } initialize_submodule ggml + initialize_submodule llama.cpp initialize_submodule third_party/cpp-httplib root=$tmp/source-install @@ -332,7 +377,8 @@ rm -rf "$prefix.old" ln -sf "$prefix/bin/nemo-speech" "$bin_dir/nemo-speech" if [ "$modify_path" -eq 1 ] && ! printf '%s' ":$PATH:" | grep -q ":$bin_dir:"; then - shell_name=${SHELL##*/} + shell_name=${SHELL:-} + shell_name=${shell_name##*/} case "$shell_name" in zsh) rc=$HOME/.zshrc ;; bash) rc=$HOME/.bashrc ;; diff --git a/scripts/windows/build.ps1 b/scripts/windows/build.ps1 index 9aa0270..6557f43 100644 --- a/scripts/windows/build.ps1 +++ b/scripts/windows/build.ps1 @@ -20,7 +20,7 @@ config), so use a distinct -BuildDir for each. .PARAMETER Profile - Component preset: core, asr, server (core + HTTP), full, or developer + Component preset: core, asr, server (core + NMT + HTTP), full, or developer (full + tests, examples, and tools). Component switches add features. .PARAMETER Grpc @@ -133,7 +133,10 @@ $BuildTools = $false switch ($Profile) { 'core' { $BuildAsr = $true; $BuildDiar = $true; $BuildTts = $true } 'asr' { $BuildAsr = $true; $BuildDiar = $true } - 'server' { $BuildAsr = $true; $BuildDiar = $true; $BuildTts = $true; $BuildHttp = $true } + 'server' { + $BuildAsr = $true; $BuildDiar = $true; $BuildTts = $true + $BuildNmt = $true; $BuildHttp = $true + } 'full' { $BuildAsr = $true; $BuildDiar = $true; $BuildTts = $true; $BuildNmt = $true $BuildHttp = $true; $BuildGrpc = $true; $BuildFlashlight = $true diff --git a/src/asr/CMakeLists.txt b/src/asr/CMakeLists.txt index b410ba8..333db6e 100644 --- a/src/asr/CMakeLists.txt +++ b/src/asr/CMakeLists.txt @@ -87,6 +87,12 @@ if(SENTENCEPIECE_STATIC_LIB) target_include_directories(nemo_speech_asr PRIVATE ${SENTENCEPIECE_INCLUDE_DIR}) target_link_options( nemo_speech_asr PRIVATE "LINKER:--exclude-libs,libsentencepiece.a") + set(_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR + "${NEMO_SPEECH_DEPENDENCY_PREFIX}/sentencepiece/share/licenses/nemo-speech/third_party/sentencepiece") + if(EXISTS "${_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR}/LICENSE") + install(DIRECTORY "${_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR}/" + DESTINATION "${NEMO_SPEECH_LICENSE_INSTALL_DIR}/third_party/sentencepiece") + endif() elseif(NEMO_SPEECH_WITH_NORM AND UNIX AND NOT APPLE) message(FATAL_ERROR "ITN + Flashlight requires private static SentencePiece; " diff --git a/tests/install/install_ps1_test.py b/tests/install/install_ps1_test.py index a774a7f..2b7ee7f 100644 --- a/tests/install/install_ps1_test.py +++ b/tests/install/install_ps1_test.py @@ -127,12 +127,11 @@ def main() -> None: class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self) -> None: # noqa: N802 requests[self.path] = requests.get(self.path, 0) + 1 - if self.path == "/releases/latest": - self.send_response(302) - self.send_header("Location", "/releases/tag/v1.2.3") - self.end_headers() - return - body = b"latest\n" if self.path == "/releases/tag/v1.2.3" else releases.get(self.path) + body = ( + b"NEMO_SPEECH_VERSION: 1.2.3\n" + if self.path == "/VERSION" + else releases.get(self.path) + ) if body is None: self.send_error(404) return @@ -160,6 +159,9 @@ def log_message(self, _format: str, *_args: object) -> None: f"http://127.0.0.1:{server.server_port}/releases" ) environment["NEMO_SPEECH_SOURCE_URL"] = str(source) + environment["NEMO_SPEECH_VERSION_URL"] = ( + f"http://127.0.0.1:{server.server_port}/VERSION" + ) def run(*arguments: str, ok: bool = True) -> subprocess.CompletedProcess[str]: result = subprocess.run( diff --git a/tests/install/install_sh_test.py b/tests/install/install_sh_test.py index f460044..07bc141 100644 --- a/tests/install/install_sh_test.py +++ b/tests/install/install_sh_test.py @@ -112,13 +112,8 @@ def main() -> None: class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API requests[self.path] = requests.get(self.path, 0) + 1 - if self.path == "/releases/latest": - self.send_response(302) - self.send_header("Location", "/releases/tag/v1.2.3") - self.end_headers() - return - if self.path == "/releases/tag/v1.2.3": - body = b"latest\n" + if self.path == "/VERSION": + body = b"NEMO_SPEECH_VERSION: 1.2.3\n" else: body = releases.get(self.path) if body is None: @@ -152,6 +147,7 @@ def log_message(self, _format: str, *_args: object) -> None: f"http://127.0.0.1:{server.server_port}/releases" ), "NEMO_SPEECH_SOURCE_URL": str(source), + "NEMO_SPEECH_VERSION_URL": (f"http://127.0.0.1:{server.server_port}/VERSION"), } ) @@ -323,6 +319,49 @@ def run(*arguments: str, ok: bool = True) -> subprocess.CompletedProcess[str]: "--dry-run", ) require(not dry_prefix.exists(), "dry run changed the filesystem") + + fake_bin = root / "fake-aarch64-bin" + fake_bin.mkdir() + fake_uname = fake_bin / "uname" + fake_uname.write_text( + """#!/bin/sh +case "$1" in + -s) echo Linux ;; + -m) echo aarch64 ;; + *) exit 2 ;; +esac +""", + encoding="utf-8", + ) + fake_uname.chmod(0o755) + for cuda_series in ("12", "13"): + cuda_env = env.copy() + cuda_env["PATH"] = f"{fake_bin}:{cuda_env['PATH']}" + cuda_env["NEMO_SPEECH_CUDA_SERIES"] = cuda_series + result = subprocess.run( + [ + "sh", + str(installer), + "--prefix", + str(root / f"cuda{cuda_series}-dry-run"), + "--version", + "9.9.9", + "--backend", + "cuda", + "--binary-only", + "--dry-run", + ], + env=cuda_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + require(result.returncode == 0, f"CUDA selector failed:\n{result.stdout}") + require( + f"linux-aarch64-cuda{cuda_series}.tar.gz" in result.stdout, + f"CUDA {cuda_series} artifact was not selected", + ) finally: server.shutdown() server.server_close() From 96a12237d34feecc6ea3ffbb6fcef935839d8dc1 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Tue, 18 Aug 2026 18:54:37 +0000 Subject: [PATCH 02/11] feat(cli): add verbose/quiet inference session logging --- README.md | 5 +- app/bench.cpp | 1 + app/cli_util.cpp | 5 + app/main.cpp | 36 +++++-- app/serve.cpp | 1 + app/transcribe.cpp | 1 + docs/cli.md | 4 +- src/asr/decoders/flashlight_decoder.cpp | 6 +- src/asr/model.cpp | 44 +++++---- src/asr/postproc/pipeline.cpp | 6 +- src/asr/recognizer.cpp | 103 ++++++++++++++++++--- src/asr/recognizer.h | 8 ++ src/common/ggml_log_filter.h | 42 +++++++++ src/nmt/translator.cpp | 23 +---- src/runtime/ggml/logging.cpp | 35 +++---- src/runtime/ggml/runtime.h | 1 - src/tts/magpietts/magpietts.cpp | 13 ++- src/tts/magpietts/magpietts.h | 2 +- src/tts/magpietts/model.cpp | 59 ++++++------ src/tts/magpietts/model.h | 2 +- src/tts/magpietts/runtime.cpp | 2 +- src/tts/nanocodec/model.cpp | 36 ++++--- src/tts/nanocodec/model.h | 2 +- tests/cli/cli_contract_test.py | 8 ++ tests/cpp/CMakeLists.txt | 2 +- tests/cpp/common/test_shared_utilities.cpp | 12 +++ tests/cpp/tts/test_magpietts_asr.cpp | 3 +- tests/cpp/tts/test_magpietts_file.cpp | 3 +- 28 files changed, 320 insertions(+), 145 deletions(-) create mode 100644 src/common/ggml_log_filter.h diff --git a/README.md b/README.md index 078aefe..87fb997 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ # NeMo-Speech.cpp -A lightweight native C++ runtime for running NVIDIA speech and voice models locally, with broad hardware support. It supports speech recognition, speaker diarization, translation, and speech synthesis in realtime and batch mode. +A lightweight native C++ runtime for running NVIDIA Nemotron Speech model family locally, with broad hardware support. It supports multilingual speech recognition, speaker diarization, translation, and speech synthesis in realtime and batch mode. -It builds on speech models and tooling from [NVIDIA NeMo Speech](https://github.com/NVIDIA-NeMo/Speech). Native inference is powered by ggml. -NVIDIA's official local speech inference solution, with day-0 support for our latest speech models. +NeMo-Speech.cpp is NVIDIA's official local speech inference solution, with day-0 support for our latest speech models. It builds on models from [NVIDIA NeMo Speech](https://github.com/NVIDIA-NeMo/Speech), with native inference powered by [ggml](https://github.com/ggerganov/ggml). ## Models and applications diff --git a/app/bench.cpp b/app/bench.cpp index 84a355d..241907d 100644 --- a/app/bench.cpp +++ b/app/bench.cpp @@ -230,6 +230,7 @@ run_bench(int argc, char** argv) { std::max(options.config.batching.max_queue_depth, max_concurrency * 2); options.config.batching.state_arena_slots = std::max(options.config.batching.state_arena_slots, max_concurrency); + options.config.log_status = !cli_quiet() && !cli_json(); nemo_speech::EngineRegistry engines; const auto load_start = Clock::now(); diff --git a/app/cli_util.cpp b/app/cli_util.cpp index 017f080..b1f41dd 100644 --- a/app/cli_util.cpp +++ b/app/cli_util.cpp @@ -13,6 +13,8 @@ #include #include "audio_file.h" +#include "ggml.h" +#include "ggml_log_filter.h" #if defined(_WIN32) #include @@ -27,6 +29,7 @@ struct CliOutputState { }; CliOutputState output_state; +nemo_speech::GgmlLogFilter dependency_logs; } // namespace @@ -269,6 +272,8 @@ open_url_in_browser(const std::string& url) { void configure_cli_output(bool json, bool quiet, bool verbose) { output_state = {json, quiet, verbose}; + dependency_logs.set_verbose(verbose); + ggml_log_set(nemo_speech::GgmlLogFilter::callback, &dependency_logs); } bool diff --git a/app/main.cpp b/app/main.cpp index 53204dc..dbb96c7 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "cli_util.h" @@ -51,6 +52,25 @@ unavailable_command(const std::string& command) { return {}; } +template +int +run_session(const char* command, int argc, char** argv, Function&& function) { + if (argc > 2 && is_help_argument(argv[2])) + return std::forward(function)(); + const bool log_status = !cli_quiet() && !cli_json(); + if (log_status) + std::fprintf(stderr, "[nemo-speech] %s session started\n", command); + const int status = std::forward(function)(); + if (log_status) { + if (status == 0) + std::fprintf(stderr, "[nemo-speech] %s session finished\n", command); + else + std::fprintf( + stderr, "[nemo-speech] %s session failed (exit code %d)\n", command, status); + } + return status; +} + void print_help(const char* program) { std::printf( @@ -187,23 +207,27 @@ main(int argc, char** argv) { } #if defined(NEMO_SPEECH_CLI_ASR) if (std::strcmp(argv[1], "transcribe") == 0) - return command_transcribe(argc - 2, argv + 2); + return run_session( + "transcribe", argc, argv, [&] { return command_transcribe(argc - 2, argv + 2); }); #endif #if defined(NEMO_SPEECH_CLI_DIAR) if (std::strcmp(argv[1], "diarize") == 0) - return command_diarize(argc - 2, argv + 2); + return run_session( + "diarize", argc, argv, [&] { return command_diarize(argc - 2, argv + 2); }); #endif #if defined(NEMO_SPEECH_CLI_NMT) if (std::strcmp(argv[1], "translate") == 0) - return command_translate(argc - 2, argv + 2); + return run_session( + "translate", argc, argv, [&] { return command_translate(argc - 2, argv + 2); }); #endif #if defined(NEMO_SPEECH_CLI_TTS) if (std::strcmp(argv[1], "synthesize") == 0) - return command_synthesize(argc - 2, argv + 2); + return run_session( + "synthesize", argc, argv, [&] { return command_synthesize(argc - 2, argv + 2); }); #endif #if defined(NEMO_SPEECH_CLI_ASR) if (std::strcmp(argv[1], "bench") == 0) - return command_bench(argc - 2, argv + 2); + return run_session("bench", argc, argv, [&] { return command_bench(argc - 2, argv + 2); }); #endif if (std::strcmp(argv[1], "model") == 0) return command_model(argc - 2, argv + 2); @@ -213,7 +237,7 @@ main(int argc, char** argv) { if (std::strcmp(argv[1], "health") == 0) return command_health(argc - 2, argv + 2); if (std::strcmp(argv[1], "serve") == 0) - return command_serve(argc - 2, argv + 2); + return run_session("serve", argc, argv, [&] { return command_serve(argc - 2, argv + 2); }); #endif const std::string unavailable = unavailable_command(argv[1]); if (!unavailable.empty()) diff --git a/app/serve.cpp b/app/serve.cpp index 9552c1b..c0dfd8f 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -460,6 +460,7 @@ run_server(int argc, char** argv) { nemo_speech::EngineRegistryConfig registry_config; #if defined(NEMO_SPEECH_CLI_ASR) + asr_config.log_status = !cli_quiet() && !cli_json(); registry_config.asr = !asr_path.empty(); #endif #if defined(NEMO_SPEECH_CLI_NMT) diff --git a/app/transcribe.cpp b/app/transcribe.cpp index c169d1c..ec79134 100644 --- a/app/transcribe.cpp +++ b/app/transcribe.cpp @@ -413,6 +413,7 @@ command_transcribe(int argc, char** argv) { inputs.size()); asr::RecognizerConfig config = options.engine; + config.log_status = !cli_quiet() && !cli_json(); if (options.device_set) config.backend.gpu = options.gpu; config.model.path = diff --git a/docs/cli.md b/docs/cli.md index 42c8dd5..af00958 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -33,7 +33,9 @@ overflow within the common 42-character subtitle limit. Plain results are written to stdout. Progress and diagnostics are written to stderr so output can be redirected safely. Global `--json`, `--quiet`, and -`--verbose` options work across commands. +`--verbose` options work across commands. The default output keeps command +lifecycle, effective inference configuration, results, warnings, and errors. +Model-loader, backend, ggml, and llama.cpp diagnostics require `--verbose`. ### Transcribe a directory diff --git a/src/asr/decoders/flashlight_decoder.cpp b/src/asr/decoders/flashlight_decoder.cpp index a02b6af..4531952 100644 --- a/src/asr/decoders/flashlight_decoder.cpp +++ b/src/asr/decoders/flashlight_decoder.cpp @@ -19,6 +19,7 @@ #include "flashlight/lib/text/decoder/lm/KenLM.h" #include "flashlight/lib/text/dictionary/Dictionary.h" #include "flashlight/lib/text/dictionary/Utils.h" +#include "runtime.h" namespace nemo_speech::asr { @@ -243,8 +244,9 @@ FlashlightDecoder::ensure_private_resources() { // + trie for OOV boost words. The trie is rebuilt (KenLM-score + insert + // smear over the whole lexicon) since flashlight's Trie has no deep-copy - // a one-time per-stream cost paid only on an OOV boost. - std::cerr << "[flashlight] OOV speech_context: building private lexicon trie " - "(one-time for this stream; shared KenLM reused, no reload)\n"; + GGMLF_LOG_INFO( + "[flashlight] OOV speech_context: building private lexicon trie " + "(one-time for this stream; shared KenLM reused, no reload)\n"); adopt_resources(shared_resources_->clone_for_mutation()); shared_resources_.reset(); // now private: shared_resources_ == nullptr means owned } diff --git a/src/asr/model.cpp b/src/asr/model.cpp index a2aea5f..cac4953 100644 --- a/src/asr/model.cpp +++ b/src/asr/model.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -1268,11 +1269,15 @@ AsrModel::AsrModel(ggml_runtime::BackendManager& bm, Common&& c, const BatchingC const auto st = sp->LoadFromSerializedProto(proto); if (st.ok()) { spm_ = std::move(sp); - std::cerr << "[asr_model] embedded SentencePiece tokenizer loaded (" - << spm_->GetPieceSize() << " pieces) for word boosting\n"; + GGMLF_LOG_INFO( + "[asr_model] embedded SentencePiece tokenizer loaded (%d pieces) for word " + "boosting\n", + spm_->GetPieceSize()); } else { - std::cerr << "[asr_model] failed to load embedded SentencePiece tokenizer: " - << st.ToString() << " (word boosting degraded)\n"; + GGMLF_LOG_WARN( + "[asr_model] failed to load embedded SentencePiece tokenizer: %s " + "(word boosting degraded)\n", + st.ToString().c_str()); } } } @@ -1353,11 +1358,12 @@ AsrModel::load( const char* head_name = head == HeadKind::Ctc ? "ctc" : (head == HeadKind::Tdt ? "tdt" : "rnnt"); - std::cerr << "[asr_model] arch=" << arch << " head=" << head_name - << " d_model=" << c.enc_cfg.d_model << " n_layers=" << c.enc_cfg.n_layers - << " n_heads=" << c.enc_cfg.n_heads << " d_ff=" << c.enc_cfg.d_ff - << " k=" << c.enc_cfg.conv_kernel_size << " feat_in=" << c.enc_cfg.feat_in - << " sr=" << c.fe_cfg.sample_rate << " vocab=" << c.vocab.size() << "\n"; + GGMLF_LOG_INFO( + "[asr_model] arch=%s head=%s d_model=%d n_layers=%d n_heads=%d d_ff=%d k=%d " + "feat_in=%d sr=%d vocab=%zu\n", + arch.c_str(), head_name, c.enc_cfg.d_model, c.enc_cfg.n_layers, c.enc_cfg.n_heads, + c.enc_cfg.d_ff, c.enc_cfg.conv_kernel_size, c.enc_cfg.feat_in, c.fe_cfg.sample_rate, + c.vocab.size()); // `new` rather than make_unique: Common is a protected nested type, which // std::make_unique (in namespace std) cannot name for template deduction; @@ -1473,7 +1479,7 @@ class CtcModel::CtcBatcher { CtcModel::CtcModel(ggml_runtime::BackendManager& bm, Common&& c, const BatchingConfig& batching) : AsrModel(bm, std::move(c), batching) { ctc_cfg_ = load_ctc_cfg(*loader(), ns_, enc_cfg_); - std::cerr << "[asr_model] ctc classes=" << ctc_cfg_.num_classes << "\n"; + GGMLF_LOG_INFO("[asr_model] ctc classes=%d\n", ctc_cfg_.num_classes); // Full utterances amortize the GPU frontend's launch overhead. offline_fe_ = std::make_unique(fe_cfg_, &bm, batching); @@ -1582,14 +1588,15 @@ CtcModel::diagnostic_sessions() const { RnntModel::RnntModel(ggml_runtime::BackendManager& bm, Common&& c, const BatchingConfig& batching) : AsrModel(bm, std::move(c), batching), batching_cfg_(batching) { rnnt_cfg_ = load_rnnt_cfg(*loader(), enc_cfg_); - std::cerr << "[asr_model] rnnt vocab=" << rnnt_cfg_.vocab_size - << " blank=" << rnnt_cfg_.blank_id << " pred_hidden=" << rnnt_cfg_.pred_hidden - << " joint_dim=" << rnnt_cfg_.joint_dim; + std::ostringstream model_info; + model_info << "[asr_model] rnnt vocab=" << rnnt_cfg_.vocab_size + << " blank=" << rnnt_cfg_.blank_id << " pred_hidden=" << rnnt_cfg_.pred_hidden + << " joint_dim=" << rnnt_cfg_.joint_dim; if (rnnt_cfg_.is_tdt()) { - std::cerr << " durations="; - for (int duration : rnnt_cfg_.durations) std::cerr << duration << ','; + model_info << " durations="; + for (int duration : rnnt_cfg_.durations) model_info << duration << ','; } - std::cerr << "\n"; + GGMLF_LOG_INFO("%s\n", model_info.str().c_str()); // Prompt fusion and joint.enc execute together once per encoder chunk. num_prompts_ = static_cast(loader()->get_u32("asr.rnnt.num_prompts", 0)); @@ -1603,8 +1610,9 @@ RnntModel::RnntModel(ggml_runtime::BackendManager& bm, Common&& c, const Batchin } } if (num_prompts_ > 0 && loader()->has_tensor("prompt_kernel.0.weight")) { - std::cerr << "[asr_model] prompt fusion enabled: num_prompts=" << num_prompts_ - << " languages=" << prompt_dictionary_.size() << "\n"; + GGMLF_LOG_INFO( + "[asr_model] prompt fusion enabled: num_prompts=%d languages=%zu\n", num_prompts_, + prompt_dictionary_.size()); prompt_fusion_ = std::make_unique(rnnt_cfg_.d_model, num_prompts_); } else { num_prompts_ = 0; // no prompt_kernel in this GGUF -> disable fusion diff --git a/src/asr/postproc/pipeline.cpp b/src/asr/postproc/pipeline.cpp index dcd4a8c..1aeb22c 100644 --- a/src/asr/postproc/pipeline.cpp +++ b/src/asr/postproc/pipeline.cpp @@ -22,6 +22,7 @@ #include "itn_align.h" #include "pnc_model.h" #include "pnc_runner.h" +#include "runtime.h" namespace nemo_speech::asr::postproc { namespace { @@ -126,8 +127,9 @@ struct Postprocessor::ItnRegistry { throw std::runtime_error( "ITN: no two-FAR grammar directories found under " + root.string()); } - std::cerr << "[itn] discovered " << grammar_dirs.size() - << " language grammar directories under " << root.string() << "\n"; + GGMLF_LOG_INFO( + "[itn] discovered %zu language grammar directories under %s\n", grammar_dirs.size(), + root.string().c_str()); #else // Preserve the existing warning for a configured grammar in a build // without text-normalization support. diff --git a/src/asr/recognizer.cpp b/src/asr/recognizer.cpp index 3699234..253272b 100644 --- a/src/asr/recognizer.cpp +++ b/src/asr/recognizer.cpp @@ -23,6 +23,19 @@ namespace nemo_speech::asr { namespace { +const char* +head_name(HeadKind head) { + switch (head) { + case HeadKind::Ctc: + return "ctc"; + case HeadKind::Rnnt: + return "rnnt"; + case HeadKind::Tdt: + return "tdt"; + } + return "unknown"; +} + std::unique_ptr make_backend(int gpu_idx) { ggml_runtime::Params p; @@ -91,7 +104,7 @@ Recognizer::Recognizer(RecognizerConfig cfg) (cfg_.vad.masker.mask_enable || (cfg_.endpointing.enable && cfg_.endpointing.vad_based)); if (need_vad) { vad_model_ = std::make_shared(*bm_, cfg_.vad.model_path, cfg_.batching); - std::cerr << "[recognizer] VAD weights/session loaded once and shared across streams\n"; + GGMLF_LOG_INFO("[recognizer] VAD weights/session loaded once and shared across streams\n"); } #ifdef NEMO_SPEECH_WITH_FLASHLIGHT // Share the loaded language model and lexicon trie across streams. @@ -106,24 +119,85 @@ Recognizer::Recognizer(RecognizerConfig cfg) fcfg.embedded_spm = ctc->embedded_tokenizer(); flashlight_resources_ = std::make_shared(ctc->ctc_config(), ctc->vocab(), fcfg); - std::cerr << "[recognizer] flashlight resources loaded (lm + lexicon trie, shared)\n"; + GGMLF_LOG_INFO("[recognizer] flashlight resources loaded (lm + lexicon trie, shared)\n"); } #endif - std::cerr << "[recognizer] streaming cfg: chunk=" << cfg_.streaming.chunk_size - << "s left=" << cfg_.streaming.ctc_left_padding - << "s right=" << cfg_.streaming.ctc_right_padding << "s\n"; if (!cfg_.diar.model_path.empty()) { diar_model_ = std::make_unique(*bm_, cfg_.diar.model_path, cfg_.batching); const DiarGeometry geo = cfg_.diar.resolved_geometry(); - std::cerr << "[recognizer] diarizer loaded: " << cfg_.diar.model_path - << " (spkcache=" << geo.spkcache_len << " fifo=" << geo.fifo_len - << " chunk=" << geo.chunk_len << " rc=" << geo.chunk_right_context << ")\n"; + GGMLF_LOG_INFO( + "[recognizer] diarizer loaded: %s (spkcache=%d fifo=%d chunk=%d rc=%d)\n", + cfg_.diar.model_path.c_str(), geo.spkcache_len, geo.fifo_len, geo.chunk_len, + geo.chunk_right_context); } + log_model_status(); } Recognizer::~Recognizer() = default; +void +Recognizer::log_model_status() const { + if (!cfg_.log_status) + return; + const ggml_backend_t gpu = bm_->gpu_backend_handle(); + std::string summary = "[asr] model=" + model_name_ + " head=" + head_name(model_->head_kind()) + + " backend=" + (gpu != nullptr ? ggml_backend_name(gpu) : "CPU"); + if (vad_model_) + summary += " vad=on"; + if (diar_model_) + summary += " diarization=on"; +#ifdef NEMO_SPEECH_WITH_FLASHLIGHT + if (flashlight_resources_) + summary += " decoder=flashlight"; +#endif + std::fprintf(stderr, "%s\n", summary.c_str()); +} + +void +Recognizer::log_execution_status(bool streaming) const { + if (!cfg_.log_status) + return; + auto emit = [this, streaming] { + const HeadKind head = model_->head_kind(); + if (!streaming) { + const auto& enc = model_->encoder_config(); + if (enc.offline_left_ctx < 0 && enc.offline_right_ctx < 0) { + std::fprintf( + stderr, "[asr] mode=offline head=%s attention=full-context\n", head_name(head)); + } else { + std::fprintf( + stderr, + "[asr] mode=offline head=%s attention-left=%d " + "attention-right=%d encoder-frames\n", + head_name(head), enc.offline_left_ctx, enc.offline_right_ctx); + } + return; + } + if (head == HeadKind::Ctc) { + const float window = cfg_.streaming.ctc_left_padding + cfg_.streaming.chunk_size + + cfg_.streaming.ctc_right_padding; + std::fprintf( + stderr, + "[asr] mode=streaming head=ctc chunk=%.2fs left=%.2fs right=%.2fs window=%.2fs\n", + cfg_.streaming.chunk_size, cfg_.streaming.ctc_left_padding, + cfg_.streaming.ctc_right_padding, window); + return; + } + const EncoderConfig enc = + make_cache_aware_config(model_->encoder_config(), cfg_.streaming.rnnt_right_context); + const int center = enc.cache_chunk_frames - enc.cache_right_ctx; + const double step_ms = enc.cache_chunk_frames * model_->ms_per_enc_frame(); + std::fprintf( + stderr, + "[asr] mode=streaming head=%s left=%d center=%d right=%d attention=%d " + "encoder-frames step=%.0fms\n", + head_name(head), enc.cache_left_ctx, center, enc.cache_right_ctx, + enc.cache_left_ctx + enc.cache_chunk_frames, step_ms); + }; + std::call_once(streaming ? streaming_status_once_ : offline_status_once_, std::move(emit)); +} + BatchMetrics Recognizer::vad_batch_metrics() const { return vad_model_ ? vad_model_->batch_metrics() : BatchMetrics{}; @@ -227,7 +301,7 @@ Recognizer::warmup() { run_batch_warmup(); run_batch_warmup(); - std::cerr << "[recognizer] warmed streaming batch shape B=" << warmup_batch << "\n"; + GGMLF_LOG_INFO("[recognizer] warmed streaming batch shape B=%d\n", warmup_batch); } // Warm the diarizer Session's graph shapes too: the streaming warmup @@ -463,6 +537,7 @@ Recognizer::streaming_recognize( AsrRequestOptions opts, const std::string& language_code, bool coordinate_ingress) { opts.language_code = language_code; auto runner = make_runner(); + log_execution_status(/*streaming=*/true); if (model_->has_prompt()) runner->set_prompt_index(model_->prompt_index_for_lang(language_code)); return std::make_unique( @@ -483,15 +558,21 @@ Recognizer::recognize( const bool supports_streaming = model_->head_kind() == HeadKind::Ctc || static_cast(model_.get())->supports_cache_streaming(); + // Vulkan RNNT support was originally validated on the cache-aware graph. + // Keep that compatibility route until the offline graph has Vulkan parity + // coverage; CTC and offline-only TDT remain offline. + const bool use_streaming = + (exceeds_offline_limit || (vulkan && model_->head_kind() != HeadKind::Ctc)) && + supports_streaming; std::unique_ptr runner; - if ((exceeds_offline_limit || (vulkan && model_->head_kind() != HeadKind::Ctc)) && - supports_streaming) { + if (use_streaming) { runner = make_runner(); } else { // Past the positional-encoding limit OfflineRunner splits the audio at // quiet points and decodes segment by segment. runner = std::make_unique(model_.get(), cfg_, flashlight_resources_); } + log_execution_status(use_streaming); if (model_->has_prompt()) runner->set_prompt_index(model_->prompt_index_for_lang(language_code)); opts.language_code = language_code; diff --git a/src/asr/recognizer.h b/src/asr/recognizer.h index 7a00b5c..e830a8c 100644 --- a/src/asr/recognizer.h +++ b/src/asr/recognizer.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,9 @@ struct RecognizerConfig { VadEndpointerCfg endpointing; DiarConfig diar; // speaker diarization sidecar (Sortformer) postproc::PostprocConfig postproc; + // Human-readable model and execution summaries. The CLI disables these + // for --quiet and --json; verbose implementation logs are separate. + bool log_status = true; void Register(common::ParameterParser& p) { p.Register("backend", backend); @@ -139,6 +143,8 @@ class Recognizer { private: friend class RecognitionStream; + void log_model_status() const; + void log_execution_status(bool streaming) const; void register_streaming_ingress() { active_streaming_ingress_.fetch_add(1, std::memory_order_relaxed); } @@ -166,6 +172,8 @@ class Recognizer { std::unique_ptr postproc_; // Shared immutable VAD model; each stream owns recurrent state. std::shared_ptr vad_model_; + mutable std::once_flag offline_status_once_; + mutable std::once_flag streaming_status_once_; }; } // namespace nemo_speech::asr diff --git a/src/common/ggml_log_filter.h b/src/common/ggml_log_filter.h new file mode 100644 index 0000000..faf619d --- /dev/null +++ b/src/common/ggml_log_filter.h @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include + +#include "ggml.h" + +namespace nemo_speech { + +class GgmlLogFilter { + public: + void set_verbose(bool verbose) { + verbose_.store(verbose, std::memory_order_relaxed); + continue_log_.store(false, std::memory_order_relaxed); + } + + bool should_emit(ggml_log_level level) { + if (level == GGML_LOG_LEVEL_CONT) + return verbose_.load(std::memory_order_relaxed) || + continue_log_.load(std::memory_order_relaxed); + + const bool emit = verbose_.load(std::memory_order_relaxed) || level >= GGML_LOG_LEVEL_ERROR; + continue_log_.store(emit, std::memory_order_relaxed); + return emit; + } + + static void callback(ggml_log_level level, const char* text, void* user_data) { + auto& filter = *static_cast(user_data); + if (filter.should_emit(level)) { + std::fputs(text, stderr); + std::fflush(stderr); + } + } + + private: + std::atomic verbose_{false}; + std::atomic continue_log_{false}; +}; + +} // namespace nemo_speech diff --git a/src/nmt/translator.cpp b/src/nmt/translator.cpp index 7565852..b95cab1 100644 --- a/src/nmt/translator.cpp +++ b/src/nmt/translator.cpp @@ -3,7 +3,6 @@ #include "translator.h" #include -#include #include #include #include @@ -11,6 +10,7 @@ #include #include +#include "ggml_log_filter.h" #include "langpairs.h" #include "llama.h" @@ -18,27 +18,12 @@ namespace nemo_speech::nmt { namespace { -std::atomic verbose_llama_logs{false}; -std::atomic continue_llama_log{false}; - -void -llama_log_callback(ggml_log_level level, const char* text, void*) { - bool emit = verbose_llama_logs.load(std::memory_order_relaxed); - if (level == GGML_LOG_LEVEL_CONT) - emit = emit || continue_llama_log.load(std::memory_order_relaxed); - else { - emit = emit || level >= GGML_LOG_LEVEL_WARN; - continue_llama_log.store(emit, std::memory_order_relaxed); - } - if (emit) - std::fputs(text, stderr); -} +GgmlLogFilter llama_logs; void configure_llama_logging(bool verbose) { - verbose_llama_logs.store(verbose, std::memory_order_relaxed); - continue_llama_log.store(false, std::memory_order_relaxed); - llama_log_set(llama_log_callback, nullptr); + llama_logs.set_verbose(verbose); + llama_log_set(GgmlLogFilter::callback, &llama_logs); } void diff --git a/src/runtime/ggml/logging.cpp b/src/runtime/ggml/logging.cpp index d170c8d..036d73a 100644 --- a/src/runtime/ggml/logging.cpp +++ b/src/runtime/ggml/logging.cpp @@ -13,49 +13,38 @@ namespace ggml_runtime { -#include -#include -#include -#include - -void -log_callback_default(ggml_log_level level, const char* text, void* user_data) { - (void)level; - (void)user_data; - fputs(text, stderr); - fflush(stderr); -} - -struct LogState { - ggml_log_callback log_callback = log_callback_default; - void* log_callback_user_data = nullptr; -}; - -static LogState g_log_state; - GGML_ATTRIBUTE_FORMAT(5, 6) void log_internal( ggml_log_level level, const char* file, int line, const char* func, const char* format, ...) { va_list args; va_start(args, format); + va_list args_copy; + va_copy(args_copy, args); char buffer[1024]; int len = vsnprintf(buffer, 1024, format, args); if (len < 1024) { char formatted_buffer[2048]; snprintf( formatted_buffer, sizeof(formatted_buffer), "%s:%d:<%s> %s", file, line, func, buffer); - g_log_state.log_callback(level, formatted_buffer, g_log_state.log_callback_user_data); + ggml_log_callback callback; + void* user_data; + ggml_log_get(&callback, &user_data); + callback(level, formatted_buffer, user_data); } else { char* buffer2 = new char[len + 1]; - vsnprintf(buffer2, len + 1, format, args); + vsnprintf(buffer2, len + 1, format, args_copy); buffer2[len] = 0; char formatted_buffer[4096]; snprintf( formatted_buffer, sizeof(formatted_buffer), "%s:%d:<%s> %s", file, line, func, buffer2); - g_log_state.log_callback(level, formatted_buffer, g_log_state.log_callback_user_data); + ggml_log_callback callback; + void* user_data; + ggml_log_get(&callback, &user_data); + callback(level, formatted_buffer, user_data); delete[] buffer2; } + va_end(args_copy); va_end(args); } diff --git a/src/runtime/ggml/runtime.h b/src/runtime/ggml/runtime.h index 7b9cd69..4864071 100644 --- a/src/runtime/ggml/runtime.h +++ b/src/runtime/ggml/runtime.h @@ -41,7 +41,6 @@ namespace ggml_runtime { GGMLF_ATTRIBUTE_FORMAT(5, 6) void log_internal( ggml_log_level level, const char* file, int line, const char* func, const char* format, ...); -void log_callback_default(ggml_log_level level, const char* text, void* user_data); } // namespace ggml_runtime // Kept in the global namespace because llama_file and other global-scope diff --git a/src/tts/magpietts/magpietts.cpp b/src/tts/magpietts/magpietts.cpp index 43cccb8..00448ba 100644 --- a/src/tts/magpietts/magpietts.cpp +++ b/src/tts/magpietts/magpietts.cpp @@ -25,6 +25,7 @@ #include "audio_pp.h" #include "decoder.h" #include "encoder.h" +#include "ggml_log_filter.h" #include "lt.h" #include "nvtx_utils.h" #include "token_utils.h" @@ -293,6 +294,10 @@ class MagpieStreamingRuntime::Impl { std::unique_ptr workspace; }; +namespace { +GgmlLogFilter magpie_ggml_logs; +} + MagpieStreamingRuntime::MagpieStreamingRuntime() : impl_(std::make_unique()) {} MagpieStreamingRuntime::~MagpieStreamingRuntime() = default; @@ -300,12 +305,14 @@ MagpieStreamingRuntime::~MagpieStreamingRuntime() = default; bool MagpieStreamingRuntime::load( const std::string& magpie_model, const std::string& codec_model, magpietts_uma_mode uma_mode, - bool magpie_cpu, bool codec_cpu) { + bool magpie_cpu, bool codec_cpu, bool verbose) { + magpie_ggml_logs.set_verbose(verbose); + ggml_log_set(GgmlLogFilter::callback, &magpie_ggml_logs); impl_->workspace.reset(); - if (!impl_->magpie.load(magpie_model, uma_mode, magpie_cpu)) { + if (!impl_->magpie.load(magpie_model, uma_mode, magpie_cpu, verbose)) { return false; } - if (!impl_->codec.load(codec_model, codec_cpu)) { + if (!impl_->codec.load(codec_model, codec_cpu, verbose)) { impl_->magpie.reset(); return false; } diff --git a/src/tts/magpietts/magpietts.h b/src/tts/magpietts/magpietts.h index 41534ae..40e4074 100644 --- a/src/tts/magpietts/magpietts.h +++ b/src/tts/magpietts/magpietts.h @@ -139,7 +139,7 @@ class MagpieStreamingRuntime { bool load( const std::string& magpie_model, const std::string& codec_model, - magpietts_uma_mode uma_mode, bool magpie_cpu, bool codec_cpu); + magpietts_uma_mode uma_mode, bool magpie_cpu, bool codec_cpu, bool verbose = false); int sampleRate() const; int speakerCount() const; std::vector speakerNames() const; diff --git a/src/tts/magpietts/model.cpp b/src/tts/magpietts/model.cpp index e7ad6a4..de1ee04 100644 --- a/src/tts/magpietts/model.cpp +++ b/src/tts/magpietts/model.cpp @@ -26,14 +26,6 @@ namespace nemo_speech::tts { -static void -magpietts_log_callback(ggml_log_level level, const char* text, void* user_data) { - (void)level; - (void)user_data; - fputs(text, stderr); - fflush(stderr); -} - const char* magpietts_backend_preference_name(magpietts_backend_preference backend) { switch (backend) { @@ -602,7 +594,8 @@ load_transformer( } static bool magpietts_model_load_impl( - const std::string& fname, magpietts_model& model, magpietts_uma_mode uma_mode, bool force_cpu); + const std::string& fname, magpietts_model& model, magpietts_uma_mode uma_mode, bool force_cpu, + bool verbose); MagpieModel::~MagpieModel() { reset(); @@ -651,8 +644,9 @@ MagpieModel::operator=(MagpieModel&& other) noexcept { } bool -MagpieModel::load(const std::string& fname, magpietts_uma_mode uma_mode, bool force_cpu) { - if (!magpietts_model_load_impl(fname, *this, uma_mode, force_cpu)) { +MagpieModel::load( + const std::string& fname, magpietts_uma_mode uma_mode, bool force_cpu, bool verbose) { + if (!magpietts_model_load_impl(fname, *this, uma_mode, force_cpu, verbose)) { reset(); return false; } @@ -695,10 +689,10 @@ MagpieModel::reset() { static bool magpietts_model_load_impl( - const std::string& fname, magpietts_model& model, magpietts_uma_mode uma_mode, bool force_cpu) { + const std::string& fname, magpietts_model& model, magpietts_uma_mode uma_mode, bool force_cpu, + bool verbose) { const ggml_nvtx::range nvtx_range("magpietts_model_load"); model.reset(); - ggml_log_set(magpietts_log_callback, nullptr); gguf_init_params params = { /*.no_alloc =*/true, @@ -820,13 +814,16 @@ magpietts_model_load_impl( } ggml_backend_dev_t dev = ggml_backend_get_device(model.backend); - fprintf( - stderr, "MagpieTTS backend: %s%s%s%s\n", ggml_backend_name(model.backend), dev ? " - " : "", - dev ? ggml_backend_dev_description(dev) : "", force_cpu ? " (forced CPU)" : ""); - if (magpietts_backend_is_cuda(model.backend)) { + if (verbose) { fprintf( - stderr, "MagpieTTS CUDA managed memory: %s (uma-mode=%s)\n", - model.cuda_unified_memory ? "on" : "off", magpietts_uma_mode_name(uma_mode)); + stderr, "MagpieTTS backend: %s%s%s%s\n", ggml_backend_name(model.backend), + dev ? " - " : "", dev ? ggml_backend_dev_description(dev) : "", + force_cpu ? " (forced CPU)" : ""); + if (magpietts_backend_is_cuda(model.backend)) { + fprintf( + stderr, "MagpieTTS CUDA managed memory: %s (uma-mode=%s)\n", + model.cuda_unified_memory ? "on" : "off", magpietts_uma_mode_name(uma_mode)); + } } model.buffer = ggml_backend_alloc_ctx_tensors(model.ctx, model.backend); @@ -920,17 +917,19 @@ magpietts_model_load_impl( model, "local_transformer_out_projections." + std::to_string(i) + ".bias"); } - fprintf( - stderr, - "loaded MagpieTTS GGUF: text_vocab=%d audio_codebooks=%d audio_vocab=%d speakers=%d " - "attention_prior=%s epsilon=%.4g lookahead=%d start_step=%d advance_threshold=%d " - "decay_threshold=%d estimate_layers=%s apply_layers=%s\n", - h.text_vocab_size, h.audio_codebooks, h.audio_vocab_size, h.baked_speakers, - h.apply_attention_prior ? "on" : "off", h.attention_prior_epsilon, - h.attention_prior_lookahead_window, h.start_prior_after_n_audio_steps, - h.attention_prior_advance_threshold, h.attention_prior_decay_threshold, - format_i32_list(h.estimate_alignment_from_layers).c_str(), - format_i32_list(h.apply_prior_to_layers).c_str()); + if (verbose) { + fprintf( + stderr, + "loaded MagpieTTS GGUF: text_vocab=%d audio_codebooks=%d audio_vocab=%d speakers=%d " + "attention_prior=%s epsilon=%.4g lookahead=%d start_step=%d advance_threshold=%d " + "decay_threshold=%d estimate_layers=%s apply_layers=%s\n", + h.text_vocab_size, h.audio_codebooks, h.audio_vocab_size, h.baked_speakers, + h.apply_attention_prior ? "on" : "off", h.attention_prior_epsilon, + h.attention_prior_lookahead_window, h.start_prior_after_n_audio_steps, + h.attention_prior_advance_threshold, h.attention_prior_decay_threshold, + format_i32_list(h.estimate_alignment_from_layers).c_str(), + format_i32_list(h.apply_prior_to_layers).c_str()); + } return true; } diff --git a/src/tts/magpietts/model.h b/src/tts/magpietts/model.h index c123527..e667f7b 100644 --- a/src/tts/magpietts/model.h +++ b/src/tts/magpietts/model.h @@ -198,7 +198,7 @@ class MagpieModel { bool load( const std::string& fname, magpietts_uma_mode uma_mode = MAGPIETTS_UMA_AUTO, - bool force_cpu = false); + bool force_cpu = false, bool verbose = false); void reset(); bool loaded() const { return gguf != nullptr && ctx != nullptr && backend != nullptr; } diff --git a/src/tts/magpietts/runtime.cpp b/src/tts/magpietts/runtime.cpp index 5d086cd..0209687 100644 --- a/src/tts/magpietts/runtime.cpp +++ b/src/tts/magpietts/runtime.cpp @@ -108,7 +108,7 @@ class MagpieTtsRuntime::Impl { stream_ = std::make_unique(); if (!stream_->load( config_.magpie_model, config_.codec_model, to_internal(config_.uma_mode), - config_.magpie_cpu, config_.codec_cpu)) { + config_.magpie_cpu, config_.codec_cpu, config_.verbose)) { throw std::runtime_error("failed to load MagpieTTS/NanoCodec GGUFs"); } diff --git a/src/tts/nanocodec/model.cpp b/src/tts/nanocodec/model.cpp index d542b64..0e20242 100644 --- a/src/tts/nanocodec/model.cpp +++ b/src/tts/nanocodec/model.cpp @@ -22,14 +22,6 @@ static constexpr int NANO_CODEC_MAX_NODES = 32768; -static void -nano_codec_log_callback(ggml_log_level level, const char* text, void* user_data) { - (void)level; - (void)user_data; - fputs(text, stderr); - fflush(stderr); -} - using nc_hparams = nemo_speech::tts::nanocodec::NanoCodecHParams; static bool @@ -196,9 +188,9 @@ load_conv(const nc_model& model, const std::string& prefix, int stride = 1, int } static bool -nc_model_load(const std::string& fname, nc_model& model, bool force_cpu = false) { +nc_model_load( + const std::string& fname, nc_model& model, bool force_cpu = false, bool verbose = false) { const ggml_nvtx::range nvtx_range("nanocodec_model_load"); - ggml_log_set(nano_codec_log_callback, nullptr); gguf_init_params params = { /*.no_alloc =*/true, @@ -245,9 +237,12 @@ nc_model_load(const std::string& fname, nc_model& model, bool force_cpu = false) } ggml_backend_dev_t dev = ggml_backend_get_device(model.backend); - fprintf( - stderr, "NanoCodec backend: %s%s%s%s\n", ggml_backend_name(model.backend), dev ? " - " : "", - dev ? ggml_backend_dev_description(dev) : "", force_cpu ? " (forced CPU)" : ""); + if (verbose) { + fprintf( + stderr, "NanoCodec backend: %s%s%s%s\n", ggml_backend_name(model.backend), + dev ? " - " : "", dev ? ggml_backend_dev_description(dev) : "", + force_cpu ? " (forced CPU)" : ""); + } model.buffer = ggml_backend_alloc_ctx_tensors(model.ctx, model.backend); if (!model.buffer) { @@ -320,10 +315,13 @@ nc_model_load(const std::string& fname, nc_model& model, bool force_cpu = false) model.post_activation = load_activation(model, "dec.post_act"); model.post_conv = load_conv(model, "dec.post"); - fprintf( - stderr, - "loaded NanoCodec GGUF: sample_rate=%d codebooks=%d codebook_size=%d frame=%d samples\n", - h.sample_rate, h.num_codebooks, h.codebook_size, h.samples_per_frame); + if (verbose) { + fprintf( + stderr, + "loaded NanoCodec GGUF: sample_rate=%d codebooks=%d codebook_size=%d frame=%d " + "samples\n", + h.sample_rate, h.num_codebooks, h.codebook_size, h.samples_per_frame); + } return true; } @@ -1106,12 +1104,12 @@ NanoCodecModel::operator=(NanoCodecModel&& other) noexcept { } bool -NanoCodecModel::load(const std::string& path, bool force_cpu) { +NanoCodecModel::load(const std::string& path, bool force_cpu, bool verbose) { if (!impl_) { impl_ = std::make_unique(); } reset(); - impl_->loaded = nc_model_load(path, impl_->model, force_cpu); + impl_->loaded = nc_model_load(path, impl_->model, force_cpu, verbose); if (!impl_->loaded) { reset(); } diff --git a/src/tts/nanocodec/model.h b/src/tts/nanocodec/model.h index ab5a599..cb85afd 100644 --- a/src/tts/nanocodec/model.h +++ b/src/tts/nanocodec/model.h @@ -43,7 +43,7 @@ class NanoCodecModel { NanoCodecModel(const NanoCodecModel&) = delete; NanoCodecModel& operator=(const NanoCodecModel&) = delete; - bool load(const std::string& path, bool force_cpu = false); + bool load(const std::string& path, bool force_cpu = false, bool verbose = false); void reset(); bool loaded() const; diff --git a/tests/cli/cli_contract_test.py b/tests/cli/cli_contract_test.py index 3c7b322..1a619d0 100644 --- a/tests/cli/cli_contract_test.py +++ b/tests/cli/cli_contract_test.py @@ -151,6 +151,14 @@ def stall_response() -> None: transcribe_help = run(binary, "transcribe", "--help") if transcribe_help.returncode == 0: assert "--backend" in transcribe_help.stdout + assert "session" not in transcribe_help.stderr + lifecycle = run(binary, "transcribe") + assert lifecycle.returncode == 2, lifecycle.stdout + lifecycle.stderr + assert "[nemo-speech] transcribe session started" in lifecycle.stderr + assert "[nemo-speech] transcribe session failed (exit code 2)" in lifecycle.stderr + quiet_lifecycle = run(binary, "--quiet", "transcribe") + assert "session" not in quiet_lifecycle.stderr + expect_json_error(run(binary, "--json", "transcribe"), 2, "invalid_argument") synthesize_help = run(binary, "synthesize", "--help") if synthesize_help.returncode == 0: diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index e951ea6..57bdcc3 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -12,7 +12,7 @@ add_executable(test_shared_utilities ${CMAKE_SOURCE_DIR}/app/cli_util.cpp) target_include_directories(test_shared_utilities PRIVATE ${CMAKE_SOURCE_DIR}/app) target_link_libraries(test_shared_utilities PRIVATE - nemo_speech_common nemo_speech_engine_registry) + nemo_speech_common nemo_speech_engine_registry ggml) add_test(NAME shared_utilities COMMAND test_shared_utilities) add_executable(test_subtitles diff --git a/tests/cpp/common/test_shared_utilities.cpp b/tests/cpp/common/test_shared_utilities.cpp index 7bc30e5..b51e0be 100644 --- a/tests/cpp/common/test_shared_utilities.cpp +++ b/tests/cpp/common/test_shared_utilities.cpp @@ -16,6 +16,7 @@ #include "audio_file.h" #include "cli_util.h" #include "engine_registry.h" +#include "ggml_log_filter.h" #include "json.h" namespace { @@ -146,6 +147,17 @@ main() { registry.set_device_label("cpu"); require(registry.device_label() == "cpu", "engine registry device label"); + nemo_speech::GgmlLogFilter log_filter; + log_filter.set_verbose(false); + require(!log_filter.should_emit(GGML_LOG_LEVEL_INFO), "default info log filtering"); + require(!log_filter.should_emit(GGML_LOG_LEVEL_CONT), "filtered continuation log"); + require(!log_filter.should_emit(GGML_LOG_LEVEL_WARN), "default warning log filtering"); + require(log_filter.should_emit(GGML_LOG_LEVEL_ERROR), "error log retention"); + require(log_filter.should_emit(GGML_LOG_LEVEL_CONT), "error continuation log"); + log_filter.set_verbose(true); + require(log_filter.should_emit(GGML_LOG_LEVEL_DEBUG), "verbose debug logging"); + require(log_filter.should_emit(GGML_LOG_LEVEL_CONT), "verbose continuation log"); + namespace fs = std::filesystem; const fs::path root = fs::temp_directory_path() / "nemo-speech-input"; require( diff --git a/tests/cpp/tts/test_magpietts_asr.cpp b/tests/cpp/tts/test_magpietts_asr.cpp index ac7f0a9..dc8fce1 100644 --- a/tests/cpp/tts/test_magpietts_asr.cpp +++ b/tests/cpp/tts/test_magpietts_asr.cpp @@ -414,7 +414,8 @@ main(int argc, char** argv) { tts::MagpieStreamingRuntime tts_runtime; if (!tts_runtime.load( - params.magpie_model, params.codec_model, params.uma_mode, false, params.codec_cpu)) { + params.magpie_model, params.codec_model, params.uma_mode, false, params.codec_cpu, + params.verbose)) { return 1; } diff --git a/tests/cpp/tts/test_magpietts_file.cpp b/tests/cpp/tts/test_magpietts_file.cpp index f78d78a..23d63cf 100644 --- a/tests/cpp/tts/test_magpietts_file.cpp +++ b/tests/cpp/tts/test_magpietts_file.cpp @@ -253,7 +253,8 @@ main(int argc, char** argv) { tts::MagpieStreamingRuntime runtime; if (!runtime.load( - params.magpie_model, params.codec_model, params.uma_mode, false, params.codec_cpu)) { + params.magpie_model, params.codec_model, params.uma_mode, false, params.codec_cpu, + params.verbose)) { return 1; } From 73f45ef49d6661bb2172c085bded4b6903df7fc6 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Tue, 18 Aug 2026 12:53:18 -0700 Subject: [PATCH 03/11] feat(windows): add cublas shim support for win/cuda builds --- CMakeLists.txt | 46 +++++++++++++++---------------- docs/development/cublas-shim.md | 30 ++++++++++++-------- docs/development/windows-build.md | 5 ++-- kernels/cublas_shim.cu | 31 +++++++++++++-------- scripts/windows/build.ps1 | 10 ++++++- 5 files changed, 71 insertions(+), 51 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb93442..9c1248a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,11 +155,10 @@ if(NEMO_SPEECH_BUILD_DIAR AND NOT NEMO_SPEECH_BUILD_ASR) "without the transcribe CLI/API surface") endif() -# Drop-in cuBLAS shim (native GEMM, no cuBLASLt). The shipping container image -# substitutes it for real cuBLAS to reduce its runtime closure; put it on -# LD_LIBRARY_PATH ahead of the system cuBLAS to reproduce that GEMM path. +# Drop-in cuBLAS shim (native GEMM, no cuBLASLt). Release builds can substitute +# it for real cuBLAS to reduce their runtime closure. # Disabled by default for source builds, which link the CUDA toolkit's cuBLAS. -# Container builds enable the shim explicitly to reduce the runtime image size. +# Portable builds enable the shim explicitly. # It is only built when GGML_CUDA is also ON (see the target below), and is a # no-op for Metal, Vulkan, and CPU builds. option(NEMO_SPEECH_CUBLAS_SHIM "Build the in-tree drop-in cuBLAS shim (native GEMM, no cuBLASLt)" OFF) @@ -281,13 +280,9 @@ if(GGML_METAL) endif() # In-tree cuBLAS shim backed by native GEMM kernels (no cuBLASLt). ggml-cuda -# links real cuBLAS at build time, but the shim shares its major-version SONAME, -# so binaries resolve it instead when it is first on LD_LIBRARY_PATH. -# The cuBLAS shim is a Linux-only container size optimization. It relies on ELF -# SONAME and symbol versioning plus a GNU-ld --version-script, neither of which -# exists with MSVC/link.exe. On Windows, ggml-cuda links the real cuBLAS DLL -# from the CUDA toolkit, so the shim is neither needed nor buildable - skip it. -if(GGML_CUDA AND NEMO_SPEECH_CUBLAS_SHIM AND NOT WIN32) +# links real cuBLAS at build time. At runtime the loader resolves its cuBLAS +# dependency to this target's matching major-version library name. +if(GGML_CUDA AND NEMO_SPEECH_CUBLAS_SHIM) enable_language(CUDA) if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.0) set(NEMO_SPEECH_CUBLAS_SOVERSION 12) @@ -303,20 +298,25 @@ if(GGML_CUDA AND NEMO_SPEECH_CUBLAS_SHIM AND NOT WIN32) set(NEMO_SPEECH_CUBLAS_SHIM_ARCHITECTURES "80-virtual") endif() set_target_properties(nemo_speech_cublas_shim PROPERTIES - OUTPUT_NAME cublas - SOVERSION "${NEMO_SPEECH_CUBLAS_SOVERSION}" CUDA_ARCHITECTURES "${NEMO_SPEECH_CUBLAS_SHIM_ARCHITECTURES}" + CUDA_RUNTIME_LIBRARY Static CUDA_SEPARABLE_COMPILATION ON) - configure_file( - kernels/ver_cublas.map - "${CMAKE_CURRENT_BINARY_DIR}/ver_cublas.map" - @ONLY) - target_link_options(nemo_speech_cublas_shim PRIVATE - "LINKER:--version-script=${CMAKE_CURRENT_BINARY_DIR}/ver_cublas.map") -elseif(GGML_CUDA AND NEMO_SPEECH_CUBLAS_SHIM AND WIN32) - message(STATUS - "NEMO_SPEECH_CUBLAS_SHIM: skipped on Windows; linking the CUDA " - "toolkit's real cuBLAS instead.") + if(WIN32) + # The toolkit import library records this exact DLL name. App-local + # deployment therefore substitutes the shim without changing ggml. + set_target_properties(nemo_speech_cublas_shim PROPERTIES + OUTPUT_NAME "cublas64_${NEMO_SPEECH_CUBLAS_SOVERSION}") + else() + set_target_properties(nemo_speech_cublas_shim PROPERTIES + OUTPUT_NAME cublas + SOVERSION "${NEMO_SPEECH_CUBLAS_SOVERSION}") + configure_file( + kernels/ver_cublas.map + "${CMAKE_CURRENT_BINARY_DIR}/ver_cublas.map" + @ONLY) + target_link_options(nemo_speech_cublas_shim PRIVATE + "LINKER:--version-script=${CMAKE_CURRENT_BINARY_DIR}/ver_cublas.map") + endif() endif() if(NEMO_SPEECH_WITH_FLASHLIGHT) diff --git a/docs/development/cublas-shim.md b/docs/development/cublas-shim.md index 560ef5d..b37f623 100644 --- a/docs/development/cublas-shim.md +++ b/docs/development/cublas-shim.md @@ -7,20 +7,20 @@ by an in-tree drop-in `libcublas`. ## The shim -`kernels/cublas_shim.cu` (with the generated symbol map from -`kernels/ver_cublas.map`) is a drop-in cuBLAS library: shape-specialized CUDA -GEMM/GEMV kernels, including WMMA tensor-core paths, but **no cuBLASLt**. It inherits -`CMAKE_CUDA_ARCHITECTURES` when set and falls back to JIT-portable `compute_80` -PTX for ad-hoc builds. Dropping real cuBLAS + cuBLASLt is the bulk of the -container size. The shim is built as a separate library from ggml. +`kernels/cublas_shim.cu` is a drop-in cuBLAS library: shape-specialized CUDA +GEMM/GEMV kernels, including WMMA tensor-core paths, but **no cuBLASLt**. +Linux uses the generated symbol map from `kernels/ver_cublas.map`; Windows exports +the same ABI from `cublas64_.dll`. The target inherits +`CMAKE_CUDA_ARCHITECTURES` when set and falls back to JIT-portable +`compute_80` PTX for ad-hoc builds. Dropping real cuBLAS and cuBLASLt is +the bulk of the package size. The shim is built separately from ggml. It's an optional CMake target, **`NEMO_SPEECH_CUBLAS_SHIM` (default `OFF`)**, -built when explicitly enabled together with `GGML_CUDA` (Linux only, -auto-skipped on Windows; a no-op for Metal, Vulkan, and CPU builds). Normal -source builds therefore link the CUDA toolkit's cuBLAS and cuBLASLt. Container -and release-archive builds enable the shim explicitly and skip those libraries -in their runtime closure. The generated SONAME and symbol version match the -CUDA toolkit major used for the build. +built when explicitly enabled with `GGML_CUDA` (a no-op for Metal, Vulkan, +and CPU builds). Normal source builds link the CUDA toolkit's cuBLAS and +cuBLASLt. Portable container and release-archive builds enable the shim and +omit those libraries from their runtime closure. Linux uses a matching SONAME +and symbol version; Windows uses the matching versioned DLL name. To build and exercise the container GEMM path outside the container, enable the shim and put its output directory first on the loader path: @@ -32,6 +32,12 @@ LD_LIBRARY_PATH=$PWD/build/cuda-asr/bin \ ./build/cuda-asr/bin/nemo-speech transcribe audio.wav --model model.gguf ``` +On Windows: + +```powershell +.\scripts\windows\build.ps1 -Backend cuda -CublasShim +``` + ## Custom GPU kernels The heavier project-specific CUDA kernels (fused rel-pos attention, skinny-Q8 GEMM, diff --git a/docs/development/windows-build.md b/docs/development/windows-build.md index fb9cf99..a7435de 100644 --- a/docs/development/windows-build.md +++ b/docs/development/windows-build.md @@ -150,9 +150,8 @@ cmake --build build-vulkan --parallel ### Windows-specific build behavior -- **cuBLAS shim is auto-skipped** on Windows (it's a Linux `.so` size hack using - a GNU-ld version script). ggml-cuda links the toolkit's real cuBLAS instead; - `-DNEMO_SPEECH_CUBLAS_SHIM=ON` is a no-op here. +- **The cuBLAS shim is optional.** Pass `-CublasShim` to the build driver for an + app-local `cublas64_.dll` that avoids shipping cuBLAS and cuBLASLt. - **ggml patches are CUDA-only.** A Vulkan/CPU build uses stock ggml; pass `-DNEMO_SPEECH_GGML_PATCHED=OFF` (the encoder uses the portable op path). - DLLs export their symbols via `WINDOWS_EXPORT_ALL_SYMBOLS` (the C ABI libs use diff --git a/kernels/cublas_shim.cu b/kernels/cublas_shim.cu index a8e8bff..54e1475 100644 --- a/kernels/cublas_shim.cu +++ b/kernels/cublas_shim.cu @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,12 @@ typedef int cublasSideMode_t; typedef int cublasFillMode_t; typedef int cublasDiagType_t; +#if defined(_WIN32) +#define NEMO_SPEECH_CUBLAS_EXPORT __declspec(dllexport) +#else +#define NEMO_SPEECH_CUBLAS_EXPORT +#endif + namespace { struct ShimHandle { struct SplitKWorkspace { @@ -1234,7 +1241,7 @@ host_scalar(const void* p, int ct) { else bits = (s << 31) | ((e - 15 + 127) << 23) | (m << 13); float f; - __builtin_memcpy(&f, &bits, 4); + std::memcpy(&f, &bits, 4); return f; } return *(const float*)p; @@ -1372,7 +1379,7 @@ launch( extern "C" { -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasCreate_v2(cublasHandle_t* h) { auto* sh = new ShimHandle{}; if (cudaGetDevice(&sh->device) != cudaSuccess) { @@ -1382,7 +1389,7 @@ cublasCreate_v2(cublasHandle_t* h) { *h = sh; return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasDestroy_v2(cublasHandle_t h) { ShimHandle* sh = (ShimHandle*)h; if (sh == nullptr) { @@ -1409,23 +1416,23 @@ cublasDestroy_v2(cublasHandle_t h) { delete sh; return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasSetStream_v2(cublasHandle_t h, cudaStream_t s) { ShimHandle* sh = (ShimHandle*)h; std::lock_guard lock(sh->mutex); sh->stream = s; return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasSetMathMode(cublasHandle_t, cublasMath_t) { return STATUS_SUCCESS; } -const char* +NEMO_SPEECH_CUBLAS_EXPORT const char* cublasGetStatusString(cublasStatus_t) { return "EDGE_SHIM_OK"; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasGemmEx( cublasHandle_t h, cublasOperation_t opA, cublasOperation_t opB, int m, int n, int k, const void* alpha, const void* A, cudaDataType ta, int lda, const void* B, cudaDataType tb, @@ -1438,7 +1445,7 @@ cublasGemmEx( host_scalar(beta, ct), 1, stream, sh); return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasGemmStridedBatchedEx( cublasHandle_t h, cublasOperation_t opA, cublasOperation_t opB, int m, int n, int k, const void* alpha, const void* A, cudaDataType ta, int lda, long long sa, const void* B, @@ -1451,7 +1458,7 @@ cublasGemmStridedBatchedEx( host_scalar(beta, ct), batch, stream, sh); return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasGemmBatchedEx( cublasHandle_t h, cublasOperation_t opA, cublasOperation_t opB, int m, int n, int k, const void* alpha, const void* const Aarray[], cudaDataType ta, int lda, @@ -1465,7 +1472,7 @@ cublasGemmBatchedEx( host_scalar(alpha, ct), host_scalar(beta, ct), batch); return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasSgemm_v2( cublasHandle_t h, cublasOperation_t opA, cublasOperation_t opB, int m, int n, int k, const float* alpha, const float* A, int lda, const float* B, int ldb, const float* beta, @@ -1477,7 +1484,7 @@ cublasSgemm_v2( stream, sh); return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasSgemmStridedBatched( cublasHandle_t h, cublasOperation_t opA, cublasOperation_t opB, int m, int n, int k, const float* alpha, const float* A, int lda, long long sa, const float* B, int ldb, @@ -1489,7 +1496,7 @@ cublasSgemmStridedBatched( batch, stream, sh); return STATUS_SUCCESS; } -cublasStatus_t +NEMO_SPEECH_CUBLAS_EXPORT cublasStatus_t cublasStrsmBatched( cublasHandle_t, cublasSideMode_t, cublasFillMode_t, cublasOperation_t, cublasDiagType_t, int, int, const float*, const float* const[], int, float* const[], int, int) { diff --git a/scripts/windows/build.ps1 b/scripts/windows/build.ps1 index 6557f43..5b81ed3 100644 --- a/scripts/windows/build.ps1 +++ b/scripts/windows/build.ps1 @@ -63,6 +63,9 @@ "89" (Ada/RTX 40xx), "86" (Ampere/RTX 30xx), "120" (Blackwell). Set a concrete value (not native) when building to ship to other GPUs. +.PARAMETER CublasShim + Build the app-local cuBLAS replacement used by portable CUDA packages. + .PARAMETER Compiler C/C++ compiler: auto (default), msvc, or clang-cl. auto picks cl on x64 and clang-cl on ARM64 (ggml's ARM CPU backend rejects MSVC). nvcc always uses @@ -94,6 +97,7 @@ param( [ValidateSet('Release', 'RelWithDebInfo', 'Debug')] [string]$Config = 'Release', [string]$CudaArch = 'native', + [switch]$CublasShim, [string]$VcpkgRoot, [string]$VcpkgTriplet, [ValidateSet('auto', 'x64', 'arm64')] @@ -189,6 +193,9 @@ $CrossCompiling = ($HostArch -eq 'ARM64') -ne ($TargetArch -eq 'arm64') if ($Backend -eq 'cuda' -and $CrossCompiling) { throw 'CUDA cross-compilation is not supported by this driver; build CUDA natively on the target architecture.' } +if ($CublasShim -and $Backend -ne 'cuda') { + throw '-CublasShim requires -Backend cuda.' +} $VcpkgArch = $TargetArch if (-not $VcpkgTriplet) { # Link vcpkg libraries statically so installed binaries do not depend on the @@ -205,7 +212,7 @@ if ($BuildExamples) { $VcpkgFeatures.Add('examples') } Write-Host "==> nemo-speech Windows build" -ForegroundColor Cyan Write-Host " backend=$Backend profile=$Profile asr=$BuildAsr diar=$BuildDiar tts=$BuildTts nmt=$BuildNmt http=$BuildHttp grpc=$BuildGrpc flashlight=$BuildFlashlight tts-ja=$BuildTtsJa tts-zh=$BuildTtsZh tests=$BuildTests examples=$BuildExamples tools=$BuildTools" -Write-Host " config=$Config compiler=$Compiler host=$HostArch target=$TargetArch build=$BuildDir jobs=$Jobs" +Write-Host " config=$Config compiler=$Compiler host=$HostArch target=$TargetArch build=$BuildDir jobs=$Jobs cublas-shim=$($CublasShim.IsPresent)" Write-Host " vcpkg=$($VcpkgFeatures -join ',') triplet=$VcpkgTriplet" if ($DryRun) { return } @@ -420,6 +427,7 @@ switch ($Backend) { 'cuda' { $cmakeArgs += '-DGGML_CUDA=ON' $cmakeArgs += '-DGGML_VULKAN=OFF' + $cmakeArgs += "-DNEMO_SPEECH_CUBLAS_SHIM=$(ConvertTo-CMakeBool $CublasShim.IsPresent)" $cmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$CudaArch" } 'vulkan' { From ee0d31496ce8f220026ea89fb336c9769411a581 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 04:23:45 +0530 Subject: [PATCH 04/11] feat(cli): add live transcribe and managed model downloads - add portable microphone capture with miniaudio - add indexed model aliases, defaults, caching, and verified downloads - fall back unsupported blas ops to CPU --- CMakeLists.txt | 13 + README.md | 32 +- THIRD_PARTY_NOTICES.md | 13 + app/CMakeLists.txt | 29 +- app/bench.cpp | 4 +- app/commands.h | 1 + app/diarize.cpp | 6 +- app/doctor.cpp | 15 + app/main.cpp | 13 +- app/microphone_capture.cpp | 127 ++++ app/microphone_capture.h | 33 + app/model.cpp | 69 +- app/model_store.cpp | 1171 ++++++++++++++++++++++++++++++ app/model_store.h | 24 + app/model_utils.cpp | 13 + app/model_utils.h | 5 + app/serve.cpp | 58 +- app/synthesize.cpp | 19 +- app/transcribe.cpp | 274 +++++-- docker/Dockerfile | 7 +- docs/asr/models.md | 22 +- docs/build.md | 3 +- docs/cli.md | 86 ++- docs/install.md | 24 +- docs/tts/models.md | 45 +- examples/CMakeLists.txt | 2 +- kernels/cublas_shim.cu | 2 +- models/index.json | 189 +++++ scripts/configure.sh | 7 + scripts/install.ps1 | 3 + scripts/install.sh | 6 +- scripts/windows/build.ps1 | 6 +- src/runtime/ggml/session.cpp | 26 +- tests/cli/cli_contract_test.py | 19 +- tests/cli/model_store_test.py | 292 ++++++++ tests/cpp/CMakeLists.txt | 18 +- tests/install/install_sh_test.py | 8 +- third_party/miniaudio/LICENSE | 18 + 38 files changed, 2512 insertions(+), 190 deletions(-) create mode 100644 app/microphone_capture.cpp create mode 100644 app/microphone_capture.h create mode 100644 app/model_store.cpp create mode 100644 app/model_store.h create mode 100644 models/index.json create mode 100644 tests/cli/model_store_test.py create mode 100644 third_party/miniaudio/LICENSE diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c1248a..cbce88c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,14 @@ install(DIRECTORY docs/ DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/nemo-speech/docs") install(DIRECTORY config/ DESTINATION "${CMAKE_INSTALL_DATADIR}/nemo-speech/config") +file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/share/nemo-speech") +configure_file( + models/index.json + "${CMAKE_BINARY_DIR}/share/nemo-speech/model-index.json" + COPYONLY) +install(FILES models/index.json + DESTINATION "${CMAKE_INSTALL_DATADIR}/nemo-speech" + RENAME model-index.json) # Windows: stop - pulled in transitively by the CUDA headers when # GGML_CUDA=ON - from defining the min()/max() macros, which otherwise clobber @@ -95,6 +103,7 @@ option(NEMO_SPEECH_BUILD_DIAR "Build standalone and ASR-integrated diarizat option(NEMO_SPEECH_BUILD_TTS "Build text-to-speech" ON) option(NEMO_SPEECH_BUILD_NMT "Build text translation (links llama.cpp)" ${NEMO_SPEECH_WITH_NMT}) option(NEMO_SPEECH_BUILD_CLI "Build the unified nemo-speech CLI" ON) +option(NEMO_SPEECH_BUILD_MIC_CAPTURE "Build microphone capture in the CLI and examples" ON) option(NEMO_SPEECH_BUILD_HTTP "Build the HTTP server and local playground" OFF) option(NEMO_SPEECH_HTTP_TLS "Enable TLS support in the HTTP server (requires OpenSSL)" OFF) option(NEMO_SPEECH_BUILD_GRPC "Build Riva-compatible gRPC adapters" ${NEMO_SPEECH_WITH_GRPC}) @@ -455,6 +464,10 @@ if(DEFINED VCPKG_INSTALLED_DIR AND DEFINED VCPKG_TARGET_TRIPLET) endif() install(FILES ggml/LICENSE DESTINATION "${NEMO_SPEECH_THIRD_PARTY_LICENSE_DIR}/ggml") +if(NEMO_SPEECH_BUILD_ASR AND NEMO_SPEECH_BUILD_CLI AND NEMO_SPEECH_BUILD_MIC_CAPTURE) + install(FILES third_party/miniaudio/LICENSE + DESTINATION "${NEMO_SPEECH_THIRD_PARTY_LICENSE_DIR}/miniaudio") +endif() if(TARGET nemo_speech_cublas_shim) install(TARGETS nemo_speech_cublas_shim RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} diff --git a/README.md b/README.md index 87fb997..550315a 100644 --- a/README.md +++ b/README.md @@ -45,23 +45,27 @@ options. ## Quick start -Download the ready-to-run Q8 GGUF from the model's Hugging Face repository, -then transcribe the bundled sample: +Transcribe the bundled sample. On first use, the CLI downloads the pinned +default Nemotron 3.5 GGUF from Hugging Face and verifies its size and SHA-256: ```bash -hf download nvidia/nemotron-speech-streaming-en-0.6b \ - nemotron-speech-streaming-en-0.6b.q8_0.gguf \ - --local-dir models +nemo-speech transcribe test_files/asr/wav/test/jfk.wav +``` + +The same command can transcribe the default microphone on builds that include +live capture: -nemo-speech transcribe test_files/asr/wav/test/jfk.wav \ - --model models/nemotron-speech-streaming-en-0.6b.q8_0.gguf +```bash +nemo-speech transcribe --live ``` -Install the `hf` command with `pip install -U huggingface_hub` if needed. The -CLI selects an available backend and handles common mono or stereo PCM WAV -sample rates automatically. Substitute your own WAV file after verifying the -bundled sample. See [ASR models](docs/asr/models.md) for the other published -GGUFs and [model conversion](docs/model-conversion.md) for custom checkpoints. +Run `nemo-speech model list` to see defaults, short names, and which command +uses each model. For example, `nemo-speech pull nemotron-en` downloads the +English-only model ahead of time, and `--model nemotron-en` selects it. Local +GGUF paths continue to work without downloading anything. The CLI selects an +available backend and handles common mono or stereo PCM WAV sample rates +automatically. See the [CLI model guide](docs/cli.md#models-and-cache) and +[model conversion](docs/model-conversion.md) for custom checkpoints. ## Command line @@ -76,7 +80,7 @@ Start the same runtime as a local HTTP service and open the playground: ```bash nemo-speech serve \ - --asr-model models/nemotron-speech-streaming-en-0.6b.q8_0.gguf \ + --asr-model nemotron-3.5 \ --open ``` @@ -106,7 +110,7 @@ Requires CMake 3.26 or newer, Ninja, C and C++17 compilers, and a supported CUDA toolkit. For a CUDA ASR and TTS server with the playground: ```bash -git submodule update --init ggml third_party/cpp-httplib +git submodule update --init ggml llama.cpp third_party/cpp-httplib scripts/configure.sh cuda-server cmake --build --preset cuda-server ``` diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 4b07117..f543c65 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -141,6 +141,19 @@ applicable agreement is installed with the archive under ## Other incorporated third-party code and data +### miniaudio + +- Source: [`mackron/miniaudio`](https://github.com/mackron/miniaudio), version + 0.11.25, vendored by the pinned llama.cpp checkout +- Path: `llama.cpp/vendor/miniaudio/miniaudio.h` +- Copyright 2026 David Reid +- License: MIT No Attribution (MIT-0); upstream text is reproduced at + [`third_party/miniaudio/LICENSE`](third_party/miniaudio/LICENSE) + +The command-line microphone capture layer compiles miniaudio directly into +`nemo-speech`. Release archives install its license under +`share/licenses/nemo-speech/third_party/miniaudio/`. + ### SentencePiece - Source: [`google/sentencepiece`](https://github.com/google/sentencepiece), diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index ab96132..02aeb43 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -6,6 +6,7 @@ add_executable(nemo_speech_cli cli_util.cpp doctor.cpp model.cpp + model_store.cpp model_utils.cpp ) set_target_properties(nemo_speech_cli PROPERTIES OUTPUT_NAME nemo-speech) @@ -21,9 +22,33 @@ if(WIN32) endif() if(NEMO_SPEECH_BUILD_ASR) - target_sources(nemo_speech_cli PRIVATE bench.cpp transcribe.cpp) + find_package(Threads REQUIRED) + target_sources(nemo_speech_cli PRIVATE + bench.cpp + transcribe.cpp) target_compile_definitions(nemo_speech_cli PRIVATE NEMO_SPEECH_CLI_ASR=1) - target_link_libraries(nemo_speech_cli PRIVATE nemo_speech_asr) + target_link_libraries(nemo_speech_cli PRIVATE + nemo_speech_asr + Threads::Threads) + if(NEMO_SPEECH_BUILD_MIC_CAPTURE) + if(NOT EXISTS "${CMAKE_SOURCE_DIR}/llama.cpp/vendor/miniaudio/miniaudio.h") + message(FATAL_ERROR + "ASR CLI microphone capture requires the vendored miniaudio header; " + "run: git submodule update --init llama.cpp") + endif() + target_sources(nemo_speech_cli PRIVATE microphone_capture.cpp) + target_compile_definitions(nemo_speech_cli PRIVATE NEMO_SPEECH_CLI_LIVE=1) + target_include_directories(nemo_speech_cli PRIVATE + ${CMAKE_SOURCE_DIR}/llama.cpp/vendor/miniaudio) + target_link_libraries(nemo_speech_cli PRIVATE ${CMAKE_DL_LIBS}) + if(APPLE) + target_link_libraries(nemo_speech_cli PRIVATE + "-framework AudioToolbox" + "-framework AudioUnit" + "-framework CoreAudio" + "-framework CoreFoundation") + endif() + endif() endif() if(NEMO_SPEECH_BUILD_HTTP) diff --git a/app/bench.cpp b/app/bench.cpp index 241907d..2aaf46f 100644 --- a/app/bench.cpp +++ b/app/bench.cpp @@ -220,8 +220,8 @@ run_bench(int argc, char** argv) { const int max_concurrency = *std::max_element(options.concurrency.begin(), options.concurrency.end()); options.config.model.path = - require_model_file( - options.model.empty() ? options.config.model.path : options.model, "ASR model") + resolve_model_file( + options.model.empty() ? options.config.model.path : options.model, "asr", "ASR model") .string(); options.config.batching.enabled = max_concurrency > 1; options.config.batching.max_batch_size = diff --git a/app/commands.h b/app/commands.h index b7266f8..4f3f3d3 100644 --- a/app/commands.h +++ b/app/commands.h @@ -8,6 +8,7 @@ int command_translate(int argc, char** argv); int command_synthesize(int argc, char** argv); int command_bench(int argc, char** argv); int command_model(int argc, char** argv); +int command_pull(int argc, char** argv); int command_doctor(int argc, char** argv); int command_health(int argc, char** argv); int command_serve(int argc, char** argv); diff --git a/app/diarize.cpp b/app/diarize.cpp index 6ca96eb..2d8ab93 100644 --- a/app/diarize.cpp +++ b/app/diarize.cpp @@ -96,7 +96,8 @@ print_diarize_help(const char* program) { "Diarize one WAV file or every WAV file in a directory. Concurrent\n" "directory work shares one model and batches compatible GPU steps.\n\n" "Options:\n" - " -m, --model MODEL Local Sortformer GGUF path\n" + " -m, --model MODEL Sortformer GGUF path or indexed HF repo\n" + " (default: nvidia/diar_streaming_sortformer_4spk-v2)\n" " --device, --backend DEVICE\n" " auto, cpu, cuda[:N], metal, or vulkan[:N]\n" " --offline Full-attention mode for short audio\n" @@ -224,7 +225,8 @@ command_diarize(int argc, char** argv) { batching.state_arena_slots = std::max(batching.state_arena_slots, workers); const auto geometry = config.resolved_geometry(); nemo_speech::EngineRegistry engines; - config.model_path = require_model_file(config.model_path, "diarization model").string(); + config.model_path = + resolve_model_file(config.model_path, "diarization", "diarization model").string(); if (cli_verbose()) std::fprintf( stderr, "diarize: model=%s mode=%s inputs=%zu concurrency=%d device=%d\n", diff --git a/app/doctor.cpp b/app/doctor.cpp index 7e34c7c..11740b8 100644 --- a/app/doctor.cpp +++ b/app/doctor.cpp @@ -9,6 +9,7 @@ #include "commands.h" #include "ggml-backend.h" #include "json.h" +#include "model_store.h" namespace { using nemo_speech::json::Value; @@ -42,6 +43,7 @@ build_features() { result["integrated_vad"] = false; result["punctuation"] = false; #endif + result["model_pull"] = true; #if defined(NEMO_SPEECH_CLI_DIAR) result["diarization"] = true; #else @@ -177,11 +179,20 @@ command_doctor(int argc, char** argv) { result["accelerator_compiled"] = accelerator_compiled; result["accelerator_available"] = accelerator_available; result["driver_runtime_compatible"] = !accelerator_compiled || accelerator_available; + const auto downloader = model_downloader_executable(); + Value model_download(Value::Object{}); + model_download["available"] = !downloader.empty(); + model_download["executable"] = downloader.u8string(); + result["model_download"] = std::move(model_download); Value::Array runtime_warnings; if (accelerator_compiled && !accelerator_available) runtime_warnings.emplace_back( "this build includes a GPU backend, but no compatible accelerator/driver was " "discovered; use --device cpu or repair the driver/runtime installation"); + if (downloader.empty()) + runtime_warnings.emplace_back( + "automatic model downloads require curl on PATH; local and cached models still " + "work"); result["runtime_warnings"] = std::move(runtime_warnings); if (json) { std::printf("%s\n", result.dump(2).c_str()); @@ -201,6 +212,10 @@ command_doctor(int argc, char** argv) { std::printf(" (%.1f GiB)", total / 1073741824.0); std::printf("\n"); } + if (downloader.empty()) + std::printf("Model downloads: unavailable (curl not found on PATH)\n"); + else + std::printf("Model downloads: %s\n", downloader.u8string().c_str()); for (const auto& warning : result.at("runtime_warnings").array()) std::printf("Runtime warning: %s\n", warning.string().c_str()); } diff --git a/app/main.cpp b/app/main.cpp index dbb96c7..51695b9 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -78,8 +78,12 @@ print_help(const char* program) { "Usage: %s [options]\n\n" "Commands:\n" #if defined(NEMO_SPEECH_CLI_ASR) +#if defined(NEMO_SPEECH_CLI_LIVE) + " transcribe Transcribe an audio file, directory, or microphone\n" +#else " transcribe Transcribe an audio file or directory\n" #endif +#endif #if defined(NEMO_SPEECH_CLI_DIAR) " diarize Identify speaker segments in an audio file\n" #endif @@ -92,7 +96,8 @@ print_help(const char* program) { #if defined(NEMO_SPEECH_CLI_ASR) " bench Benchmark an end-to-end ASR workload\n" #endif - " model Inspect local GGUF metadata\n" + " pull Download a pinned model from Hugging Face\n" + " model List, pull, or inspect models\n" " doctor Inspect runtime and device availability\n" #if defined(NEMO_SPEECH_CLI_HTTP) " health Check a running local HTTP server\n" @@ -184,7 +189,9 @@ main(int argc, char** argv) { return 0; } #endif - if (std::strcmp(argv[2], "model") == 0) + if (std::strcmp(argv[2], "pull") == 0) { + std::printf("Usage: %s pull REPO\n", argv[0]); + } else if (std::strcmp(argv[2], "model") == 0) print_model_help(argv[0]); else if (std::strcmp(argv[2], "doctor") == 0) print_doctor_help(argv[0]); @@ -231,6 +238,8 @@ main(int argc, char** argv) { #endif if (std::strcmp(argv[1], "model") == 0) return command_model(argc - 2, argv + 2); + if (std::strcmp(argv[1], "pull") == 0) + return command_pull(argc - 2, argv + 2); if (std::strcmp(argv[1], "doctor") == 0) return command_doctor(argc - 2, argv + 2); #if defined(NEMO_SPEECH_CLI_HTTP) diff --git a/app/microphone_capture.cpp b/app/microphone_capture.cpp new file mode 100644 index 0000000..77b228e --- /dev/null +++ b/app/microphone_capture.cpp @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#include "microphone_capture.h" + +#include +#include +#include + +// miniaudio is already vendored by the repository. Compiling its capture-only +// device layer here gives the CLI one implementation across CoreAudio, WASAPI, +// ALSA, PulseAudio, and the other supported host APIs without a runtime audio +// library dependency. +#define MA_NO_DECODING +#define MA_NO_ENCODING +#define MA_NO_RESOURCE_MANAGER +#define MA_NO_NODE_GRAPH +#define MA_NO_ENGINE +#define MA_NO_GENERATION +#define MINIAUDIO_IMPLEMENTATION +#include "miniaudio.h" + +namespace nemo_speech::cli { + +struct MicrophoneCapture::Impl { + static constexpr int kSampleRate = 16000; + + ma_device device{}; + bool initialized = false; + bool running = false; + std::atomic callback_failed{false}; + std::mutex mutex; + std::vector pending; + std::string name = "default microphone"; + + static void data_callback( + ma_device* device, void* /*output*/, const void* input, ma_uint32 frame_count) { + if (input == nullptr || frame_count == 0) + return; + auto* self = static_cast(device->pUserData); + const auto* samples = static_cast(input); + try { + std::lock_guard lock(self->mutex); + self->pending.insert(self->pending.end(), samples, samples + frame_count); + } + catch (...) { + // Exceptions must never escape the operating system's audio callback. + self->callback_failed.store(true, std::memory_order_release); + } + } + + void dispose() noexcept { + if (running) { + (void)ma_device_stop(&device); + running = false; + } + if (initialized) { + ma_device_uninit(&device); + initialized = false; + } + } +}; + +MicrophoneCapture::MicrophoneCapture() : impl_(std::make_unique()) {} + +MicrophoneCapture::~MicrophoneCapture() { + impl_->dispose(); +} + +void +MicrophoneCapture::start() { + if (impl_->initialized) + throw std::logic_error("microphone capture is already started"); + + ma_device_config config = ma_device_config_init(ma_device_type_capture); + config.capture.format = ma_format_f32; + config.capture.channels = 1; + config.sampleRate = Impl::kSampleRate; + config.periodSizeInMilliseconds = 40; + config.dataCallback = Impl::data_callback; + config.pUserData = impl_.get(); + impl_->pending.reserve(Impl::kSampleRate * 2); + impl_->callback_failed.store(false, std::memory_order_release); + + ma_result result = ma_device_init(nullptr, &config, &impl_->device); + if (result != MA_SUCCESS) + throw std::runtime_error( + "could not open the default microphone: " + std::string(ma_result_description(result))); + impl_->initialized = true; + impl_->name = + impl_->device.capture.name[0] != '\0' ? impl_->device.capture.name : "default microphone"; + + result = ma_device_start(&impl_->device); + if (result != MA_SUCCESS) { + impl_->dispose(); + throw std::runtime_error( + "could not start microphone capture: " + std::string(ma_result_description(result)) + + ". Check the operating system's microphone permission for this terminal"); + } + impl_->running = true; +} + +void +MicrophoneCapture::stop() { + impl_->dispose(); +} + +std::vector +MicrophoneCapture::drain() { + if (impl_->callback_failed.load(std::memory_order_acquire)) + throw std::runtime_error("microphone capture ran out of buffer memory"); + std::vector samples; + std::lock_guard lock(impl_->mutex); + samples.swap(impl_->pending); + return samples; +} + +int +MicrophoneCapture::sample_rate() const { + return Impl::kSampleRate; +} + +const std::string& +MicrophoneCapture::device_name() const { + return impl_->name; +} + +} // namespace nemo_speech::cli diff --git a/app/microphone_capture.h b/app/microphone_capture.h new file mode 100644 index 0000000..c0457bd --- /dev/null +++ b/app/microphone_capture.h @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include + +namespace nemo_speech::cli { + +// Captures mono float32 samples from the default microphone. miniaudio's host +// backend stays behind the pimpl so transcribe.cpp remains independent of OS +// audio headers. +class MicrophoneCapture { + public: + MicrophoneCapture(); + ~MicrophoneCapture(); + + MicrophoneCapture(const MicrophoneCapture&) = delete; + MicrophoneCapture& operator=(const MicrophoneCapture&) = delete; + + void start(); + void stop(); + std::vector drain(); + int sample_rate() const; + const std::string& device_name() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace nemo_speech::cli diff --git a/app/model.cpp b/app/model.cpp index 3bdce89..4c5099f 100644 --- a/app/model.cpp +++ b/app/model.cpp @@ -8,20 +8,60 @@ #include "cli_util.h" #include "commands.h" #include "json.h" +#include "model_store.h" #include "model_utils.h" namespace { namespace fs = std::filesystem; using nemo_speech::json::Value; +int +run_pull(const std::string& repo) { + const auto artifacts = pull_indexed_model(repo); + if (cli_json()) { + Value result(Value::Object{}); + result["repo"] = repo; + Value::Array values; + for (const auto& artifact : artifacts) { + Value value(Value::Object{}); + value["repo"] = artifact.repo; + value["role"] = artifact.role; + value["path"] = artifact.path.u8string(); + value["cached"] = artifact.cached; + values.emplace_back(std::move(value)); + } + result["artifacts"] = std::move(values); + std::printf("%s\n", result.dump(2).c_str()); + } else { + for (const auto& artifact : artifacts) + std::printf( + "%s\t%s\t%s\n", artifact.repo.c_str(), artifact.role.c_str(), + artifact.path.u8string().c_str()); + } + return kCliExitSuccess; +} + int run_model(int argc, char** argv) { if (argc == 0 || is_help_argument(argv[0])) { print_model_help("nemo-speech"); return 0; } - if (std::string(argv[0]) != "info") - throw std::invalid_argument("unknown model action: " + std::string(argv[0])); + const std::string action = argv[0]; + if (action == "list") { + if (argc != 1) + throw std::invalid_argument("model list does not accept arguments"); + const std::string result = cli_json() ? indexed_models_json() : indexed_models_text(); + std::fwrite(result.data(), 1, result.size(), stdout); + return kCliExitSuccess; + } + if (action == "pull") { + if (argc != 2) + throw std::invalid_argument("model pull requires one indexed Hugging Face repository"); + return run_pull(argv[1]); + } + if (action != "info") + throw std::invalid_argument("unknown model action: " + action); if (argc != 2) throw std::invalid_argument("model info requires one local GGUF file"); @@ -36,8 +76,13 @@ run_model(int argc, char** argv) { void print_model_help(const char* program) { std::printf( - "Usage: %s model info FILE\n\n" - "Inspect the metadata and runtime compatibility of a local GGUF file.\n", + "Usage: %s model [arguments]\n\n" + "Actions:\n" + " list List indexed repositories and command defaults\n" + " pull REPO Download and verify an indexed Hugging Face repository\n" + " info FILE Inspect a local GGUF file\n\n" + "Models are cached under the platform user cache directory. Override it\n" + "with NEMO_SPEECH_MODEL_DIR. Downloads require curl on PATH.\n", program); } @@ -50,3 +95,19 @@ command_model(int argc, char** argv) { return print_cli_exception("model", error); } } + +int +command_pull(int argc, char** argv) { + try { + if (argc > 0 && is_help_argument(argv[0])) { + std::printf("Usage: nemo-speech pull REPO\n"); + return kCliExitSuccess; + } + if (argc != 1) + throw std::invalid_argument("pull requires one indexed Hugging Face repository"); + return run_pull(argv[0]); + } + catch (const std::exception& error) { + return print_cli_exception("pull", error); + } +} diff --git a/app/model_store.cpp b/app/model_store.cpp new file mode 100644 index 0000000..3b1a2dc --- /dev/null +++ b/app/model_store.cpp @@ -0,0 +1,1171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "model_store.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cli_util.h" +#include "json.h" + +#if defined(_WIN32) +#include +#include +#include +#elif defined(__APPLE__) +#include +#include +#include +#include +#include +#else +#include +#include +#include +#include +#endif + +namespace { +namespace fs = std::filesystem; +using nemo_speech::json::Value; + +struct ArchiveMember { + std::string name; + std::string sha256; + uint64_t size = 0; +}; + +struct Artifact { + std::string role; + std::string type; + std::string filename; + std::string directory; + std::string sha256; + uint64_t size = 0; + uint64_t range_end = 0; + std::string stop_before; + std::vector members; +}; + +struct Model { + std::string repo; + std::vector aliases; + std::string revision; + std::string license; + std::string license_url; + std::vector companions; + std::vector artifacts; +}; + +struct Index { + std::map defaults; + std::vector models; +}; + +std::string +path_utf8(const fs::path& path) { + return path.u8string(); +} + +uint32_t +rotate_right(uint32_t value, uint32_t count) { + return (value >> count) | (value << (32 - count)); +} + +class Sha256 { + public: + void update(const unsigned char* data, size_t size) { + total_size_ += size; + while (size > 0) { + const size_t count = std::min(size, block_.size() - block_size_); + std::memcpy(block_.data() + block_size_, data, count); + block_size_ += count; + data += count; + size -= count; + if (block_size_ == block_.size()) { + transform(block_.data()); + block_size_ = 0; + } + } + } + + std::string finish() { + const uint64_t bits = total_size_ * 8; + block_[block_size_++] = 0x80; + if (block_size_ > 56) { + std::fill(block_.begin() + static_cast(block_size_), block_.end(), 0); + transform(block_.data()); + block_size_ = 0; + } + std::fill(block_.begin() + static_cast(block_size_), block_.begin() + 56, 0); + for (size_t i = 0; i < 8; ++i) block_[63 - i] = static_cast(bits >> (i * 8)); + transform(block_.data()); + std::ostringstream output; + output << std::hex << std::setfill('0'); + for (const uint32_t value : state_) output << std::setw(8) << value; + return output.str(); + } + + private: + void transform(const unsigned char* block) { + static constexpr std::array constants = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2}; + std::array words{}; + for (size_t i = 0; i < 16; ++i) { + words[i] = static_cast(block[i * 4]) << 24 | + static_cast(block[i * 4 + 1]) << 16 | + static_cast(block[i * 4 + 2]) << 8 | + static_cast(block[i * 4 + 3]); + } + for (size_t i = 16; i < words.size(); ++i) { + const uint32_t s0 = rotate_right(words[i - 15], 7) ^ rotate_right(words[i - 15], 18) ^ + (words[i - 15] >> 3); + const uint32_t s1 = rotate_right(words[i - 2], 17) ^ rotate_right(words[i - 2], 19) ^ + (words[i - 2] >> 10); + words[i] = words[i - 16] + s0 + words[i - 7] + s1; + } + uint32_t a = state_[0], b = state_[1], c = state_[2], d = state_[3]; + uint32_t e = state_[4], f = state_[5], g = state_[6], h = state_[7]; + for (size_t i = 0; i < words.size(); ++i) { + const uint32_t s1 = rotate_right(e, 6) ^ rotate_right(e, 11) ^ rotate_right(e, 25); + const uint32_t choice = (e & f) ^ (~e & g); + const uint32_t t1 = h + s1 + choice + constants[i] + words[i]; + const uint32_t s0 = rotate_right(a, 2) ^ rotate_right(a, 13) ^ rotate_right(a, 22); + const uint32_t majority = (a & b) ^ (a & c) ^ (b & c); + const uint32_t t2 = s0 + majority; + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + state_[0] += a; + state_[1] += b; + state_[2] += c; + state_[3] += d; + state_[4] += e; + state_[5] += f; + state_[6] += g; + state_[7] += h; + } + + std::array state_ = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + std::array block_{}; + size_t block_size_ = 0; + uint64_t total_size_ = 0; +}; + +std::string +sha256_file(const fs::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) + throw std::runtime_error("cannot read downloaded artifact " + path_utf8(path)); + Sha256 digest; + std::array buffer{}; + while (input) { + input.read(reinterpret_cast(buffer.data()), buffer.size()); + const auto count = input.gcount(); + if (count > 0) + digest.update(buffer.data(), static_cast(count)); + } + if (!input.eof()) + throw std::runtime_error("failed while reading downloaded artifact " + path_utf8(path)); + return digest.finish(); +} + +fs::path +executable_path() { +#if defined(_WIN32) + std::wstring buffer(32768, L'\0'); + const DWORD size = + GetModuleFileNameW(nullptr, buffer.data(), static_cast(buffer.size())); + if (size == 0 || size == buffer.size()) + return {}; + buffer.resize(size); + return fs::path(buffer); +#elif defined(__APPLE__) + uint32_t size = 0; + _NSGetExecutablePath(nullptr, &size); + std::string buffer(size, '\0'); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) + return {}; + buffer.resize(std::strlen(buffer.c_str())); + return fs::weakly_canonical(buffer); +#else + std::array buffer{}; + const ssize_t size = readlink("/proc/self/exe", buffer.data(), buffer.size() - 1); + return size > 0 ? fs::path(std::string(buffer.data(), static_cast(size))) : fs::path{}; +#endif +} + +fs::path +index_path() { +#if defined(_WIN32) + if (const wchar_t* override_path = _wgetenv(L"NEMO_SPEECH_MODEL_INDEX")) { + if (*override_path) + return override_path; + } +#else + if (const char* override_path = std::getenv("NEMO_SPEECH_MODEL_INDEX")) { + if (*override_path) + return override_path; + } +#endif + const fs::path executable = executable_path(); + if (!executable.empty()) { + const fs::path installed = + executable.parent_path().parent_path() / "share" / "nemo-speech" / "model-index.json"; + if (fs::is_regular_file(installed)) + return installed; + } + throw std::runtime_error( + "model index is missing; reinstall NeMo-Speech.cpp or set NEMO_SPEECH_MODEL_INDEX"); +} + +uint64_t +integer(const Value& object, const std::string& key) { + const double value = object.at(key).number(); + if (value < 0 || value > static_cast(UINT64_MAX) || value != std::floor(value)) + throw std::runtime_error("invalid model index integer: " + key); + return static_cast(value); +} + +void +validate_component(const std::string& value, const std::string& field) { + if (value.empty() || value == "." || value == ".." || + value.find_first_not_of( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") != + std::string::npos) + throw std::runtime_error("unsafe " + field + " in model index: " + value); +} + +void +validate_repo(const std::string& repo) { + const size_t slash = repo.find('/'); + if (slash == std::string::npos || slash == 0 || slash + 1 == repo.size() || + repo.find('/', slash + 1) != std::string::npos) + throw std::runtime_error("invalid repository in model index: " + repo); + validate_component(repo.substr(0, slash), "repository owner"); + validate_component(repo.substr(slash + 1), "repository name"); +} + +bool +looks_like_repo(const std::string& reference) { + try { + validate_repo(reference); + return true; + } + catch (const std::exception&) { + return false; + } +} + +Index +load_index() { + Value root = Value::parse(read_text_file(index_path())); + if (integer(root, "schema_version") != 1) + throw std::runtime_error("unsupported model index schema"); + Index index; + for (const auto& item : root.at("defaults").object()) + index.defaults.emplace(item.first, item.second.string()); + for (const auto& model_value : root.at("models").array()) { + Model model; + model.repo = model_value.at("repo").string(); + model.revision = model_value.at("revision").string(); + model.license = model_value.at("license").string(); + model.license_url = model_value.at("license_url").string(); + validate_repo(model.repo); + if (model.license.empty() || model.license_url.rfind("https://", 0) != 0) + throw std::runtime_error("invalid license metadata in model index: " + model.repo); + if (const Value* aliases = model_value.find("aliases")) { + for (const auto& alias : aliases->array()) { + validate_component(alias.string(), "model alias"); + model.aliases.push_back(alias.string()); + } + } + if (model.revision.size() != 40 || + model.revision.find_first_not_of("0123456789abcdef") != std::string::npos) + throw std::runtime_error( + "model index revision must be a full commit SHA: " + model.repo); + if (const Value* companions = model_value.find("companions")) { + for (const auto& companion : companions->array()) { + validate_repo(companion.string()); + model.companions.push_back(companion.string()); + } + } + std::set artifact_roles; + for (const auto& artifact_value : model_value.at("artifacts").array()) { + Artifact artifact; + artifact.role = artifact_value.at("role").string(); + artifact.type = artifact_value.at("type").string(); + artifact.filename = artifact_value.at("filename").string(); + artifact.directory = artifact_value.string_or("directory"); + artifact.sha256 = artifact_value.at("sha256").string(); + artifact.size = integer(artifact_value, "size"); + artifact.range_end = + artifact_value.find("range_end") ? integer(artifact_value, "range_end") : 0; + artifact.stop_before = artifact_value.string_or("stop_before"); + static const std::set allowed_roles = { + "asr", "diarization", "tts", "codec", "tokenizer"}; + if (allowed_roles.find(artifact.role) == allowed_roles.end()) + throw std::runtime_error("unsupported artifact role in model index"); + if (!artifact_roles.insert(artifact.role).second) + throw std::runtime_error( + "duplicate artifact role in model index: " + model.repo + "/" + artifact.role); + validate_component(artifact.filename, "artifact filename"); + if (!artifact.directory.empty()) + validate_component(artifact.directory, "artifact directory"); + if (artifact.sha256.size() != 64 || + artifact.sha256.find_first_not_of("0123456789abcdef") != std::string::npos) + throw std::runtime_error("invalid SHA-256 in model index"); + if (const Value* members = artifact_value.find("members")) { + for (const auto& member_value : members->array()) { + ArchiveMember member; + member.name = member_value.at("name").string(); + member.size = integer(member_value, "size"); + member.sha256 = member_value.at("sha256").string(); + validate_component(member.name, "archive member"); + if (member.sha256.size() != 64 || + member.sha256.find_first_not_of("0123456789abcdef") != std::string::npos) + throw std::runtime_error("invalid archive member SHA-256 in model index"); + artifact.members.push_back(std::move(member)); + } + } + if (artifact.type != "file" && artifact.type != "tar-prefix") + throw std::runtime_error("unsupported artifact type in model index"); + if (artifact.size == 0) + throw std::runtime_error("model index artifact size must be positive"); + if (artifact.type == "file" && + (!artifact.directory.empty() || !artifact.stop_before.empty() || + !artifact.members.empty())) + throw std::runtime_error("regular model artifact contains archive-only fields"); + if (artifact.type == "tar-prefix" && + (artifact.directory.empty() || artifact.stop_before.empty() || + artifact.members.empty() || artifact.range_end != artifact.size - 1)) + throw std::runtime_error("invalid tokenizer archive metadata in model index"); + model.artifacts.push_back(std::move(artifact)); + } + if (model.artifacts.empty()) + throw std::runtime_error("model has no artifacts in model index: " + model.repo); + index.models.push_back(std::move(model)); + } + std::set identifiers; + for (const auto& model : index.models) { + if (!identifiers.insert(model.repo).second) + throw std::runtime_error("duplicate repository in model index: " + model.repo); + for (const auto& alias : model.aliases) + if (!identifiers.insert(alias).second) + throw std::runtime_error("duplicate alias in model index: " + alias); + } + for (const auto& item : index.defaults) { + if (identifiers.find(item.second) == identifiers.end()) + throw std::runtime_error( + "model index default references an unknown model: " + item.second); + const Model* model = nullptr; + for (const auto& candidate : index.models) + if (candidate.repo == item.second || + std::find(candidate.aliases.begin(), candidate.aliases.end(), item.second) != + candidate.aliases.end()) + model = &candidate; + const bool provides_role = std::any_of( + model->artifacts.begin(), model->artifacts.end(), + [&](const Artifact& artifact) { return artifact.role == item.first; }); + if (!provides_role) + throw std::runtime_error("model index default does not provide role: " + item.first); + } + for (const auto& model : index.models) + for (const auto& companion : model.companions) + if (identifiers.find(companion) == identifiers.end()) + throw std::runtime_error( + "model index companion references an unknown model: " + companion); + return index; +} + +const Model& +find_model(const Index& index, const std::string& repo) { + for (const auto& model : index.models) { + if (model.repo == repo || + std::find(model.aliases.begin(), model.aliases.end(), repo) != model.aliases.end()) + return model; + } + throw MissingModelError( + "unknown model repository: " + repo + " (run 'nemo-speech model list')"); +} + +const Artifact& +find_artifact(const Model& model, const std::string& role, bool directory) { + for (const auto& artifact : model.artifacts) { + if (artifact.role == role && (artifact.type == "tar-prefix") == directory) + return artifact; + } + throw MissingModelError( + model.repo + " does not provide the indexed " + role + + (directory ? " directory" : " model")); +} + +std::vector +commands_for(const Model& model) { + std::vector result; + auto add = [&](const std::string& command) { + if (std::find(result.begin(), result.end(), command) == result.end()) + result.push_back(command); + }; + for (const auto& artifact : model.artifacts) { + if (artifact.role == "asr") { + add("transcribe"); + add("bench"); + add("serve"); + } else if (artifact.role == "diarization") { + add("diarize"); + add("transcribe --diarize"); + add("serve"); + } else if ( + artifact.role == "tts" || artifact.role == "tokenizer" || artifact.role == "codec") { + add("synthesize"); + add("serve"); + } + } + return result; +} + +std::vector +defaults_for(const Index& index, const Model& model) { + std::vector result; + for (const auto& item : index.defaults) + if (item.second == model.repo) + result.push_back(item.first); + return result; +} + +fs::path +cache_root() { +#if defined(_WIN32) + if (const wchar_t* override_path = _wgetenv(L"NEMO_SPEECH_MODEL_DIR")) { + if (*override_path) + return fs::path(override_path); + } +#else + if (const char* override_path = std::getenv("NEMO_SPEECH_MODEL_DIR")) { + if (*override_path) + return override_path; + } +#endif +#if defined(_WIN32) + if (const wchar_t* local = _wgetenv(L"LOCALAPPDATA")) + return fs::path(local) / "NeMoSpeech" / "models"; +#elif defined(__APPLE__) + if (const char* home = std::getenv("HOME")) + return fs::path(home) / "Library" / "Caches" / "NeMoSpeech" / "models"; +#else + if (const char* cache = std::getenv("XDG_CACHE_HOME")) + return fs::path(cache) / "nemo-speech" / "models"; + if (const char* home = std::getenv("HOME")) + return fs::path(home) / ".cache" / "nemo-speech" / "models"; +#endif + throw std::runtime_error( + "cannot determine the model cache directory; set NEMO_SPEECH_MODEL_DIR"); +} + +fs::path +model_directory(const Model& model) { + const size_t slash = model.repo.find('/'); + return cache_root() / model.repo.substr(0, slash) / model.repo.substr(slash + 1) / + model.revision; +} + +class ArtifactLock { + public: + explicit ArtifactLock(const fs::path& path) { + fs::create_directories(path.parent_path()); +#if defined(_WIN32) + for (;;) { + handle_ = CreateFileW( + path.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (handle_ != INVALID_HANDLE_VALUE) + break; + if (GetLastError() != ERROR_SHARING_VIOLATION) + throw std::runtime_error("cannot lock model cache " + path_utf8(path)); + Sleep(100); + } +#else + descriptor_ = open(path.c_str(), O_CREAT | O_RDWR, 0600); + if (descriptor_ < 0 || flock(descriptor_, LOCK_EX) != 0) { + if (descriptor_ >= 0) + close(descriptor_); + throw std::runtime_error("cannot lock model cache " + path_utf8(path)); + } +#endif + } + + ~ArtifactLock() { +#if defined(_WIN32) + if (handle_ != INVALID_HANDLE_VALUE) + CloseHandle(handle_); +#else + if (descriptor_ >= 0) { + flock(descriptor_, LOCK_UN); + close(descriptor_); + } +#endif + } + + ArtifactLock(const ArtifactLock&) = delete; + ArtifactLock& operator=(const ArtifactLock&) = delete; + + private: +#if defined(_WIN32) + HANDLE handle_ = INVALID_HANDLE_VALUE; +#else + int descriptor_ = -1; +#endif +}; + +std::string +base_url() { + std::string result = "https://huggingface.co"; + if (const char* override_url = std::getenv("NEMO_SPEECH_HF_BASE_URL")) + if (*override_url) + result = override_url; + while (!result.empty() && result.back() == '/') result.pop_back(); + const bool https = result.rfind("https://", 0) == 0; + auto loopback_authority = [](const std::string& url) { + if (url.rfind("http://", 0) != 0) + return false; + const size_t end = url.find_first_of("/?#", 7); + const std::string authority = url.substr(7, end == std::string::npos ? end : end - 7); + auto matches = [&](const std::string& host) { + if (authority == host) + return true; + if (authority.rfind(host + ":", 0) != 0) + return false; + const std::string port = authority.substr(host.size() + 1); + if (port.empty() || port.find_first_not_of("0123456789") != std::string::npos) + return false; + try { + const unsigned long value = std::stoul(port); + return value > 0 && value <= 65535; + } + catch (const std::exception&) { + return false; + } + }; + return matches("127.0.0.1") || matches("localhost") || matches("[::1]"); + }; + const bool loopback = loopback_authority(result); + if (!https && !loopback) + throw std::runtime_error( + "NEMO_SPEECH_HF_BASE_URL must use HTTPS (HTTP is allowed only for loopback tests)"); + return result; +} + +std::string +download_url(const Model& model, const Artifact& artifact) { + return base_url() + "/" + model.repo + "/resolve/" + model.revision + "/" + artifact.filename + + "?download=true"; +} + +int +run_curl(const std::vector& arguments) { + const fs::path executable = model_downloader_executable(); + if (executable.empty()) + return 127; +#if defined(_WIN32) + auto widen_utf8 = [](const std::string& value) { + if (value.empty()) + return std::wstring(); + const int size = MultiByteToWideChar( + CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), static_cast(value.size()), nullptr, + 0); + if (size <= 0) + throw std::runtime_error("could not encode a curl argument for Windows"); + std::wstring result(static_cast(size), L'\0'); + if (MultiByteToWideChar( + CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), static_cast(value.size()), + result.data(), size) != size) + throw std::runtime_error("could not encode a curl argument for Windows"); + return result; + }; + std::vector wide; + wide.reserve(arguments.size() + 1); + wide.emplace_back(executable.wstring()); + for (const auto& argument : arguments) wide.emplace_back(widen_utf8(argument)); + std::vector argv; + argv.reserve(wide.size() + 1); + for (const auto& argument : wide) argv.push_back(argument.c_str()); + argv.push_back(nullptr); + const intptr_t status = _wspawnv(_P_WAIT, executable.c_str(), argv.data()); + return status < 0 ? 127 : static_cast(status); +#else + const pid_t child = fork(); + if (child < 0) + throw std::runtime_error("could not start curl"); + if (child == 0) { + std::vector argv; + argv.reserve(arguments.size() + 2); + const std::string executable_string = executable.string(); + argv.push_back(const_cast(executable_string.c_str())); + for (const auto& argument : arguments) argv.push_back(const_cast(argument.c_str())); + argv.push_back(nullptr); + execv(executable.c_str(), argv.data()); + _exit(127); + } + int status = 0; + if (waitpid(child, &status, 0) != child) + throw std::runtime_error("could not wait for curl"); + return WIFEXITED(status) ? WEXITSTATUS(status) : 128; +#endif +} + +std::string +curl_missing_message() { + std::string message = + "automatic model download requires the curl executable, but it was not found on PATH. " + "Local model paths and models already in NEMO_SPEECH_MODEL_DIR still work. "; +#if defined(_WIN32) + message += + "Windows 10 and 11 include curl.exe; ensure %SystemRoot%\\System32 is on PATH or repair " + "the Windows curl installation."; +#elif defined(__APPLE__) + message += "macOS includes /usr/bin/curl; ensure /usr/bin is on PATH."; +#else + message += + "Install it with your package manager (for example, 'sudo apt install curl', " + "'sudo dnf install curl', or 'sudo pacman -S curl')."; +#endif + return message; +} + +bool +stderr_is_terminal() { +#if defined(_WIN32) + return _isatty(_fileno(stderr)) != 0; +#else + return isatty(STDERR_FILENO) != 0; +#endif +} + +void +download(const Model& model, const Artifact& artifact, const fs::path& output) { + const bool loopback = base_url().rfind("http://", 0) == 0; + const std::string protocols = loopback ? "=http,https" : "=https"; + fs::path curl_errors = output; + curl_errors += ".curl-errors"; + auto invoke = [&](bool resume) { + std::error_code remove_error; + fs::remove(curl_errors, remove_error); + std::vector arguments = { + "--fail", + "--location", + "--max-redirs", + "5", + "--proto", + protocols, + "--proto-redir", + protocols, + "--retry", + "3", + "--retry-delay", + "1", + "--connect-timeout", + "20", + "--speed-limit", + "1024", + "--speed-time", + "30", + "--output", + output.u8string()}; + if (artifact.type == "tar-prefix") { + arguments.push_back("--range"); + arguments.push_back("0-" + std::to_string(artifact.range_end)); + } else if (resume && fs::exists(output)) { + arguments.push_back("--continue-at"); + arguments.push_back("-"); + } + if (cli_quiet() || cli_json() || !stderr_is_terminal()) { + arguments.push_back("--silent"); + arguments.push_back("--show-error"); + arguments.push_back("--stderr"); + arguments.push_back(curl_errors.u8string()); + } else { + arguments.push_back("--progress-bar"); + } + arguments.push_back(download_url(model, artifact)); + return run_curl(arguments); + }; + int status = invoke(true); + if (status == 33 && artifact.type == "file") { + std::error_code error; + fs::remove(output, error); + status = invoke(false); + } + if (status == 127) + throw std::runtime_error(curl_missing_message()); + if (status != 0) { + std::string detail; + if (fs::is_regular_file(curl_errors)) { + detail = read_text_file(curl_errors); + while (!detail.empty() && (detail.back() == '\n' || detail.back() == '\r')) + detail.pop_back(); + } + std::error_code error; + fs::remove(curl_errors, error); + throw std::runtime_error( + "curl failed while downloading " + model.repo + " (exit code " + + std::to_string(status) + ")" + (detail.empty() ? "" : ": " + detail)); + } + std::error_code error; + fs::remove(curl_errors, error); +} + +bool +valid_file(const fs::path& path, const Artifact& artifact) { + std::error_code error; + if (!fs::is_regular_file(path, error) || error || fs::file_size(path, error) != artifact.size || + error) + return false; + return sha256_file(path) == artifact.sha256; +} + +bool +valid_member(const fs::path& path, const ArchiveMember& member) { + std::error_code error; + return fs::is_regular_file(path, error) && !error && + fs::file_size(path, error) == member.size && !error && + sha256_file(path) == member.sha256; +} + +uint64_t +tar_octal(const char* value, size_t size) { + uint64_t result = 0; + size_t index = 0; + while (index < size && (value[index] == ' ' || value[index] == '\0')) ++index; + for (; index < size && value[index] != '\0' && value[index] != ' '; ++index) { + if (value[index] < '0' || value[index] > '7') + throw std::runtime_error("invalid TAR numeric field in tokenizer artifact"); + result = result * 8 + static_cast(value[index] - '0'); + } + return result; +} + +std::string +tar_string(const char* value, size_t size) { + size_t length = 0; + while (length < size && value[length] != '\0') ++length; + return std::string(value, length); +} + +void +validate_pax_metadata(const std::string& data) { + size_t offset = 0; + while (offset < data.size()) { + const size_t space = data.find(' ', offset); + if (space == std::string::npos || space == offset) + throw std::runtime_error("invalid PAX metadata in tokenizer artifact"); + size_t length = 0; + for (size_t i = offset; i < space; ++i) { + if (data[i] < '0' || data[i] > '9') + throw std::runtime_error("invalid PAX metadata in tokenizer artifact"); + length = length * 10 + static_cast(data[i] - '0'); + } + if (length == 0 || length > data.size() - offset || data[offset + length - 1] != '\n') + throw std::runtime_error("invalid PAX metadata in tokenizer artifact"); + const size_t equals = data.find('=', space + 1); + if (equals == std::string::npos || equals >= offset + length) + throw std::runtime_error("invalid PAX metadata in tokenizer artifact"); + const std::string key = data.substr(space + 1, equals - space - 1); + if (key != "mtime" && key != "atime" && key != "ctime") + throw std::runtime_error("unsupported PAX field in tokenizer artifact: " + key); + offset += length; + } +} + +void +extract_tar_prefix(const fs::path& archive, const fs::path& destination, const Artifact& artifact) { + std::ifstream input(archive, std::ios::binary); + if (!input) + throw std::runtime_error("cannot read tokenizer archive " + path_utf8(archive)); + fs::create_directories(destination); + std::array header{}; + bool reached_stop = false; + while (input.read(header.data(), header.size())) { + if (std::all_of(header.begin(), header.end(), [](char c) { return c == '\0'; })) + break; + uint64_t checksum = 0; + for (size_t i = 0; i < header.size(); ++i) + checksum += static_cast(i >= 148 && i < 156 ? ' ' : header[i]); + if (checksum != tar_octal(header.data() + 148, 8)) + throw std::runtime_error("invalid TAR checksum in tokenizer artifact"); + std::string name = tar_string(header.data(), 100); + const std::string prefix = tar_string(header.data() + 345, 155); + if (!prefix.empty()) + name = prefix + "/" + name; + const fs::path relative(name); + if (relative.empty() || relative.is_absolute()) + throw std::runtime_error("unsafe path in tokenizer artifact"); + for (const auto& component : relative) + if (component == "..") + throw std::runtime_error("unsafe path in tokenizer artifact"); + if (relative.filename() == artifact.stop_before) { + reached_stop = true; + break; + } + const uint64_t size = tar_octal(header.data() + 124, 12); + const char type = header[156]; + const fs::path output = destination / relative; + if (type == '5') { + fs::create_directories(output); + } else if (type == '\0' || type == '0') { + fs::create_directories(output.parent_path()); + std::ofstream file(output, std::ios::binary | std::ios::trunc); + if (!file) + throw std::runtime_error("cannot extract " + path_utf8(output)); + uint64_t remaining = size; + std::array buffer{}; + while (remaining > 0) { + const size_t count = + static_cast(std::min(remaining, buffer.size())); + if (!input.read(buffer.data(), static_cast(count))) + throw std::runtime_error("truncated tokenizer artifact"); + file.write(buffer.data(), static_cast(count)); + remaining -= count; + } + if (!file) + throw std::runtime_error("cannot extract " + path_utf8(output)); + } else if ((type == 'x' || type == 'g') && size <= 64 * 1024) { + std::string metadata(static_cast(size), '\0'); + if (!input.read(metadata.data(), static_cast(metadata.size()))) + throw std::runtime_error("truncated tokenizer artifact"); + validate_pax_metadata(metadata); + } else { + throw std::runtime_error("unsupported TAR entry in tokenizer artifact"); + } + const uint64_t padding = (512 - (size % 512)) % 512; + if (type == '5') { + if (size != 0) + input.seekg(static_cast(size + padding), std::ios::cur); + } else if (padding != 0) { + input.seekg(static_cast(padding), std::ios::cur); + } + if (!input) + throw std::runtime_error("truncated tokenizer artifact"); + } + if (!reached_stop) + throw std::runtime_error( + "tokenizer archive prefix did not reach the expected model weights"); + for (const auto& member : artifact.members) { + if (!valid_member(destination / member.name, member)) + throw std::runtime_error( + "tokenizer artifact member failed verification: " + member.name); + } +} + +PulledModelArtifact +materialize(const Model& model, const Artifact& artifact) { + const fs::path directory = model_directory(model); + fs::create_directories(directory); + const fs::path destination = + artifact.type == "file" ? directory / artifact.filename : directory / artifact.directory; + fs::path lock_path = destination; + lock_path += ".lock"; + ArtifactLock lock(lock_path); + if (artifact.type == "file" && valid_file(destination, artifact)) { + if (cli_verbose()) + std::fprintf(stderr, "[model] cached: %s\n", path_utf8(destination).c_str()); + return {model.repo, artifact.role, destination, true}; + } + if (artifact.type == "tar-prefix") { + bool valid = fs::is_directory(destination); + for (const auto& member : artifact.members) + valid = valid && valid_member(destination / member.name, member); + if (valid) { + if (cli_verbose()) + std::fprintf(stderr, "[model] cached: %s\n", path_utf8(destination).c_str()); + return {model.repo, artifact.role, destination, true}; + } + } + + if (!cli_quiet() && !cli_json()) { + std::fprintf( + stderr, + "[model] downloading %s@%.12s (%s, %.1f MiB)\n" + "[model] license: %s — %s\n", + model.repo.c_str(), model.revision.c_str(), artifact.role.c_str(), + artifact.size / 1048576.0, model.license.c_str(), model.license_url.c_str()); + } + const fs::path partial = directory / (artifact.filename + ".partial"); + if (!valid_file(partial, artifact)) { + if (artifact.type == "tar-prefix") { + std::error_code error; + fs::remove(partial, error); + } else { + std::error_code error; + if (fs::is_regular_file(partial, error) && + fs::file_size(partial, error) > artifact.size) + fs::remove(partial, error); + } + download(model, artifact, partial); + } + if (!cli_quiet() && !cli_json()) + std::fprintf(stderr, "[model] verifying size and SHA-256...\n"); + if (!valid_file(partial, artifact)) { + std::error_code error; + fs::remove(partial, error); + throw std::runtime_error( + "downloaded artifact failed size or SHA-256 verification: " + model.repo + "/" + + artifact.filename); + } + + std::error_code error; + if (artifact.type == "file") { + fs::remove(destination, error); + error.clear(); + fs::rename(partial, destination, error); + if (error) + throw std::runtime_error("cannot install model artifact: " + error.message()); + } else { + const fs::path extracting = directory / (artifact.directory + ".extracting"); + fs::remove_all(extracting, error); + extract_tar_prefix(partial, extracting, artifact); + fs::remove_all(destination, error); + error.clear(); + fs::rename(extracting, destination, error); + if (error) { + fs::remove_all(extracting); + throw std::runtime_error("cannot install tokenizer artifact: " + error.message()); + } + fs::remove(partial, error); + } + if (!cli_quiet() && !cli_json()) + std::fprintf(stderr, "[model] ready: %s\n", path_utf8(destination).c_str()); + return {model.repo, artifact.role, destination, false}; +} + +fs::path +resolve( + const std::string& reference, const std::string& role, const std::string& description, + bool directory) { + std::error_code error; + if (!reference.empty()) { + const fs::path local(reference); + const bool exists = + directory ? fs::is_directory(local, error) : fs::is_regular_file(local, error); + if (exists && !error) + return fs::absolute(local); + } + const Index index = load_index(); + std::string repo = reference; + if (repo.empty()) { + const auto found = index.defaults.find(role); + if (found == index.defaults.end()) + throw MissingModelError(description + " path is required"); + repo = found->second; + } + const Model* model = nullptr; + for (const auto& candidate : index.models) + if (candidate.repo == repo || + std::find(candidate.aliases.begin(), candidate.aliases.end(), repo) != + candidate.aliases.end()) + model = &candidate; + if (!model) { + if (reference.empty() || looks_like_repo(reference)) + throw MissingModelError( + "unknown " + description + " repository: " + repo + + " (run 'nemo-speech model list')"); + throw MissingModelError( + description + (directory ? " directory does not exist: " : " file does not exist: ") + + reference); + } + return materialize(*model, find_artifact(*model, role, directory)).path; +} + +} // namespace + +fs::path +resolve_indexed_model_file( + const std::string& reference, const std::string& role, const std::string& description) { + return resolve(reference, role, description, false); +} + +fs::path +resolve_indexed_model_directory( + const std::string& reference, const std::string& role, const std::string& description) { + return resolve(reference, role, description, true); +} + +std::vector +pull_indexed_model(const std::string& repo) { + const Index index = load_index(); + std::vector result; + std::set visited; + std::function pull = [&](const std::string& current_repo) { + const Model& model = find_model(index, current_repo); + if (!visited.insert(model.repo).second) + return; + for (const auto& artifact : model.artifacts) result.push_back(materialize(model, artifact)); + for (const auto& companion : model.companions) pull(companion); + }; + pull(repo); + return result; +} + +std::string +indexed_models_json() { + const Index index = load_index(); + Value output(Value::Object{}); + Value::Object defaults; + for (const auto& item : index.defaults) defaults.emplace(item.first, item.second); + output["defaults"] = std::move(defaults); + Value::Array models; + for (const auto& model : index.models) { + Value item(Value::Object{}); + item["repo"] = model.repo; + Value::Array aliases; + for (const auto& alias : model.aliases) aliases.emplace_back(alias); + item["aliases"] = std::move(aliases); + item["revision"] = model.revision; + item["license"] = model.license; + item["license_url"] = model.license_url; + Value::Array companions; + for (const auto& companion : model.companions) companions.emplace_back(companion); + item["companions"] = std::move(companions); + Value::Array roles; + for (const auto& artifact : model.artifacts) roles.emplace_back(artifact.role); + item["roles"] = std::move(roles); + Value::Array commands; + for (const auto& command : commands_for(model)) commands.emplace_back(command); + item["commands"] = std::move(commands); + Value::Array default_roles; + for (const auto& role : defaults_for(index, model)) default_roles.emplace_back(role); + item["default_for"] = std::move(default_roles); + models.emplace_back(std::move(item)); + } + output["models"] = std::move(models); + return output.dump(2) + "\n"; +} + +std::string +indexed_models_text() { + const Index index = load_index(); + std::ostringstream output; + auto has_role = [](const Model& model, const std::string& role) { + return std::any_of( + model.artifacts.begin(), model.artifacts.end(), + [&](const Artifact& artifact) { return artifact.role == role; }); + }; + auto is_default = [&](const Model& model) { + return std::any_of(index.defaults.begin(), index.defaults.end(), [&](const auto& item) { + return item.second == model.repo; + }); + }; + auto print_model = [&](const Model& model, const std::string& components) { + const std::string name = model.aliases.empty() ? model.repo : model.aliases.front(); + output << " " << (is_default(model) ? "* " : " ") << name; + if (!components.empty()) + output << " (" << components << ')'; + output << "\n repo: " << model.repo << '\n'; + if (model.aliases.size() > 1) { + output << " also: "; + for (size_t i = 1; i < model.aliases.size(); ++i) { + if (i > 1) + output << ", "; + output << model.aliases[i]; + } + output << '\n'; + } + if (!model.companions.empty()) { + output << " pulls: "; + for (size_t i = 0; i < model.companions.size(); ++i) { + if (i) + output << ", "; + const Model& companion = find_model(index, model.companions[i]); + output << (companion.aliases.empty() ? companion.repo : companion.aliases.front()); + } + output << '\n'; + } + }; + + output << "Available models (* = command default)\n\n" + << "ASR — transcribe, bench, serve\n"; + for (const auto& model : index.models) + if (has_role(model, "asr")) + print_model(model, ""); + + output << "\nDiarization — diarize, transcribe --diarize, serve\n"; + for (const auto& model : index.models) + if (has_role(model, "diarization")) + print_model(model, ""); + + output << "\nTTS — synthesize, serve\n"; + for (const auto& model : index.models) { + const bool tts = has_role(model, "tts"); + const bool tokenizer = has_role(model, "tokenizer"); + const bool codec = has_role(model, "codec"); + if (tts || tokenizer || codec) { + std::string components; + if (tts) + components = "speech model"; + if (tokenizer) + components += (components.empty() ? "" : " + ") + std::string("tokenizer"); + if (codec) + components += (components.empty() ? "" : " + ") + std::string("codec"); + print_model(model, components); + } + } + output << "\nUse a short name or full repository ID wherever MODEL is accepted.\n" + << "Download ahead of time with: nemo-speech pull \n"; + return output.str(); +} + +fs::path +model_downloader_executable() { +#if defined(_WIN32) + std::wstring buffer(32768, L'\0'); + const DWORD size = SearchPathW( + nullptr, L"curl.exe", nullptr, static_cast(buffer.size()), buffer.data(), nullptr); + if (size == 0 || size >= buffer.size()) + return {}; + buffer.resize(size); + return fs::path(buffer); +#else + std::istringstream paths(std::getenv("PATH") ? std::getenv("PATH") : ""); + std::string directory; + while (std::getline(paths, directory, ':')) { + const fs::path candidate = fs::path(directory.empty() ? "." : directory) / "curl"; + if (access(candidate.c_str(), X_OK) == 0) + return fs::absolute(candidate); + } + return {}; +#endif +} diff --git a/app/model_store.h b/app/model_store.h new file mode 100644 index 0000000..3800130 --- /dev/null +++ b/app/model_store.h @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include + +struct PulledModelArtifact { + std::string repo; + std::string role; + std::filesystem::path path; + bool cached = false; +}; + +std::filesystem::path resolve_indexed_model_file( + const std::string& reference, const std::string& role, const std::string& description); +std::filesystem::path resolve_indexed_model_directory( + const std::string& reference, const std::string& role, const std::string& description); + +std::vector pull_indexed_model(const std::string& repo); +std::string indexed_models_json(); +std::string indexed_models_text(); +std::filesystem::path model_downloader_executable(); diff --git a/app/model_utils.cpp b/app/model_utils.cpp index 4bf528e..a43ec8c 100644 --- a/app/model_utils.cpp +++ b/app/model_utils.cpp @@ -12,6 +12,7 @@ #include "ggml.h" #include "gguf.h" #include "json.h" +#include "model_store.h" namespace { namespace fs = std::filesystem; @@ -43,6 +44,18 @@ require_model_directory(const std::string& path, const std::string& description) return require_path(path, description, true); } +fs::path +resolve_model_file( + const std::string& reference, const std::string& role, const std::string& description) { + return resolve_indexed_model_file(reference, role, description); +} + +fs::path +resolve_model_directory( + const std::string& reference, const std::string& role, const std::string& description) { + return resolve_indexed_model_directory(reference, role, description); +} + std::string inspect_gguf_json(const fs::path& path) { gguf_init_params params{true, nullptr}; diff --git a/app/model_utils.h b/app/model_utils.h index 7d0a2eb..5f6334d 100644 --- a/app/model_utils.h +++ b/app/model_utils.h @@ -9,4 +9,9 @@ std::filesystem::path require_model_file(const std::string& path, const std::str std::filesystem::path require_model_directory( const std::string& path, const std::string& description); +std::filesystem::path resolve_model_file( + const std::string& reference, const std::string& role, const std::string& description); +std::filesystem::path resolve_model_directory( + const std::string& reference, const std::string& role, const std::string& description); + std::string inspect_gguf_json(const std::filesystem::path& path); diff --git a/app/serve.cpp b/app/serve.cpp index c0dfd8f..de4e2ab 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -51,15 +51,18 @@ request_shutdown(int) { [[maybe_unused]] std::string optional_model( - const std::string& reference, const std::string& description, bool required, - bool directory = false) { + const std::string& reference, const std::string& role, const std::string& description, + bool required, bool directory = false) { if (reference.empty()) { - if (required) + if (!required) + return {}; + if (role.empty()) throw MissingModelError(description + " path is required"); - return {}; } - return (directory ? require_model_directory(reference, description) - : require_model_file(reference, description)) + return (role.empty() ? (directory ? require_model_directory(reference, description) + : require_model_file(reference, description)) + : (directory ? resolve_model_directory(reference, role, description) + : resolve_model_file(reference, role, description))) .string(); } @@ -356,7 +359,8 @@ run_server(int argc, char** argv) { !pnc_model.empty() || !itn_model.empty(); const auto asr_path = asr_enabled == 0 ? std::string() : resolve("ASR model", [&] { return optional_model( - asr_model.empty() ? asr_config.model.path : asr_model, "ASR model", asr_requested); + asr_model.empty() ? asr_config.model.path : asr_model, "asr", "ASR model", + asr_requested); }); if (!asr_path.empty()) { if (device_set) @@ -366,25 +370,25 @@ run_server(int argc, char** argv) { if (asr_enabled != 0 && (asr_requested || !asr_path.empty())) { asr_config.vad.model_path = resolve("VAD model", [&] { return optional_model( - vad_model.empty() ? asr_config.vad.model_path : vad_model, "VAD model", false); + vad_model.empty() ? asr_config.vad.model_path : vad_model, "", "VAD model", false); }); #if defined(NEMO_SPEECH_CLI_DIAR) if (asr_config.diar.model_path.empty() && !diar_config.model_path.empty()) asr_config.diar = diar_config; asr_config.diar.model_path = resolve("diarization model", [&] { return optional_model( - diar_model.empty() ? asr_config.diar.model_path : diar_model, "diarization model", - false); + diar_model.empty() ? asr_config.diar.model_path : diar_model, "diarization", + "diarization model", false); }); #endif asr_config.postproc.pnc_model_path = resolve("punctuation model", [&] { return optional_model( - pnc_model.empty() ? asr_config.postproc.pnc_model_path : pnc_model, + pnc_model.empty() ? asr_config.postproc.pnc_model_path : pnc_model, "", "punctuation and capitalization model", false); }); asr_config.postproc.itn_model_dir = resolve("ITN grammar", [&] { return optional_model( - itn_model.empty() ? asr_config.postproc.itn_model_dir : itn_model, "ITN model", + itn_model.empty() ? asr_config.postproc.itn_model_dir : itn_model, "", "ITN model", false, true); }); } @@ -396,8 +400,8 @@ run_server(int argc, char** argv) { #endif standalone_diar = resolve("diarization model", [&] { return optional_model( - diar_model.empty() ? diar_config.model_path : diar_model, "diarization model", - false); + diar_model.empty() ? diar_config.model_path : diar_model, "diarization", + "diarization model", false); }); #endif #if defined(NEMO_SPEECH_CLI_NMT) @@ -405,7 +409,7 @@ run_server(int argc, char** argv) { nmt_enabled > 0 || !nmt_model.empty() || !nmt_config.model.path.empty(); const auto nmt_path = nmt_enabled == 0 ? std::string() : resolve("NMT model", [&] { return optional_model( - nmt_model.empty() ? nmt_config.model.path : nmt_model, "translation model", + nmt_model.empty() ? nmt_config.model.path : nmt_model, "", "translation model", nmt_requested); }); if (!nmt_path.empty()) { @@ -420,24 +424,24 @@ run_server(int argc, char** argv) { !tokenizer_model.empty() || !tn_model.empty(); const auto magpie_path = tts_enabled == 0 ? std::string() : resolve("TTS model", [&] { return optional_model( - tts_model.empty() ? tts_config.runtime.magpie_model : tts_model, "MagpieTTS model", - tts_requested); + tts_model.empty() ? tts_config.runtime.magpie_model : tts_model, "tts", + "MagpieTTS model", tts_requested); }); if (tts_enabled != 0 && (tts_requested || !magpie_path.empty())) { tts_config.runtime.codec_model = resolve("TTS codec model", [&] { return optional_model( - codec_model.empty() ? tts_config.runtime.codec_model : codec_model, + codec_model.empty() ? tts_config.runtime.codec_model : codec_model, "codec", "NanoCodec model", true); }); tts_config.tokenizer_model_dir = resolve("TTS tokenizer", [&] { return optional_model( tokenizer_model.empty() ? tts_config.tokenizer_model_dir : tokenizer_model, - "tokenizer model", true, true); + "tokenizer", "tokenizer model", true, true); }); tts_config.tn_model_dir = resolve("TTS normalization grammar", [&] { return optional_model( - tn_model.empty() ? tts_config.tn_model_dir : tn_model, "text normalization model", - false, true); + tn_model.empty() ? tts_config.tn_model_dir : tn_model, "", + "text normalization model", false, true); }); } #endif @@ -603,7 +607,7 @@ print_serve_help(const char* program) { std::printf( "Usage: %s serve [options]\n\n" "Start the OpenAI-compatible HTTP API and browser playground. Models are\n" - "provided as explicit local paths or through a YAML configuration file.\n\n" + "provided as local paths, indexed names, or through a YAML configuration.\n\n" "Server:\n" " --host ADDRESS Bind address (default: 127.0.0.1)\n" " --port N HTTP port (default: 8080)\n" @@ -623,18 +627,18 @@ print_serve_help(const char* program) { " Realtime ASR: WebSocket /v1/realtime\n\n" "Models:\n" #if defined(NEMO_SPEECH_CLI_ASR) - " --asr-model MODEL ASR GGUF path\n" + " --asr-model MODEL ASR GGUF path or indexed model\n" " --vad-model MODEL Optional VAD model\n" " --pnc-model MODEL Optional punctuation model\n" " --itn-model-dir MODEL Optional ITN grammar directory\n" #endif #if defined(NEMO_SPEECH_CLI_DIAR) - " --diar-model MODEL Optional Sortformer model\n" + " --diar-model MODEL Optional Sortformer path or indexed model\n" #endif #if defined(NEMO_SPEECH_CLI_TTS) - " --tts-model MODEL Optional MagpieTTS model\n" - " --codec-model MODEL NanoCodec model used by TTS\n" - " --tokenizer-dir MODEL TTS tokenizer assets\n" + " --tts-model MODEL Optional MagpieTTS path or indexed model\n" + " --codec-model MODEL NanoCodec path or indexed model\n" + " --tokenizer-dir MODEL TTS tokenizer directory or indexed model\n" " --tn-model-dir MODEL Optional TTS text-normalization assets\n" #endif #if defined(NEMO_SPEECH_CLI_NMT) diff --git a/app/synthesize.cpp b/app/synthesize.cpp index 5c7388a..a820c87 100644 --- a/app/synthesize.cpp +++ b/app/synthesize.cpp @@ -60,12 +60,14 @@ write_audio(const std::filesystem::path& path, const std::string& audio, bool fo void print_synthesize_help(const char* program) { std::printf( - "Usage: %s synthesize TEXT --magpie-model MODEL --codec-model MODEL\n" - " --tokenizer-dir DIR [options]\n\n" + "Usage: %s synthesize TEXT [options]\n\n" "Options:\n" - " --magpie-model PATH MagpieTTS GGUF\n" - " --codec-model PATH NanoCodec GGUF\n" - " --tokenizer-dir DIR Extracted tokenizer assets\n" + " --magpie-model MODEL MagpieTTS GGUF path or indexed HF repo\n" + " (default: nvidia/magpie_tts_multilingual_357m)\n" + " --codec-model MODEL NanoCodec GGUF path or indexed HF repo\n" + " (default: nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps)\n" + " --tokenizer-dir MODEL Tokenizer directory or indexed HF repo\n" + " (default: MagpieTTS repository)\n" " --tn-model-dir DIR Optional text-normalization grammars\n" " -i, --input PATH Read text from a UTF-8 file\n" " -o, --output PATH Output path (default: speech.wav; '-' = stdout)\n" @@ -190,11 +192,12 @@ command_synthesize(int argc, char** argv) { throw std::invalid_argument("--json cannot be combined with --output -"); parsed.runtime.magpie_model = - require_model_file(parsed.runtime.magpie_model, "MagpieTTS model").string(); + resolve_model_file(parsed.runtime.magpie_model, "tts", "MagpieTTS model").string(); parsed.runtime.codec_model = - require_model_file(parsed.runtime.codec_model, "NanoCodec model").string(); + resolve_model_file(parsed.runtime.codec_model, "codec", "NanoCodec model").string(); parsed.tokenizer_model_dir = - require_model_directory(parsed.tokenizer_model_dir, "tokenizer model").string(); + resolve_model_directory(parsed.tokenizer_model_dir, "tokenizer", "tokenizer model") + .string(); if (!parsed.tn_model_dir.empty()) parsed.tn_model_dir = require_model_directory(parsed.tn_model_dir, "text normalization model").string(); diff --git a/app/transcribe.cpp b/app/transcribe.cpp index ec79134..a5fab87 100644 --- a/app/transcribe.cpp +++ b/app/transcribe.cpp @@ -3,6 +3,10 @@ #include #include +#if defined(NEMO_SPEECH_CLI_LIVE) +#include +#include +#endif #include #include #include @@ -17,6 +21,9 @@ #include "cli_util.h" #include "commands.h" #include "engine_registry.h" +#if defined(NEMO_SPEECH_CLI_LIVE) +#include "microphone_capture.h" +#endif #include "model_utils.h" #include "parameter_parser.h" #include "recognizer.h" @@ -65,6 +72,7 @@ struct Options { bool punctuation = true; bool verbatim = false; bool diarize = false; + bool live = false; bool stream = false; bool warmup = true; bool batching = true; @@ -162,6 +170,10 @@ parse_options(int argc, char** argv) { o.force = true; else if (arg == "--word-times") o.word_times = true; +#if defined(NEMO_SPEECH_CLI_LIVE) + else if (arg == "--live") + o.live = true; +#endif else if (arg == "--stream") o.stream = true; else if (arg == "--no-warmup") @@ -198,10 +210,18 @@ parse_options(int argc, char** argv) { else throw std::invalid_argument("unexpected argument: " + arg); } - if (o.input.empty()) + if (o.live && !o.input.empty()) + throw std::invalid_argument("--live does not accept an input file or directory"); + if (!o.live && o.input.empty()) throw std::invalid_argument("an input WAV file or directory is required"); if (!o.output.empty() && !o.output_dir.empty()) throw std::invalid_argument("use only one of --output and --output-dir"); + if (o.live && !o.output_dir.empty()) + throw std::invalid_argument("--output-dir is not valid with --live; use --output"); + if (o.live && o.recursive) + throw std::invalid_argument("--recursive is not valid with --live"); + if (o.live && o.concurrency > 0) + throw std::invalid_argument("--concurrency is not valid with --live"); #if defined(NEMO_SPEECH_CLI_NMT) if (!o.translate_to.empty() && (o.format == OutputFormat::Srt || o.format == OutputFormat::Vtt)) throw std::invalid_argument("--translate-to currently supports text and json output"); @@ -211,9 +231,8 @@ parse_options(int argc, char** argv) { return o; } -Transcript -transcribe_one(asr::Recognizer& recognizer, const Options& options, const fs::path& path) { - const auto audio = nemo_speech::audio::load_wav_file(path.string()); +asr::AsrRequestOptions +make_request_options(const Options& options) { asr::AsrRequestOptions request = options.request; request.language_code = options.language; request.enable_word_time_offsets = options.word_times || options.format == OutputFormat::Json || @@ -224,33 +243,43 @@ transcribe_one(asr::Recognizer& recognizer, const Options& options, const fs::pa request.enable_speaker_diarization = options.diarize; if (!options.speech_contexts.empty()) request.speech_contexts.push_back({options.speech_contexts, options.speech_context_boost}); + return request; +} + +void +append_result(Transcript& transcript, const asr::Result& result) { + if (result.alternatives.empty()) + return; + const auto& alternative = result.alternatives.front(); + if (!alternative.transcript.empty()) { + if (!transcript.text.empty() && alternative.transcript.front() != '.' && + alternative.transcript.front() != ',' && alternative.transcript.front() != '!' && + alternative.transcript.front() != '?') + transcript.text += ' '; + transcript.text += alternative.transcript; + } + transcript.confidence = alternative.confidence; + transcript.audio_seconds = std::max(transcript.audio_seconds, result.audio_processed); + for (const auto& language : alternative.language_codes) + if (std::find(transcript.languages.begin(), transcript.languages.end(), language) == + transcript.languages.end()) + transcript.languages.push_back(language); + for (const auto& word : alternative.words) + transcript.words.push_back( + {word.word, word.start_time, word.end_time, word.confidence, word.speaker_tag}); +} + +Transcript +transcribe_one(asr::Recognizer& recognizer, const Options& options, const fs::path& path) { + const auto audio = nemo_speech::audio::load_wav_file(path.string()); + const asr::AsrRequestOptions request = make_request_options(options); Transcript transcript; - auto append = [&](const asr::Result& result) { - if (result.alternatives.empty()) - return; - const auto& alternative = result.alternatives.front(); - if (!alternative.transcript.empty()) { - if (!transcript.text.empty() && alternative.transcript.front() != '.' && - alternative.transcript.front() != ',' && alternative.transcript.front() != '!' && - alternative.transcript.front() != '?') - transcript.text += ' '; - transcript.text += alternative.transcript; - } - transcript.confidence = alternative.confidence; - transcript.audio_seconds = std::max(transcript.audio_seconds, result.audio_processed); - for (const auto& language : alternative.language_codes) - if (std::find(transcript.languages.begin(), transcript.languages.end(), language) == - transcript.languages.end()) - transcript.languages.push_back(language); - for (const auto& word : alternative.words) - transcript.words.push_back( - {word.word, word.start_time, word.end_time, word.confidence, word.speaker_tag}); - }; if (!options.stream) { - append(recognizer.recognize( - audio.samples.data(), audio.samples.size(), request, options.language, - audio.sample_rate)); + append_result( + transcript, recognizer.recognize( + audio.samples.data(), audio.samples.size(), request, options.language, + audio.sample_rate)); } else { auto stream = recognizer.streaming_recognize(request, options.language); const size_t chunk = std::max(1, audio.sample_rate * 160 / 1000); @@ -259,18 +288,111 @@ transcribe_one(asr::Recognizer& recognizer, const Options& options, const fs::pa stream->push(audio.samples.data() + offset, count, audio.sample_rate); while (auto result = stream->next()) { if (result->is_final) - append(*result); + append_result(transcript, *result); else break; } } - append(stream->finish()); + append_result(transcript, stream->finish()); } if (transcript.text.empty() && transcript.words.empty()) transcript.audio_seconds = audio.samples.size() / static_cast(audio.sample_rate); return transcript; } +#if defined(NEMO_SPEECH_CLI_LIVE) +volatile std::sig_atomic_t live_running = 1; + +void +stop_live_capture(int /*signal*/) { + live_running = 0; +} + +class SignalHandlerGuard { + public: + SignalHandlerGuard() : previous_(std::signal(SIGINT, stop_live_capture)) {} + ~SignalHandlerGuard() { std::signal(SIGINT, previous_); } + + SignalHandlerGuard(const SignalHandlerGuard&) = delete; + SignalHandlerGuard& operator=(const SignalHandlerGuard&) = delete; + + private: + using Handler = void (*)(int); + Handler previous_; +}; + +Transcript +transcribe_live(asr::Recognizer& recognizer, const Options& options) { + auto stream = recognizer.streaming_recognize(make_request_options(options), options.language); + nemo_speech::cli::MicrophoneCapture microphone; + microphone.start(); + + if (!cli_quiet() && !cli_json()) + std::fprintf( + stderr, "[live] listening on \"%s\" at %d Hz; press Ctrl-C to stop\n", + microphone.device_name().c_str(), microphone.sample_rate()); + + live_running = 1; + SignalHandlerGuard signal_guard; + Transcript transcript; + std::string last_interim; + size_t captured_samples = 0; + + auto drain_results = [&] { + while (auto result = stream->next()) { + if (result->is_final) { + append_result(transcript, *result); + last_interim.clear(); + if (!cli_quiet() && !cli_json() && !result->alternatives.empty() && + !result->alternatives.front().transcript.empty()) + std::fprintf( + stderr, "[live final @ %.2fs] %s\n", result->audio_processed, + result->alternatives.front().transcript.c_str()); + continue; + } + if (!result->alternatives.empty()) { + const std::string& text = result->alternatives.front().transcript; + if (!text.empty() && text != last_interim) { + if (!cli_quiet() && !cli_json()) + std::fprintf( + stderr, "[live partial @ %.2fs] %s\n", result->audio_processed, + text.c_str()); + last_interim = text; + } + } + // By contract, an interim is the last result until more audio is pushed. + break; + } + }; + + while (live_running) { + auto samples = microphone.drain(); + if (samples.empty()) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + continue; + } + captured_samples += samples.size(); + stream->push(samples.data(), samples.size(), microphone.sample_rate()); + drain_results(); + } + + microphone.stop(); + auto tail = microphone.drain(); + if (!tail.empty()) { + captured_samples += tail.size(); + stream->push(tail.data(), tail.size(), microphone.sample_rate()); + drain_results(); + } + append_result(transcript, stream->finish()); + transcript.audio_seconds = std::max( + transcript.audio_seconds, captured_samples / static_cast(microphone.sample_rate())); + if (!cli_quiet() && !cli_json()) + std::fprintf( + stderr, "[live] stopped after %.2fs of captured audio\n", transcript.audio_seconds); + return transcript; +} +#endif + std::string timestamp(int milliseconds, bool vtt) { milliseconds = std::max(0, milliseconds); @@ -350,12 +472,22 @@ extension(OutputFormat format) { void print_transcribe_help(const char* program) { std::printf( - "Usage: %s transcribe INPUT [--model MODEL] [options]\n\n" - "Transcribe one WAV file or every WAV file in a directory. Directory\n" + "Usage: %s transcribe INPUT [--model MODEL] [options]\n" +#if defined(NEMO_SPEECH_CLI_LIVE) + " %s transcribe --live [--model MODEL] [options]\n\n" + "Transcribe a WAV file, directory, or the default microphone. Directory\n" +#else + "\n" + "Transcribe a WAV file or directory. Directory\n" +#endif "work shares one recognizer; --concurrency feeds multiple utterances to\n" "that recognizer so compatible inference is batched on the GPU.\n\n" "Options:\n" - " -m, --model MODEL Local ASR GGUF path\n" + " -m, --model MODEL ASR GGUF path or indexed HF repo\n" + " (default: nvidia/nemotron-3.5-asr-streaming-0.6b)\n" +#if defined(NEMO_SPEECH_CLI_LIVE) + " --live Transcribe the default microphone until Ctrl-C\n" +#endif " -l, --language CODE Language code or prompt\n" " --device, --backend DEVICE\n" " auto, cpu, cuda[:N], metal, or vulkan[:N]\n" @@ -376,7 +508,7 @@ print_transcribe_help(const char* program) { #endif " --no-punctuation Disable automatic punctuation\n" " --verbatim Disable ordinary ITN\n" - " --stream Exercise cache-aware streaming inference\n" + " --stream Stream chunks from a recorded WAV input\n" " --max-alternatives N Request N-best hypotheses\n" " --speech-context PHRASE Add a decoder boost phrase (repeatable)\n" " --speech-context-boost N Boost applied to speech context phrases\n" @@ -389,7 +521,12 @@ print_transcribe_help(const char* program) { " --no-warmup Skip model warmup\n" " --no-batching Disable dynamic batching\n" " --force Replace existing output files\n", - program); + program +#if defined(NEMO_SPEECH_CLI_LIVE) + , + program +#endif + ); } int @@ -400,25 +537,31 @@ command_transcribe(int argc, char** argv) { return 0; } Options options = parse_options(argc, argv); - const auto inputs = collect_wav_inputs(options.input, options.recursive); - const bool directory = fs::is_directory(options.input); + std::vector inputs; + bool directory = false; + if (!options.live) { + inputs = collect_wav_inputs(options.input, options.recursive); + directory = fs::is_directory(options.input); + } if (directory && !options.output.empty()) throw std::invalid_argument("--output is only valid for one input; use --output-dir"); if (!directory && !options.output_dir.empty()) throw std::invalid_argument("--output-dir is only valid for a directory input"); const int configured_gpu = options.device_set ? options.gpu : options.engine.backend.gpu; - const int concurrency = std::min( - options.concurrency > 0 ? options.concurrency - : (directory && configured_gpu >= 0 ? 4 : 1), - inputs.size()); + const int concurrency = + options.live ? 1 + : std::min( + options.concurrency > 0 ? options.concurrency + : (directory && configured_gpu >= 0 ? 4 : 1), + inputs.size()); asr::RecognizerConfig config = options.engine; config.log_status = !cli_quiet() && !cli_json(); if (options.device_set) config.backend.gpu = options.gpu; config.model.path = - require_model_file( - options.model.empty() ? config.model.path : options.model, "ASR model") + resolve_model_file( + options.model.empty() ? config.model.path : options.model, "asr", "ASR model") .string(); if (!options.vad_model.empty() || !config.vad.model_path.empty()) config.vad.model_path = @@ -426,11 +569,11 @@ command_transcribe(int argc, char** argv) { options.vad_model.empty() ? config.vad.model_path : options.vad_model, "VAD model") .string(); - if (!options.diar_model.empty() || !config.diar.model_path.empty()) + if (options.diarize || !options.diar_model.empty() || !config.diar.model_path.empty()) config.diar.model_path = - require_model_file( + resolve_model_file( options.diar_model.empty() ? config.diar.model_path : options.diar_model, - "diarization model") + "diarization", "diarization model") .string(); if (!options.itn_model_dir.empty() || !config.postproc.itn_model_dir.empty()) config.postproc.itn_model_dir = @@ -451,10 +594,16 @@ command_transcribe(int argc, char** argv) { std::max(config.batching.max_queue_depth, concurrency * 4); config.batching.state_arena_slots = std::max(config.batching.state_arena_slots, concurrency); - if (cli_verbose()) - std::fprintf( - stderr, "transcribe: model=%s inputs=%zu concurrency=%d device=%d\n", - config.model.path.c_str(), inputs.size(), concurrency, config.backend.gpu); + if (cli_verbose()) { + if (options.live) + std::fprintf( + stderr, "transcribe: model=%s input=microphone device=%d\n", + config.model.path.c_str(), config.backend.gpu); + else + std::fprintf( + stderr, "transcribe: model=%s inputs=%zu concurrency=%d device=%d\n", + config.model.path.c_str(), inputs.size(), concurrency, config.backend.gpu); + } nemo_speech::EngineRegistryConfig registry_config; registry_config.asr = true; #if defined(NEMO_SPEECH_CLI_NMT) @@ -481,6 +630,31 @@ command_transcribe(int argc, char** argv) { if (options.warmup) engines.warmup(); +#if defined(NEMO_SPEECH_CLI_LIVE) + if (options.live) { + Transcript transcript = transcribe_live(*recognizer, options); +#if defined(NEMO_SPEECH_CLI_NMT) + if (speech_translator && !transcript.text.empty()) { + const auto translated = speech_translator->translate_text( + transcript.text, options.language, options.translate_to, transcript.languages); + transcript.source_text = translated.transcript; + transcript.text = translated.text; + transcript.target_language = translated.language_code; + } +#endif + const std::string contents = + render(transcript, options.format, fs::path("")); + if (options.output.empty()) + std::fwrite(contents.data(), 1, contents.size(), stdout); + else { + write_text_file(options.output, contents, options.force); + if (!cli_quiet()) + std::fprintf(stderr, "microphone -> %s\n", options.output.string().c_str()); + } + return 0; + } +#endif + std::vector transcripts(inputs.size()); std::vector errors(inputs.size()); std::atomic next{0}; diff --git a/docker/Dockerfile b/docker/Dockerfile index 47cfb0a..5621ece 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -92,7 +92,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ protobuf-compiler-grpc \ libabsl-dev \ libre2-dev \ - portaudio19-dev \ autoconf \ automake \ libtool \ @@ -116,6 +115,7 @@ COPY THIRD_PARTY_NOTICES.md /work/ COPY README.md CONTRIBUTING.md /work/ COPY docs /work/docs COPY config /work/config +COPY models /work/models # Core submodules: ggml (every backend) and llama.cpp (NMT decoder; built only # when ENABLE_NMT=ON, copied unconditionally so the layer cache is stable). COPY ggml /work/ggml @@ -178,6 +178,7 @@ RUN cmake -G Ninja -S /work -B /work/build \ ${CUDA_ARCH:+-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}} \ -DNEMO_SPEECH_BUILD_GRPC=${ENABLE_GRPC} \ -DNEMO_SPEECH_BUILD_HTTP=${ENABLE_HTTP} \ + -DNEMO_SPEECH_BUILD_MIC_CAPTURE=OFF \ -DNEMO_SPEECH_WITH_FLASHLIGHT=${ENABLE_FLASHLIGHT} \ -DNEMO_SPEECH_WITH_NORM=${ENABLE_NORM} \ -DNEMO_SPEECH_TTS_WITH_JA=${ENABLE_TTS_JA} \ @@ -195,7 +196,7 @@ RUN mkdir -p /out/bin /out/lib /out/share/nemo-speech \ | grep -q 'Shared library: \[libkenlm.so\]'; \ fi \ && for binary in \ - nemo-speech riva_server transcribe_file transcribe_live diarize_file synthesize_text \ + nemo-speech riva_server transcribe_file diarize_file synthesize_text \ translate_text speech_translate_file; \ do \ if [ -f "/work/build/bin/${binary}" ]; then \ @@ -213,6 +214,8 @@ RUN mkdir -p /out/bin /out/lib /out/share/nemo-speech \ done \ && cp /work/LICENSE /work/NOTICE /work/THIRD_PARTY_NOTICES.md \ /out/share/licenses/nemo-speech/ \ + && install -Dm0644 /work/build/share/nemo-speech/model-index.json \ + /out/share/nemo-speech/model-index.json \ && license_dir=/out/share/licenses/nemo-speech/third_party \ && install -Dm0644 /work/ggml/LICENSE "$license_dir/ggml/LICENSE" \ && install -Dm0644 /work/llama.cpp/LICENSE "$license_dir/llama.cpp/LICENSE" \ diff --git a/docs/asr/models.md b/docs/asr/models.md index dfc6f70..75fcf80 100644 --- a/docs/asr/models.md +++ b/docs/asr/models.md @@ -1,20 +1,23 @@ # ASR models The runtime loads one **GGUF** per ASR model. Ready-to-run Q8 GGUFs are -published alongside the original checkpoints on Hugging Face. Install the -Hugging Face CLI if needed: +published alongside the original checkpoints on Hugging Face and indexed by +the CLI: ```bash -pip install -U huggingface_hub +nemo-speech model list +nemo-speech pull nemotron-3.5 ``` +`nemotron-3.5` is the default when `--model` is omitted. A short name, full +repository ID, or existing local GGUF path can be passed to `--model`. + ## Parakeet CTC (1.1B, offline / buffered streaming) Hugging Face: [nvidia/parakeet-ctc-1.1b](https://huggingface.co/nvidia/parakeet-ctc-1.1b) ```bash -hf download nvidia/parakeet-ctc-1.1b \ - parakeet-ctc-1.1b.q8_0.gguf --local-dir models +nemo-speech pull parakeet-ctc ``` ## Parakeet TDT (0.6B v3, multilingual, offline transducer) @@ -24,8 +27,7 @@ frame span. 25 European languages, self-punctuating. Hugging Face: [nvidia/parakeet-tdt-0.6b-v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) ```bash -hf download nvidia/parakeet-tdt-0.6b-v3 \ - parakeet-tdt-0.6b-v3.q8_0.gguf --local-dir models +nemo-speech pull parakeet-tdt ``` The model is not cache-aware trained: inference is full-utterance only. @@ -38,8 +40,7 @@ Streaming requests are rejected with an error; use offline recognition Hugging Face: [nvidia/nemotron-speech-streaming-en-0.6b](https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b) ```bash -hf download nvidia/nemotron-speech-streaming-en-0.6b \ - nemotron-speech-streaming-en-0.6b.q8_0.gguf --local-dir models +nemo-speech pull nemotron-en ``` ## Nemotron 3.5 (0.6B, multilingual, prompt-conditioned RNNT) @@ -49,8 +50,7 @@ across 40+ language-locales (`EncDecRNNTBPEModelWithPrompt`). Hugging Face: [nvidia/nemotron-3.5-asr-streaming-0.6b](https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b) ```bash -hf download nvidia/nemotron-3.5-asr-streaming-0.6b \ - nemotron-3.5-asr-streaming-0.6b.q8_0.gguf --local-dir models +nemo-speech pull nemotron-3.5 ``` The GGUF contains the prompt metadata (`asr.rnnt.num_prompts`, diff --git a/docs/build.md b/docs/build.md index 1436aee..ed50e98 100644 --- a/docs/build.md +++ b/docs/build.md @@ -55,7 +55,7 @@ Initialize the submodules needed by the selected components: ```bash git submodule update --init ggml git submodule update --init third_party/cpp-httplib # HTTP server only -git submodule update --init llama.cpp # NMT only +git submodule update --init llama.cpp # ASR live capture or NMT git submodule update --init proto/riva-common # gRPC only ``` @@ -122,6 +122,7 @@ runtime tradeoffs. | `NEMO_SPEECH_BUILD_TTS` | ON | TTS runtime, C ABI, and CLI | | `NEMO_SPEECH_BUILD_NMT` | OFF | NMT through llama.cpp | | `NEMO_SPEECH_BUILD_CLI` | ON | Unified `nemo-speech` executable | +| `NEMO_SPEECH_BUILD_MIC_CAPTURE` | ON | Microphone capture in the CLI and live example | | `NEMO_SPEECH_BUILD_HTTP` | OFF | HTTP, realtime WebSocket, and playground | | `NEMO_SPEECH_BUILD_GRPC` | OFF | Riva-compatible gRPC adapters | | `NEMO_SPEECH_BUILD_EXAMPLES` | OFF | Public in-process C ABI examples | diff --git a/docs/cli.md b/docs/cli.md index af00958..9cc0689 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,17 +1,60 @@ # Command-line guide -`nemo-speech` is the primary local interface. Inference commands accept -explicit local model paths. +`nemo-speech` is the primary local interface. Inference commands accept local +model paths, indexed Hugging Face repository IDs, or short model names. ASR, +diarization, and TTS also have ready-to-run defaults. Run `nemo-speech --help` for the command inventory and `nemo-speech help ` for the options compiled into the installed build. +## Models and cache + +List the models built into this CLI release: + +```bash +nemo-speech model list +nemo-speech --json model list +``` + +The human output is grouped by ASR, diarization, and TTS. `*` marks models used +by default. The JSON form also exposes every alias, artifact role, applicable +command, companion, pinned revision, and license. + +Inference downloads a missing indexed model automatically. You can download it +ahead of time with either a short name or the full repository ID: + +```bash +nemo-speech pull nemotron-3.5 +nemo-speech pull nvidia/nemotron-3.5-asr-streaming-0.6b +nemo-speech pull magpie +``` + +Pulling `magpie` also installs its tokenizer assets and the required NanoCodec +companion. Downloads use the system `curl` executable, follow HTTPS only, +resume interrupted regular-file downloads, and are accepted only after the +pinned size and SHA-256 match. Concurrent processes share per-artifact cache +locks. Run `nemo-speech doctor` to confirm that `curl` is available. + +The cache location is platform-specific: + +| Platform | Default cache | +|---|---| +| macOS | `~/Library/Caches/NeMoSpeech/models` | +| Linux | `${XDG_CACHE_HOME:-~/.cache}/nemo-speech/models` | +| Windows | `%LOCALAPPDATA%\NeMoSpeech\models` | + +Set `NEMO_SPEECH_MODEL_DIR` to use another location. Passing an existing local +GGUF path or tokenizer directory always takes precedence and does not use the +network. + ## Transcribe audio Transcribe one WAV file: ```bash +nemo-speech transcribe recording.wav +nemo-speech transcribe recording.wav --model nemotron-en nemo-speech transcribe recording.wav --model ./models/asr.q8_0.gguf ``` @@ -19,6 +62,25 @@ The file CLI accepts mono or stereo PCM16 and float32 WAV input from 8-96 kHz. It downmixes and resamples to the model rate. Unsupported containers or codecs produce an error with a conversion command. +### Transcribe a microphone live + +```bash +nemo-speech transcribe --live \ + --backend auto +``` + +The command captures the system's default microphone and prints interim and +endpointed transcripts to stderr while you speak. Press Ctrl-C once to stop; +the stream is flushed and the complete final transcript is written to stdout. +Use `--output transcript.txt` to write it to a file, or select `json`, `srt`, +or `vtt` with `--format`. + +Live capture is compiled directly into the CLI through miniaudio and uses the +native host audio API: CoreAudio on macOS, WASAPI on Windows, and ALSA or +PulseAudio on Linux. No PortAudio runtime is required. The operating system may +ask for microphone permission the first time; grant access to the terminal or +shell running `nemo-speech`. + ### Subtitles and structured output ```bash @@ -73,9 +135,9 @@ Use only the companion models needed by the workflow. Standalone diarization does not require an ASR model: ```bash -nemo-speech diarize meeting.wav --model sortformer.gguf +nemo-speech diarize meeting.wav nemo-speech diarize meeting.wav --format rttm --output meeting.rttm -nemo-speech diarize recordings/ --model sortformer.gguf \ +nemo-speech diarize recordings/ \ --format rttm --output-dir rttms --concurrency 4 ``` @@ -100,11 +162,7 @@ to a file. ## Synthesize speech ```bash -nemo-speech synthesize "Hello" \ - --magpie-model models/magpie-tts/magpie_tts_multilingual_357m.v2602.f16.gguf \ - --codec-model models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ - --tokenizer-dir models/magpie-tts/extracted \ - --output hello.wav +nemo-speech synthesize "Hello" --output hello.wav ``` ## Select a backend @@ -123,8 +181,8 @@ Run `nemo-speech doctor` to see the compiled backends and detected devices. ## Convert and inspect models -Published model repositories provide ready-to-run GGUFs. Use the converter when -working with a custom checkpoint or producing a different quantization: +The built-in index covers the published ready-to-run GGUFs. Use the converter +when working with a custom checkpoint or producing a different quantization: ```bash python convert_model.py custom-model.nemo --outfile custom-model.q8_0.gguf @@ -133,9 +191,9 @@ nemo-speech model info custom-model.q8_0.gguf The converter can also resolve Hugging Face repository IDs through the standard cache. See [model conversion](model-conversion.md) for the isolated Python -environment and supported model families. Model files remain local; pass their -paths explicitly or record a reusable multi-model setup in a -[YAML configuration file](server.md#engine-and-listener-configuration). +environment and supported model families. Custom files remain local; pass +their paths explicitly or record a reusable multi-model setup in a [YAML +configuration file](server.md#engine-and-listener-configuration). ## Benchmark diff --git a/docs/install.md b/docs/install.md index 5da5d37..fd3ba17 100644 --- a/docs/install.md +++ b/docs/install.md @@ -3,10 +3,10 @@ The installer selects a backend-matched native release containing the ASR, diarization, translation, and TTS CLI, HTTP API, realtime WebSocket endpoint, browser playground, SDK, and notices. It builds from source when a matching -archive is unavailable. Models are distributed separately and are never -downloaded when the server starts. Ready-to-run GGUFs are available from the -linked Hugging Face repositories in the [ASR](asr/models.md) and -[TTS](tts/models.md) model guides. +archive is unavailable. Models are distributed separately; inference commands +download missing indexed defaults on first use, while the server downloads a +model only when explicitly enabled with an indexed name. See [models and +cache](cli.md#models-and-cache). ## Linux and macOS @@ -49,6 +49,13 @@ explicit version, it builds that checkout's current branch. Override the source for a fork or local mirror with `NEMO_SPEECH_SOURCE_URL` and `NEMO_SPEECH_SOURCE_REF`. +Automatic model pulls require the `curl` executable. The Linux/macOS installer +also uses it for release downloads; the Windows installer uses PowerShell's +HTTPS support. macOS and current Windows releases include `curl`, while Linux +users can install it with their distribution package manager. `nemo-speech +doctor` reports whether model downloads are available. Existing local model +paths and already cached models still work if `curl` later becomes unavailable. + ## Windows Inspect [`scripts/install.ps1`](../scripts/install.ps1), then run from @@ -119,11 +126,14 @@ nemo-speech--windows--.zip ``` Linux aarch64 CUDA archives use `cuda12` or `cuda13` as the backend suffix. +On Apple Silicon, both `--backend metal` and `--backend cpu` install the +`macos-aarch64-metal` archive because it contains both runtime backends. To uninstall on Linux or macOS, remove the prefix printed during installation and `~/.local/bin/nemo-speech`; remove the two-line NeMo-Speech.cpp PATH entry from the shell startup file if the installer added it. On Windows, remove `%LOCALAPPDATA%\Programs\NeMoSpeech` (or the selected prefix) and that -prefix's `bin` directory from the current-user PATH. Models downloaded through -Hugging Face or another artifact tool are stored separately and are not removed -by uninstalling the runtime. +prefix's `bin` directory from the current-user PATH. The model cache is stored +separately and is not removed with the runtime: `~/Library/Caches/NeMoSpeech/models` +on macOS, `${XDG_CACHE_HOME:-~/.cache}/nemo-speech/models` on Linux, and +`%LOCALAPPDATA%\NeMoSpeech\models` on Windows. diff --git a/docs/tts/models.md b/docs/tts/models.md index 19e11be..3957613 100644 --- a/docs/tts/models.md +++ b/docs/tts/models.md @@ -1,34 +1,26 @@ # TTS models The TTS pipeline loads two GGUFs: a **MagpieTTS** token generator and a **NeMo -NanoCodec** decoder. Ready-to-run F16 GGUFs are published in their Hugging Face -repositories. Install the Hugging Face CLI if needed: +NanoCodec** decoder. The CLI downloads the complete default stack, including +Magpie's tokenizer assets, with one command: ```bash -pip install -U huggingface_hub +nemo-speech pull magpie +nemo-speech synthesize "Hello from Magpie Multilingual." --output output.wav ``` +`synthesize` performs the same verified pull automatically when its model +options are omitted. + ## MagpieTTS token generator Hugging Face: [nvidia/magpie_tts_multilingual_357m](https://huggingface.co/nvidia/magpie_tts_multilingual_357m) -```bash -# Download the v2602 GGUF and the original archive containing its tokenizer. -hf download nvidia/magpie_tts_multilingual_357m \ - --include magpie_tts_multilingual_357m.v2602.f16.gguf \ - --include magpie_tts_multilingual_357m.nemo \ - --local-dir models/magpie-tts - -# Extract the tokenizer assets loaded by the runtime. -mkdir -p models/magpie-tts/extracted -tar -xf models/magpie-tts/magpie_tts_multilingual_357m.nemo \ - -C models/magpie-tts/extracted -``` - **Tokenizer.** MagpieTTS's tokenizer assets live *inside* the `.nemo` archive - -they are not part of the GGUF. Extract the `.nemo` and pass that directory to -the server as `--tts.tokenizer-model-dir` (here -`models/magpie-tts/extracted`). +they are not part of the GGUF. The built-in pull extracts only the required, +pinned tokenizer members and verifies each one. For a custom Magpie checkpoint, +extract its `.nemo` archive and pass that directory as `--tokenizer-dir` or +`--tts.tokenizer-model-dir`. The model-specific IPA/text tokenizer assets are loaded from this directory. Japanese tokenization requires a build with `NEMO_SPEECH_TTS_WITH_JA=ON` (disabled by default), which builds Open JTalk, MeCab, and the NAIST dictionary. @@ -56,20 +48,13 @@ server, YAML, and offline runner examples. Hugging Face: [nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps](https://huggingface.co/nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps) (no tokenizer is needed for the codec decoder). -```bash -hf download nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps \ - nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ - --local-dir models/nano-codec -``` +Pull it independently with `nemo-speech pull nano-codec`. Pulling `magpie` +does this automatically because the two models must run together. -Run both models together: +Run the default stack: ```bash -nemo-speech synthesize "Hello from Magpie Multilingual." \ - --magpie-model models/magpie-tts/magpie_tts_multilingual_357m.v2602.f16.gguf \ - --codec-model models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ - --tokenizer-dir models/magpie-tts/extracted \ - --output output.wav +nemo-speech synthesize "Hello from Magpie Multilingual." --output output.wav ``` ## Converting custom TTS checkpoints diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a1cc672..20672d8 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -83,7 +83,7 @@ endif() # If it isn't found, transcribe_live is skipped (a STATUS message says so) # rather than failing the build - transcribe_file and the core library don't # need it. -if(NOT NEMO_SPEECH_BUILD_ASR) +if(NOT NEMO_SPEECH_BUILD_ASR OR NOT NEMO_SPEECH_BUILD_MIC_CAPTURE) return() endif() diff --git a/kernels/cublas_shim.cu b/kernels/cublas_shim.cu index 54e1475..4024426 100644 --- a/kernels/cublas_shim.cu +++ b/kernels/cublas_shim.cu @@ -25,10 +25,10 @@ #include #include -#include #include #include #include +#include #include #include diff --git a/models/index.json b/models/index.json new file mode 100644 index 0000000..b105028 --- /dev/null +++ b/models/index.json @@ -0,0 +1,189 @@ +{ + "schema_version": 1, + "defaults": { + "asr": "nvidia/nemotron-3.5-asr-streaming-0.6b", + "diarization": "nvidia/diar_streaming_sortformer_4spk-v2", + "tts": "nvidia/magpie_tts_multilingual_357m", + "codec": "nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps", + "tokenizer": "nvidia/magpie_tts_multilingual_357m" + }, + "models": [ + { + "repo": "nvidia/nemotron-3.5-asr-streaming-0.6b", + "aliases": ["nemotron-3.5", "nemotron-asr"], + "revision": "1c8deaecc64b91f034d73e08dd8b64625eb3395d", + "license": "NVIDIA Open Model License (OpenMDW 1.1)", + "license_url": "https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b", + "artifacts": [ + { + "role": "asr", + "type": "file", + "filename": "nemotron-3.5-asr-streaming-0.6b.q8_0.gguf", + "size": 741548352, + "sha256": "a5c435f294eea8f88ce68dd27b8c3bfea7f777cb2fbba04fcd30eaa555f429ae" + } + ] + }, + { + "repo": "nvidia/nemotron-speech-streaming-en-0.6b", + "aliases": ["nemotron-en"], + "revision": "ebe59e5a817142986528bbbee5dba8db7b38ed50", + "license": "NVIDIA Open Model License", + "license_url": "https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b", + "artifacts": [ + { + "role": "asr", + "type": "file", + "filename": "nemotron-speech-streaming-en-0.6b.q8_0.gguf", + "size": 699872960, + "sha256": "d9a01898d2a611c8764e23a1c2f45e70bbd5a425dc4de93692ac951dd603812d" + } + ] + }, + { + "repo": "nvidia/parakeet-ctc-1.1b", + "aliases": ["parakeet-ctc"], + "revision": "20e63a0fed6aedba145b74b826dbd41df0941730", + "license": "CC-BY-4.0", + "license_url": "https://huggingface.co/nvidia/parakeet-ctc-1.1b", + "artifacts": [ + { + "role": "asr", + "type": "file", + "filename": "parakeet-ctc-1.1b.q8_0.gguf", + "size": 1178100960, + "sha256": "6584fc0fdacf1c220401ea4c3a1d5b44454b655c141cb8672178072c203d92b8" + } + ] + }, + { + "repo": "nvidia/parakeet-tdt-0.6b-v3", + "aliases": ["parakeet-tdt"], + "revision": "541d1f99c6b0c3cd0b11a95167540bb8edefd82b", + "license": "CC-BY-4.0", + "license_url": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3", + "artifacts": [ + { + "role": "asr", + "type": "file", + "filename": "parakeet-tdt-0.6b-v3.q8_0.gguf", + "size": 713975456, + "sha256": "e3880d0aaaaf2c308ea2c35016b2b895c423eb3fda924c1b463d1c19b7f4d32e" + } + ] + }, + { + "repo": "nvidia/diar_streaming_sortformer_4spk-v2", + "aliases": ["sortformer", "sortformer-diar"], + "revision": "5240a64075176943f677d30fa2171c780229f341", + "license": "CC-BY-4.0", + "license_url": "https://huggingface.co/nvidia/diar_streaming_sortformer_4spk-v2", + "artifacts": [ + { + "role": "diarization", + "type": "file", + "filename": "diar_streaming_sortformer_4spk-v2.q8_0.gguf", + "size": 147075776, + "sha256": "0679cfeb1ce356d0dea9470b31274f4bfc7eb927497d82005483770666da998a" + } + ] + }, + { + "repo": "nvidia/magpie_tts_multilingual_357m", + "aliases": ["magpie", "magpie-tts"], + "revision": "452ef560f972c38d5fc16476259aac9456453547", + "license": "NVIDIA Open Model License", + "license_url": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "companions": [ + "nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps" + ], + "artifacts": [ + { + "role": "tts", + "type": "file", + "filename": "magpie_tts_multilingual_357m.v2602.f16.gguf", + "size": 448604832, + "sha256": "901d299a8b1df016cf81cae0089a7a7c15627b9633d033357e15a47d9a219a75" + }, + { + "role": "tokenizer", + "type": "tar-prefix", + "filename": "magpie_tts_multilingual_357m.nemo", + "directory": "tokenizer", + "size": 33554432, + "sha256": "2b930b399933dd384b17ec16a5f1ebf921e2017b760c4fd608c11077a68793fc", + "range_end": 33554431, + "stop_before": "model_weights.ckpt", + "members": [ + { + "name": "05adc40366e149b69319acd4b28a4919_pt_br_prondict-v1.0.dict", + "size": 3580284, + "sha256": "6492abc404db16bbad83dd9d7f9a60eb617699f0a3ff54b3be6184e14507b745" + }, + { + "name": "339da71c54b046f98cbcf38ef6d4ff67_hindi_phoneme_merged_phoneme_dict.dict", + "size": 7320312, + "sha256": "978a7aa5a0e3334b13c19015cc61f3ff1b96ce6ac7a31c482e5c22a61f19586c" + }, + { + "name": "41913ebaa70342058574da74293b7630_magpie_tts_multilingual_357m.nemo.speakers.json", + "size": 68, + "sha256": "36cdcf01ebc0afb506660ac71f2d0a211236374ad5c872d7d8d985a3f9f6ccaf" + }, + { + "name": "61d57a4ccb064d1e8e3d9f871c571932_heteronyms-052722", + "size": 1606, + "sha256": "b701909aedf753172eff223950f8859cd4b9b4c80199cf0a6e9ac4a307c8f8ec" + }, + { + "name": "74b3e832fe914a569fcd42b51d06b2f1_ipa_dict_nv23.05.txt", + "size": 5419, + "sha256": "252e8eaf60dfd891520759913dc8534393eebfa8d57192365f76f9229f294e9e" + }, + { + "name": "7dbc31751f224f2486090d59dc95b9f7_es_ES_nv230301.dict", + "size": 2230910, + "sha256": "94c9a25bb359f733cd863c887d0112e7ae19a744b0333ef64b6a88e576428669" + }, + { + "name": "c5e4ec2af5a14ce294f4b9edcc936535_de_nv230119.heteronym", + "size": 44566, + "sha256": "771cc585a574fd35bd14f4ce6108edf1e0e512a8dbc810db7de4c1165ba9d0ef" + }, + { + "name": "cf01ab5c48c84f3282ef7888263361e5_de_nv230119.dict", + "size": 4313907, + "sha256": "5c7bbf3346ebd6dc57769b5cc805124215cb1bcc3b8174c4254e9e928c5d6094" + }, + { + "name": "dc7d60d6b15a4651b21c9ca2932b62c6_ipa_cmudict-0.7b_nv23.01.txt", + "size": 3093097, + "sha256": "dd0927fffc89e8539ea0a26ccbc164a908f4b3de9613d924c90afa5300e00f72" + }, + { + "name": "model_config.yaml", + "size": 7496, + "sha256": "fda01948a40a04316b26553b53546cbd2103952b8d0e2813176ea33852bb96a2" + } + ] + } + ] + }, + { + "repo": "nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps", + "aliases": ["nano-codec", "nanocodec"], + "revision": "fc00890b604aa2de298d2641ffc6c5f6caf8c4d7", + "license": "NVIDIA Open Model License", + "license_url": "https://huggingface.co/nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps", + "artifacts": [ + { + "role": "codec", + "type": "file", + "filename": "nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf", + "size": 78823104, + "sha256": "cc86d36d821a27cdc1d4ef600a3e2b0dabe76e88fcc2a8652d9543134c07ef2d" + } + ] + } + ] +} diff --git a/scripts/configure.sh b/scripts/configure.sh index d2bfe75..9dff5a2 100755 --- a/scripts/configure.sh +++ b/scripts/configure.sh @@ -75,6 +75,7 @@ cmake_bool_override() { # cmake_bool_override VARIABLE DEFAULT ARGS... } need_nmt=OFF +need_asr=OFF need_grpc=OFF need_http=OFF need_flashlight=OFF @@ -83,6 +84,9 @@ need_zh=OFF case "$PRESET" in *-nmt|*-speech|*-server|cuda-full|developer) need_nmt=ON ;; esac +case "$PRESET" in + *-asr|*-speech|*-server|cuda-full|developer) need_asr=ON ;; +esac case "$PRESET" in cuda-full|developer) need_grpc=ON ;; esac @@ -97,6 +101,7 @@ fi need_nmt="$(cmake_bool_override NEMO_SPEECH_BUILD_NMT "$need_nmt" "$@")" need_nmt="$(cmake_bool_override NEMO_SPEECH_WITH_NMT "$need_nmt" "$@")" +need_asr="$(cmake_bool_override NEMO_SPEECH_BUILD_ASR "$need_asr" "$@")" need_grpc="$(cmake_bool_override NEMO_SPEECH_BUILD_GRPC "$need_grpc" "$@")" need_grpc="$(cmake_bool_override NEMO_SPEECH_WITH_GRPC "$need_grpc" "$@")" need_http="$(cmake_bool_override NEMO_SPEECH_BUILD_HTTP "$need_http" "$@")" @@ -128,6 +133,8 @@ if [ "$need_http" = ON ]; then fi if [ "$need_nmt" = ON ]; then require_submodule llama.cpp CMakeLists.txt +elif [ "$need_asr" = ON ]; then + require_submodule llama.cpp vendor/miniaudio/miniaudio.h fi if [ "$need_grpc" = ON ]; then require_submodule proto/riva-common LICENSE diff --git a/scripts/install.ps1 b/scripts/install.ps1 index a4b07c5..f995016 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -197,6 +197,9 @@ if (-not $Source -and $binaryCandidate) { Write-Host "Artifact: $url" } if (-not $BinaryOnly) { Write-Host "Source: $sourceUrl#$sourceRef ($Backend/$Profile)" } Write-Host "Prefix: $Prefix" if ($DryRun) { return } +if (-not (Get-Command curl.exe -ErrorAction SilentlyContinue)) { + Write-Warning "curl.exe was not found. Installation can continue, but automatic model downloads will be unavailable; local and already cached models still work. Windows 10 and 11 normally include curl.exe in %SystemRoot%\\System32." +} $installIdentity = "$releaseVersion windows $arch $Backend" $extraComponents = [Collections.Generic.List[string]]::new() diff --git a/scripts/install.sh b/scripts/install.sh index 6f89230..602f9aa 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -130,7 +130,11 @@ if [ "$backend" = auto ]; then fi artifact_backend=$backend -if [ "$os" = linux ] && [ "$arch" = aarch64 ] && [ "$backend" = cuda ]; then +if [ "$os" = macos ] && [ "$arch" = aarch64 ] && [ "$backend" = cpu ]; then + # The Apple Silicon Metal package also contains the general CPU backend. + # Keep one portable arm64 archive while allowing CPU-only execution. + artifact_backend=metal +elif [ "$os" = linux ] && [ "$arch" = aarch64 ] && [ "$backend" = cuda ]; then cuda_series=${NEMO_SPEECH_CUDA_SERIES:-} if [ -z "$cuda_series" ]; then case "$device_model" in diff --git a/scripts/windows/build.ps1 b/scripts/windows/build.ps1 index 5b81ed3..59ded9e 100644 --- a/scripts/windows/build.ps1 +++ b/scripts/windows/build.ps1 @@ -337,7 +337,11 @@ function Initialize-RequiredSubmodule { } Initialize-RequiredSubmodule 'ggml' 'CMakeLists.txt' -if ($BuildNmt) { Initialize-RequiredSubmodule 'llama.cpp' 'CMakeLists.txt' } +if ($BuildNmt) { + Initialize-RequiredSubmodule 'llama.cpp' 'CMakeLists.txt' +} elseif ($BuildAsr) { + Initialize-RequiredSubmodule 'llama.cpp' 'vendor\miniaudio\miniaudio.h' +} if ($BuildHttp) { Initialize-RequiredSubmodule 'third_party\cpp-httplib' 'httplib.h' } if ($BuildGrpc) { Initialize-RequiredSubmodule 'proto\riva-common' 'LICENSE' } if ($BuildFlashlight) { diff --git a/src/runtime/ggml/session.cpp b/src/runtime/ggml/session.cpp index 76e165c..f5b12dc 100644 --- a/src/runtime/ggml/session.cpp +++ b/src/runtime/ggml/session.cpp @@ -42,6 +42,24 @@ ggml_graph_compute_helper_async( return ggml_backend_sched_graph_compute_async(sched, graph) == GGML_STATUS_SUCCESS; } +// A scheduler split can cover a graph range containing nodes that are not +// active for the current run. Before bypassing the scheduler on a cache hit, +// verify the backend can execute every active node in the full graph. This is +// especially important for accelerator backends such as BLAS, which support +// large matrix multiplications but rely on the CPU backend for ops such as PAD. +static bool +backend_supports_compute_graph(ggml_backend_t backend, ggml_cgraph* graph) { + const int n_nodes = ggml_graph_n_nodes(graph); + for (int i = 0; i < n_nodes; ++i) { + const ggml_tensor* node = ggml_graph_node(graph, i); + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 && + !ggml_backend_supports_op(backend, node)) { + return false; + } + } + return true; +} + // Cache-aware encoder graphs grow with the true batch dimension. Keep enough // scheduler capacity for large batches while bounding its proportional hash, // backend-ID, and graph-copy allocations. Near-capacity graphs fail explicitly. @@ -822,8 +840,9 @@ Session::run_impl( } // Record direct-compute eligibility for subsequent hits: one split, - // one non-CPU backend (the CPU path routes thread-count setup through - // ggml_graph_compute_helper, so keep it on the scheduler). + // one non-CPU backend that supports every active node in the complete + // graph. The CPU path routes thread-count setup through + // ggml_graph_compute_helper, so keep it on the scheduler. cr.direct_ok = false; cr.direct_backend = nullptr; if (ggml_backend_sched_get_n_splits(sched.get()) == 1 && ggml_graph_n_nodes(cr.gf) > 0) { @@ -831,7 +850,8 @@ Session::run_impl( ggml_backend_sched_get_tensor_backend(sched.get(), ggml_graph_node(cr.gf, 0)); if (b != nullptr) { ggml_backend_dev_t dev = ggml_backend_get_device(b); - if (dev != nullptr && ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) { + if (dev != nullptr && ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU && + backend_supports_compute_graph(b, cr.gf)) { cr.direct_ok = true; cr.direct_backend = b; } diff --git a/tests/cli/cli_contract_test.py b/tests/cli/cli_contract_test.py index 1a619d0..fa4bb53 100644 --- a/tests/cli/cli_contract_test.py +++ b/tests/cli/cli_contract_test.py @@ -34,6 +34,7 @@ def expect_json_error(result: subprocess.CompletedProcess, exit_code: int, error def main() -> None: binary = sys.argv[1] + expect_live = "--expect-live" in sys.argv[2:] with tempfile.TemporaryDirectory(prefix="nemo-speech-cli-contract-") as temporary: help_result = run(binary, "--help") assert help_result.returncode == 0, help_result.stderr @@ -41,9 +42,9 @@ def main() -> None: model_help = run(binary, "model", "--help") assert model_help.returncode == 0, model_help.stderr - assert "model info FILE" in model_help.stdout - assert "model path" not in model_help.stdout - assert "pull" not in model_help.stdout + assert "info FILE" in model_help.stdout + assert "list" in model_help.stdout + assert "pull REPO" in model_help.stdout expect_json_error(run(binary, "--json", "not-a-command"), 2, "invalid_argument") expect_json_error( @@ -151,6 +152,7 @@ def stall_response() -> None: transcribe_help = run(binary, "transcribe", "--help") if transcribe_help.returncode == 0: assert "--backend" in transcribe_help.stdout + assert ("--live" in transcribe_help.stdout) == expect_live assert "session" not in transcribe_help.stderr lifecycle = run(binary, "transcribe") assert lifecycle.returncode == 2, lifecycle.stdout + lifecycle.stderr @@ -159,6 +161,17 @@ def stall_response() -> None: quiet_lifecycle = run(binary, "--quiet", "transcribe") assert "session" not in quiet_lifecycle.stderr expect_json_error(run(binary, "--json", "transcribe"), 2, "invalid_argument") + if expect_live: + expect_json_error( + run(binary, "--json", "transcribe", "--live", "recording.wav"), + 2, + "invalid_argument", + ) + expect_json_error( + run(binary, "--json", "transcribe", "--live", "--output-dir", temporary), + 2, + "invalid_argument", + ) synthesize_help = run(binary, "synthesize", "--help") if synthesize_help.returncode == 0: diff --git a/tests/cli/model_store_test.py b/tests/cli/model_store_test.py new file mode 100644 index 0000000..a36242a --- /dev/null +++ b/tests/cli/model_store_test.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import http.server +import io +import json +import os +import pathlib +import subprocess +import sys +import tarfile +import tempfile +import threading +import urllib.parse + +PAYLOAD = b"NeMo-Speech.cpp model-store fixture\n" +CODEC_PAYLOAD = b"NeMo-Speech.cpp codec fixture\n" +TTS_PAYLOAD = b"NeMo-Speech.cpp TTS fixture\n" +TOKENIZER_PAYLOAD = b"tokenizer configuration\n" + + +class ArtifactHandler(http.server.BaseHTTPRequestHandler): + requests = 0 + request_counts: dict[str, int] = {} + payloads: dict[str, bytes] = {} + + def do_GET(self) -> None: + type(self).requests += 1 + filename = pathlib.PurePosixPath(urllib.parse.urlsplit(self.path).path).name + type(self).request_counts[filename] = type(self).request_counts.get(filename, 0) + 1 + payload = type(self).payloads.get(filename) + if payload is None: + self.send_error(404) + return + start = 0 + end = len(payload) - 1 + range_header = self.headers.get("Range") + if range_header: + assert range_header.startswith("bytes=") + bounds = range_header.removeprefix("bytes=").split("-", 1) + start = int(bounds[0]) + if bounds[1]: + end = min(end, int(bounds[1])) + body = payload[start : end + 1] + self.send_response(206 if range_header else 200) + self.send_header("Content-Length", str(len(body))) + if range_header: + self.send_header("Content-Range", f"bytes {start}-{end}/{len(payload)}") + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args: object) -> None: + pass + + +def run(binary: str, environment: dict[str, str], *arguments: str) -> subprocess.CompletedProcess: + return subprocess.run( + [binary, *arguments], + env=environment, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def tokenizer_archive() -> bytes: + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w", format=tarfile.PAX_FORMAT) as bundle: + tokenizer = tarfile.TarInfo("tokenizer.txt") + tokenizer.size = len(TOKENIZER_PAYLOAD) + tokenizer.pax_headers = {"mtime": "0.0"} + bundle.addfile(tokenizer, io.BytesIO(TOKENIZER_PAYLOAD)) + weights = tarfile.TarInfo("model_weights.ckpt") + weights.size = 1 + bundle.addfile(weights, io.BytesIO(b"x")) + return output.getvalue() + + +def file_artifact(role: str, filename: str, payload: bytes, sha256: str | None = None) -> dict: + return { + "role": role, + "type": "file", + "filename": filename, + "size": len(payload), + "sha256": sha256 or hashlib.sha256(payload).hexdigest(), + } + + +def write_index(path: pathlib.Path, asr_sha256: str, tokenizer_tar: bytes) -> None: + path.write_text( + json.dumps( + { + "schema_version": 1, + "defaults": { + "asr": "acme/tiny-asr", + "tts": "acme/tiny-tts", + "tokenizer": "acme/tiny-tts", + "codec": "acme/tiny-codec", + }, + "models": [ + { + "repo": "acme/tiny-asr", + "aliases": ["tiny-asr"], + "revision": "0" * 40, + "license": "Test only", + "license_url": "https://example.invalid/license", + "artifacts": [file_artifact("asr", "tiny.gguf", PAYLOAD, asr_sha256)], + }, + { + "repo": "acme/tiny-tts", + "aliases": ["tiny-tts"], + "revision": "1" * 40, + "license": "Test only", + "license_url": "https://example.invalid/license", + "companions": ["acme/tiny-codec"], + "artifacts": [ + file_artifact("tts", "tiny-tts.gguf", TTS_PAYLOAD), + { + "role": "tokenizer", + "type": "tar-prefix", + "filename": "tiny-tts.nemo", + "directory": "tokenizer", + "size": len(tokenizer_tar), + "range_end": len(tokenizer_tar) - 1, + "sha256": hashlib.sha256(tokenizer_tar).hexdigest(), + "stop_before": "model_weights.ckpt", + "members": [ + { + "name": "tokenizer.txt", + "size": len(TOKENIZER_PAYLOAD), + "sha256": hashlib.sha256(TOKENIZER_PAYLOAD).hexdigest(), + } + ], + }, + ], + }, + { + "repo": "acme/tiny-codec", + "aliases": ["tiny-codec"], + "revision": "2" * 40, + "license": "Test only", + "license_url": "https://example.invalid/license", + "artifacts": [file_artifact("codec", "tiny-codec.gguf", CODEC_PAYLOAD)], + }, + ], + } + ), + encoding="utf-8", + ) + + +def expect_json_error(result: subprocess.CompletedProcess, code: int) -> dict: + assert result.returncode == code, result.stdout + result.stderr + assert result.stdout == "" + value = json.loads(result.stderr) + assert value["error"]["exit_code"] == code + return value["error"] + + +def main() -> None: + binary = sys.argv[1] + with tempfile.TemporaryDirectory(prefix="nemo-speech-model-store-") as temporary: + root = pathlib.Path(temporary) + index = root / "indéx.json" + cache = root / "caché" + tokenizer_tar = tokenizer_archive() + write_index(index, hashlib.sha256(PAYLOAD).hexdigest(), tokenizer_tar) + ArtifactHandler.requests = 0 + ArtifactHandler.request_counts = {} + ArtifactHandler.payloads = { + "tiny.gguf": PAYLOAD, + "tiny-tts.gguf": TTS_PAYLOAD, + "tiny-tts.nemo": tokenizer_tar, + "tiny-codec.gguf": CODEC_PAYLOAD, + } + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), ArtifactHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + environment = os.environ.copy() + environment.update( + { + "NEMO_SPEECH_MODEL_INDEX": str(index), + "NEMO_SPEECH_MODEL_DIR": str(cache), + "NEMO_SPEECH_HF_BASE_URL": f"http://127.0.0.1:{server.server_port}", + } + ) + try: + plain_listing = run(binary, environment, "model", "list") + assert plain_listing.returncode == 0, plain_listing.stderr + assert "ASR — transcribe, bench, serve" in plain_listing.stdout + assert "Diarization — diarize, transcribe --diarize, serve" in plain_listing.stdout + assert "TTS — synthesize, serve" in plain_listing.stdout + assert "* tiny-asr" in plain_listing.stdout + assert "repo: acme/tiny-asr" in plain_listing.stdout + + listed = run(binary, environment, "--json", "model", "list") + assert listed.returncode == 0, listed.stderr + listing = json.loads(listed.stdout) + assert listing["defaults"]["asr"] == "acme/tiny-asr" + assert listing["models"][0]["aliases"] == ["tiny-asr"] + assert listing["models"][0]["commands"] == ["transcribe", "bench", "serve"] + assert listing["models"][0]["default_for"] == ["asr"] + assert listing["models"][1]["commands"] == ["synthesize", "serve"] + assert listing["models"][1]["default_for"] == ["tokenizer", "tts"] + + pulled = run(binary, environment, "--json", "pull", "tiny-asr") + assert pulled.returncode == 0, pulled.stderr + artifact = json.loads(pulled.stdout)["artifacts"][0] + destination = pathlib.Path(artifact["path"]) + assert artifact["repo"] == "acme/tiny-asr" + assert artifact["cached"] is False + assert destination.read_bytes() == PAYLOAD + requests_after_pull = ArtifactHandler.requests + + cached = run(binary, environment, "--json", "model", "pull", "acme/tiny-asr") + assert cached.returncode == 0, cached.stderr + assert json.loads(cached.stdout)["artifacts"][0]["cached"] is True + assert ArtifactHandler.requests == requests_after_pull + + tts = run(binary, environment, "--json", "pull", "tiny-tts") + assert tts.returncode == 0, tts.stderr + tts_artifacts = json.loads(tts.stdout)["artifacts"] + assert [(item["repo"], item["role"]) for item in tts_artifacts] == [ + ("acme/tiny-tts", "tts"), + ("acme/tiny-tts", "tokenizer"), + ("acme/tiny-codec", "codec"), + ] + tokenizer = pathlib.Path(tts_artifacts[1]["path"]) + assert (tokenizer / "tokenizer.txt").read_bytes() == TOKENIZER_PAYLOAD + assert not (tokenizer / "model_weights.ckpt").exists() + assert ArtifactHandler.request_counts["tiny-tts.gguf"] == 1 + assert ArtifactHandler.request_counts["tiny-tts.nemo"] == 1 + assert ArtifactHandler.request_counts["tiny-codec.gguf"] == 1 + + cached_tts = run(binary, environment, "--json", "model", "pull", "acme/tiny-tts") + assert cached_tts.returncode == 0, cached_tts.stderr + assert all(item["cached"] for item in json.loads(cached_tts.stdout)["artifacts"]) + assert ArtifactHandler.request_counts["tiny-tts.gguf"] == 1 + assert ArtifactHandler.request_counts["tiny-tts.nemo"] == 1 + assert ArtifactHandler.request_counts["tiny-codec.gguf"] == 1 + + destination.write_bytes(b"x" * len(PAYLOAD)) + repaired = run(binary, environment, "--json", "pull", "tiny-asr") + assert repaired.returncode == 0, repaired.stderr + assert json.loads(repaired.stdout)["artifacts"][0]["cached"] is False + assert destination.read_bytes() == PAYLOAD + + unknown = expect_json_error( + run(binary, environment, "--json", "pull", "unknown/repository"), 3 + ) + assert unknown["type"] == "missing_model" + + write_index(index, "0" * 64, tokenizer_tar) + bad_cache = root / "bad-cache" + bad_environment = {**environment, "NEMO_SPEECH_MODEL_DIR": str(bad_cache)} + invalid = expect_json_error( + run(binary, bad_environment, "--json", "pull", "tiny-asr"), 1 + ) + assert "SHA-256 verification" in invalid["message"] + assert not list(bad_cache.rglob("tiny.gguf")) + + missing_environment = { + **environment, + "PATH": str(root / "empty-path"), + "NEMO_SPEECH_MODEL_DIR": str(root / "missing-curl-cache"), + } + missing = expect_json_error( + run(binary, missing_environment, "--json", "pull", "tiny-asr"), 1 + ) + assert "curl executable" in missing["message"] + assert "Local model paths" in missing["message"] + + unsafe_environment = { + **environment, + "NEMO_SPEECH_HF_BASE_URL": "http://localhost.example.invalid", + "NEMO_SPEECH_MODEL_DIR": str(root / "unsafe-url-cache"), + } + unsafe = expect_json_error( + run(binary, unsafe_environment, "--json", "pull", "tiny-asr"), 1 + ) + assert "must use HTTPS" in unsafe["message"] + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 57bdcc3..9ff5030 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -33,10 +33,24 @@ endif() if(TARGET nemo_speech_cli) if(Python3_Interpreter_FOUND) + set(cli_contract_args + ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/cli/cli_contract_test.py + $) + if(NEMO_SPEECH_BUILD_ASR AND NEMO_SPEECH_BUILD_MIC_CAPTURE) + list(APPEND cli_contract_args --expect-live) + endif() add_test( NAME cli_contract - COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/cli/cli_contract_test.py - $) + COMMAND ${cli_contract_args}) + find_program(NEMO_SPEECH_CURL_EXECUTABLE NAMES curl curl.exe) + if(NEMO_SPEECH_CURL_EXECUTABLE) + add_test( + NAME cli_model_store + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/cli/model_store_test.py + $) + else() + message(WARNING "curl not found; skipping cli_model_store") + endif() if(UNIX AND NOT APPLE) add_test( NAME cli_install_linux diff --git a/tests/install/install_sh_test.py b/tests/install/install_sh_test.py index 07bc141..91c7581 100644 --- a/tests/install/install_sh_test.py +++ b/tests/install/install_sh_test.py @@ -22,7 +22,10 @@ def require(condition: bool, message: str) -> None: def archive(version: str, arch: str) -> tuple[str, bytes, str]: os_name = "macos" if platform.system() == "Darwin" else "linux" - name = f"nemo-speech-{version}-{os_name}-{arch}-cpu.tar.gz" + # Apple Silicon uses one Metal package containing both Metal and CPU + # backends; requesting --backend cpu selects this same archive. + backend = "metal" if os_name == "macos" and arch == "aarch64" else "cpu" + name = f"nemo-speech-{version}-{os_name}-{arch}-{backend}.tar.gz" output = io.BytesIO() with tarfile.open(fileobj=output, mode="w:gz") as bundle: files = { @@ -99,6 +102,7 @@ def main() -> None: installer = source_root / "scripts" / "install.sh" os_name = "macos" if platform.system() == "Darwin" else "linux" arch = "aarch64" if platform.machine().lower() in {"aarch64", "arm64"} else "x86_64" + binary_backend = "metal" if os_name == "macos" and arch == "aarch64" else "cpu" releases = {} for version in ("1.2.3", "1.2.4", "1.2.5", "nightly"): name, contents, checksum = archive(version, arch) @@ -285,7 +289,7 @@ def run(*arguments: str, ok: bool = True) -> subprocess.CompletedProcess[str]: ) require( (source_prefix / ".nemo-speech-install").read_text().strip() - == f"1.2.6 {os_name} {arch} cpu", + == f"1.2.6 {os_name} {arch} {binary_backend}", "published binary did not replace the source installation", ) require( diff --git a/third_party/miniaudio/LICENSE b/third_party/miniaudio/LICENSE new file mode 100644 index 0000000..d88e4f3 --- /dev/null +++ b/third_party/miniaudio/LICENSE @@ -0,0 +1,18 @@ +MIT No Attribution + +Copyright 2026 David Reid + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 68719b80293ac2160a87dec66a6b43391bde7953 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 00:02:43 -0700 Subject: [PATCH 05/11] fix(windows): binary installs and model downloads --- README.md | 8 ++++++++ app/model_store.cpp | 2 +- scripts/install.ps1 | 5 +++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 550315a..b231ad2 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,19 @@ NeMo-Speech.cpp is NVIDIA's official local speech inference solution, with day-0 Install the `nemo-speech` CLI for the detected platform and backend: +On Linux or macOS, run: + ```bash curl -fsSL https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.sh | sh export PATH="$HOME/.local/bin:$PATH" # current shell; future shells are updated ``` +On Windows, run from PowerShell: + +```powershell +irm https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.ps1 | iex +``` + The installer prefers a verified native release and falls back to a source build when an artifact is unavailable. A source build requires Git, CMake 3.26 or newer, Ninja, a C++17 compiler, and the selected GPU toolkit. See diff --git a/app/model_store.cpp b/app/model_store.cpp index 3b1a2dc..66d1c2c 100644 --- a/app/model_store.cpp +++ b/app/model_store.cpp @@ -192,7 +192,7 @@ sha256_file(const fs::path& path) { if (!input) throw std::runtime_error("cannot read downloaded artifact " + path_utf8(path)); Sha256 digest; - std::array buffer{}; + std::vector buffer(1024 * 1024); while (input) { input.read(reinterpret_cast(buffer.data()), buffer.size()); const auto count = input.gcount(); diff --git a/scripts/install.ps1 b/scripts/install.ps1 index f995016..6e39455 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -23,6 +23,7 @@ param( [switch]$DryRun ) $ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" $releaseBase = if ($env:NEMO_SPEECH_RELEASE_BASE_URL) { $env:NEMO_SPEECH_RELEASE_BASE_URL.TrimEnd('/') } else { @@ -52,7 +53,7 @@ function Invoke-DownloadWithRetry { for ($attempt = 1; $attempt -le 3; $attempt++) { try { - Invoke-WebRequest -Uri $Uri -OutFile $OutFile + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $OutFile return } catch { if ($attempt -eq 3) { throw } @@ -162,7 +163,7 @@ if ($Version -eq "latest") { Write-Host "No release endpoint is configured; building from the current source branch." } else { try { - $manifest = (Invoke-WebRequest -Uri $versionUrl).Content + $manifest = (Invoke-WebRequest -UseBasicParsing -Uri $versionUrl).Content if ($manifest -notmatch '(?m)^NEMO_SPEECH_VERSION:\s*([^\s]+)\s*$') { throw "VERSION does not contain NEMO_SPEECH_VERSION" } From ac8d2fa38aac9f8c588e1bd13ef0df929894a1ac Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 12:53:27 +0530 Subject: [PATCH 06/11] install(mac): support cpu-only apple silicon archives --- docs/install.md | 4 +-- scripts/install.sh | 6 +---- tests/install/install_sh_test.py | 45 +++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/docs/install.md b/docs/install.md index fd3ba17..17711fa 100644 --- a/docs/install.md +++ b/docs/install.md @@ -126,8 +126,8 @@ nemo-speech--windows--.zip ``` Linux aarch64 CUDA archives use `cuda12` or `cuda13` as the backend suffix. -On Apple Silicon, both `--backend metal` and `--backend cpu` install the -`macos-aarch64-metal` archive because it contains both runtime backends. +On Apple Silicon, automatic selection installs `macos-aarch64-metal`; pass +`--backend cpu` to install the smaller `macos-aarch64-cpu` archive. To uninstall on Linux or macOS, remove the prefix printed during installation and `~/.local/bin/nemo-speech`; remove the two-line NeMo-Speech.cpp PATH diff --git a/scripts/install.sh b/scripts/install.sh index 602f9aa..6f89230 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -130,11 +130,7 @@ if [ "$backend" = auto ]; then fi artifact_backend=$backend -if [ "$os" = macos ] && [ "$arch" = aarch64 ] && [ "$backend" = cpu ]; then - # The Apple Silicon Metal package also contains the general CPU backend. - # Keep one portable arm64 archive while allowing CPU-only execution. - artifact_backend=metal -elif [ "$os" = linux ] && [ "$arch" = aarch64 ] && [ "$backend" = cuda ]; then +if [ "$os" = linux ] && [ "$arch" = aarch64 ] && [ "$backend" = cuda ]; then cuda_series=${NEMO_SPEECH_CUDA_SERIES:-} if [ -z "$cuda_series" ]; then case "$device_model" in diff --git a/tests/install/install_sh_test.py b/tests/install/install_sh_test.py index 91c7581..bf34012 100644 --- a/tests/install/install_sh_test.py +++ b/tests/install/install_sh_test.py @@ -22,9 +22,7 @@ def require(condition: bool, message: str) -> None: def archive(version: str, arch: str) -> tuple[str, bytes, str]: os_name = "macos" if platform.system() == "Darwin" else "linux" - # Apple Silicon uses one Metal package containing both Metal and CPU - # backends; requesting --backend cpu selects this same archive. - backend = "metal" if os_name == "macos" and arch == "aarch64" else "cpu" + backend = "cpu" name = f"nemo-speech-{version}-{os_name}-{arch}-{backend}.tar.gz" output = io.BytesIO() with tarfile.open(fileobj=output, mode="w:gz") as bundle: @@ -102,7 +100,7 @@ def main() -> None: installer = source_root / "scripts" / "install.sh" os_name = "macos" if platform.system() == "Darwin" else "linux" arch = "aarch64" if platform.machine().lower() in {"aarch64", "arm64"} else "x86_64" - binary_backend = "metal" if os_name == "macos" and arch == "aarch64" else "cpu" + binary_backend = "cpu" releases = {} for version in ("1.2.3", "1.2.4", "1.2.5", "nightly"): name, contents, checksum = archive(version, arch) @@ -330,6 +328,45 @@ def run(*arguments: str, ok: bool = True) -> subprocess.CompletedProcess[str]: fake_uname.write_text( """#!/bin/sh case "$1" in + -s) echo Darwin ;; + -m) echo arm64 ;; + *) exit 2 ;; +esac +""", + encoding="utf-8", + ) + fake_uname.chmod(0o755) + mac_env = env.copy() + mac_env["PATH"] = f"{fake_bin}:{mac_env['PATH']}" + for backend, artifact_backend in (("cpu", "cpu"), ("auto", "metal")): + result = subprocess.run( + [ + "sh", + str(installer), + "--prefix", + str(root / f"macos-{backend}-dry-run"), + "--version", + "9.9.9", + "--backend", + backend, + "--binary-only", + "--dry-run", + ], + env=mac_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + require(result.returncode == 0, f"macOS selector failed:\n{result.stdout}") + require( + f"macos-aarch64-{artifact_backend}.tar.gz" in result.stdout, + f"macOS {backend} artifact was not selected", + ) + + fake_uname.write_text( + """#!/bin/sh +case "$1" in -s) echo Linux ;; -m) echo aarch64 ;; *) exit 2 ;; From 047f97bb008bfb5cdb73288c17958cec1edf8961 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 09:53:04 +0000 Subject: [PATCH 07/11] fix(diar): improve word attribution and suppress phantom speakers - gate transient speaker channels and revise recent diarization frames - align word attribution to speaker onsets - preserve CTC confidence and valid JSON output --- app/transcribe.cpp | 12 ++- src/asr/decoders/greedy_ctc_decoder.cpp | 37 ++++---- src/asr/decoders/greedy_ctc_decoder.h | 4 + src/asr/diar/aosc_state.cpp | 113 ++++++++++++++++++++++++ src/asr/diar/aosc_state.h | 26 ++++++ src/asr/diar/diar_pipeline.cpp | 12 ++- src/asr/diar/diar_pipeline.h | 6 +- src/asr/recognizer.cpp | 10 ++- tests/cpp/asr/CMakeLists.txt | 4 + tests/cpp/asr/test_diar_state.cpp | 79 +++++++++++++++++ 10 files changed, 277 insertions(+), 26 deletions(-) create mode 100644 tests/cpp/asr/test_diar_state.cpp diff --git a/app/transcribe.cpp b/app/transcribe.cpp index a5fab87..c219a2a 100644 --- a/app/transcribe.cpp +++ b/app/transcribe.cpp @@ -3,6 +3,7 @@ #include #include +#include #if defined(NEMO_SPEECH_CLI_LIVE) #include #include @@ -408,6 +409,11 @@ timestamp(int milliseconds, bool vtt) { return output.str(); } +float +json_number(float value) { + return std::isfinite(value) ? value : 0.0f; +} + std::string render(const Transcript& t, OutputFormat format, const fs::path& source) { std::ostringstream output; @@ -416,8 +422,8 @@ render(const Transcript& t, OutputFormat format, const fs::path& source) { } else if (format == OutputFormat::Json) { output << "{\n \"file\": \"" << json_escape(source.string()) << "\",\n" << " \"text\": \"" << json_escape(t.text) << "\",\n" - << " \"confidence\": " << t.confidence << ",\n" - << " \"duration\": " << t.audio_seconds << ",\n \"languages\": ["; + << " \"confidence\": " << json_number(t.confidence) << ",\n" + << " \"duration\": " << json_number(t.audio_seconds) << ",\n \"languages\": ["; for (size_t i = 0; i < t.languages.size(); ++i) output << (i ? ", " : "") << '"' << json_escape(t.languages[i]) << '"'; output << "],"; @@ -429,7 +435,7 @@ render(const Transcript& t, OutputFormat format, const fs::path& source) { const auto& w = t.words[i]; output << (i ? "," : "") << "\n {\"word\": \"" << json_escape(w.text) << "\", \"start\": " << w.start_ms / 1000.0 << ", \"end\": " << w.end_ms / 1000.0 - << ", \"confidence\": " << w.confidence; + << ", \"confidence\": " << json_number(w.confidence); if (w.speaker > 0) output << ", \"speaker\": " << w.speaker; output << '}'; diff --git a/src/asr/decoders/greedy_ctc_decoder.cpp b/src/asr/decoders/greedy_ctc_decoder.cpp index ae3b09f..8973865 100644 --- a/src/asr/decoders/greedy_ctc_decoder.cpp +++ b/src/asr/decoders/greedy_ctc_decoder.cpp @@ -32,8 +32,8 @@ CtcHeadModule::define_tensors(ggml_runtime::Session* session) { session->model_tensor_container->create_tensor_2d(argmax_eye_name_, GGML_TYPE_F32, C, C); } -ggml_runtime::TensorBag -CtcHeadModule::build_graph( +ggml_runtime::ggml_bf_tensor +CtcHeadModule::build_probs( ggml_runtime::Session* session, ggml_runtime::TensorBag input_tensors, ggml_runtime::TensorContainer* tc) { // Input: encoder features (d_model=C, T, 1, 1). @@ -49,11 +49,18 @@ CtcHeadModule::build_graph( // (T, num_classes+1, 1, 1) -> (num_classes+1, T, 1, 1) auto logits_t = ggml_cont(bf_ctx.ctx, ggml_permute(bf_ctx.ctx, logits.tensor, 1, 0, 2, 3)); - auto sm = ggml_soft_max(bf_ctx.ctx, logits_t); - auto log_sm = ggml_log(bf_ctx.ctx, sm); + return ggml_runtime::ggml_bf_tensor(ggml_soft_max(bf_ctx.ctx, logits_t), x.buft); +} + +ggml_runtime::TensorBag +CtcHeadModule::build_graph( + ggml_runtime::Session* session, ggml_runtime::TensorBag input_tensors, + ggml_runtime::TensorContainer* tc) { + auto probs = build_probs(session, input_tensors, tc); + auto bf_ctx = tc->get_ctx_of_buffer_type(probs.buft); ggml_runtime::TensorBag out; - out.add_tensor(ggml_runtime::ggml_bf_tensor(log_sm, x.buft)); + out.add_tensor(ggml_runtime::ggml_bf_tensor(ggml_log(bf_ctx.ctx, probs.tensor), probs.buft)); return out; } @@ -61,27 +68,25 @@ ggml_runtime::TensorBag CtcHeadModule::build_greedy_graph( ggml_runtime::Session* session, ggml_runtime::TensorBag input_tensors, ggml_runtime::TensorContainer* tc) { - auto full = build_graph(session, input_tensors, tc); - auto log_probs = full.get_tensor(0); - auto bf_ctx = tc->get_ctx_of_buffer_type(log_probs.buft); + auto probs = build_probs(session, input_tensors, tc); + auto bf_ctx = tc->get_ctx_of_buffer_type(probs.buft); auto eye = session->model_tensor_container->get_tensor_by_name(argmax_eye_name_); - const int64_t C = log_probs.tensor->ne[0]; - const int64_t T = log_probs.tensor->ne[1]; - const int64_t B = log_probs.tensor->ne[2]; + const int64_t C = probs.tensor->ne[0]; + const int64_t T = probs.tensor->ne[1]; + const int64_t B = probs.tensor->ne[2]; // Argmax reduces each column independently, so flatten the outer axes and // avoid backend-specific handling of integer concat and strided views. - auto flat = ggml_reshape_2d(bf_ctx.ctx, ggml_cont(bf_ctx.ctx, log_probs.tensor), C, T * B); + auto flat = ggml_reshape_2d(bf_ctx.ctx, ggml_cont(bf_ctx.ctx, probs.tensor), C, T * B); auto ids = ggml_argmax(bf_ctx.ctx, flat); auto onehot = ggml_get_rows(bf_ctx.ctx, eye.tensor, ids); - auto winning_prob = - ggml_exp(bf_ctx.ctx, ggml_sum_rows(bf_ctx.ctx, ggml_mul(bf_ctx.ctx, flat, onehot))); + auto winning_prob = ggml_sum_rows(bf_ctx.ctx, ggml_mul(bf_ctx.ctx, flat, onehot)); ids = ggml_reshape_2d(bf_ctx.ctx, ids, T, B); winning_prob = ggml_reshape_2d(bf_ctx.ctx, winning_prob, T, B); ggml_runtime::TensorBag out; - out.add_tensor(ggml_runtime::ggml_bf_tensor(ids, log_probs.buft)); - out.add_tensor(ggml_runtime::ggml_bf_tensor(winning_prob, log_probs.buft)); + out.add_tensor(ggml_runtime::ggml_bf_tensor(ids, probs.buft)); + out.add_tensor(ggml_runtime::ggml_bf_tensor(winning_prob, probs.buft)); return out; } diff --git a/src/asr/decoders/greedy_ctc_decoder.h b/src/asr/decoders/greedy_ctc_decoder.h index 3da5cb6..51f95f5 100644 --- a/src/asr/decoders/greedy_ctc_decoder.h +++ b/src/asr/decoders/greedy_ctc_decoder.h @@ -45,6 +45,10 @@ class CtcHeadModule : public ggml_runtime::Module { void set_data(ggml_runtime::Session* session) override; private: + ggml_runtime::ggml_bf_tensor build_probs( + ggml_runtime::Session* session, ggml_runtime::TensorBag input_tensors, + ggml_runtime::TensorContainer* tc); + std::string name_; CtcConfig cfg_; ggml_runtime::Conv1D* proj_; // decoder.decoder_layers.0 diff --git a/src/asr/diar/aosc_state.cpp b/src/asr/diar/aosc_state.cpp index 2de3d56..c12e0db 100644 --- a/src/asr/diar/aosc_state.cpp +++ b/src/asr/diar/aosc_state.cpp @@ -17,6 +17,16 @@ constexpr float kPosInf = std::numeric_limits::infinity(); // NeMo's placeholder for disabled top-k picks (sortformer_modules.max_index). constexpr int64_t kMaxIndex = 99999; +constexpr float kBirthSpeech = 0.30f; +constexpr float kBirthClean = 0.95f; +constexpr float kEstablishedQuiet = 0.02f; +constexpr float kBirthFading = 0.90f; +constexpr float kEstablishedFading = 0.15f; +constexpr int kBirthCleanFrames = 4; +constexpr int kBirthFadingFrames = 20; +constexpr int kBirthEpisodeGapFrames = 25; +constexpr int kBirthRevisionFrames = 128; + // Indices of the k largest values in column `spk` of `scores` (n x n_spk). // Ties break toward the lower frame index (deterministic; torch's order for // exact ties is unspecified, and exact float ties among finite scores are @@ -38,6 +48,109 @@ topk_column(const std::vector& scores, int n, int n_spk, int spk, int k) } } // namespace +ChannelBirthGate::ChannelBirthGate(int n_spk) : n_spk_(n_spk) { + if (n_spk_ <= 0) + throw std::invalid_argument("ChannelBirthGate: n_spk must be positive"); + reset(); +} + +void +ChannelBirthGate::reset() { + frame_ = 0; + established_.assign(n_spk_, false); + clean_frames_.assign(n_spk_, 0); + fading_frames_.assign(n_spk_, 0); + last_win_.assign(n_spk_, std::numeric_limits::min() / 2); + raw_ring_.clear(); +} + +bool +ChannelBirthGate::observe(const float* probs) { + int winner = 0; + for (int s = 1; s < n_spk_; s++) + if (probs[s] > probs[winner]) + winner = s; + + bool changed = false; + if (probs[winner] >= kBirthSpeech && !established_[winner]) { + if (frame_ - last_win_[winner] > kBirthEpisodeGapFrames) { + clean_frames_[winner] = 0; + fading_frames_[winner] = 0; + } + last_win_[winner] = frame_; + + float established_prob = 0.0f; + for (int s = 0; s < n_spk_; s++) + if (established_[s]) + established_prob = std::max(established_prob, probs[s]); + if (probs[winner] >= kBirthClean && established_prob <= kEstablishedQuiet) + clean_frames_[winner]++; + if (probs[winner] >= kBirthFading && established_prob <= kEstablishedFading) + fading_frames_[winner]++; + if (clean_frames_[winner] >= kBirthCleanFrames || + fading_frames_[winner] >= kBirthFadingFrames) { + established_[winner] = true; + changed = true; + } + } + frame_++; + return changed; +} + +void +ChannelBirthGate::relabel(float* probs) const { + int target = -1; + for (int s = 0; s < n_spk_; s++) + if (established_[s] && (target < 0 || probs[s] > probs[target])) + target = s; + if (target < 0) + return; + + for (int s = 0; s < n_spk_; s++) { + if (!established_[s] && probs[s] > 0.0f) { + probs[target] = std::max(probs[target], probs[s]); + probs[s] = 0.0f; + } + } +} + +void +ChannelBirthGate::push_raw(const float* probs) { + raw_ring_.insert(raw_ring_.end(), probs, probs + n_spk_); + const size_t cap = static_cast(kBirthRevisionFrames) * n_spk_; + if (raw_ring_.size() > cap) + raw_ring_.erase(raw_ring_.begin(), raw_ring_.begin() + n_spk_); +} + +void +ChannelBirthGate::append(const std::vector& raw, std::vector& timeline) { + if (raw.size() % n_spk_ != 0) + throw std::invalid_argument("ChannelBirthGate: incomplete probability frame"); + + bool changed = false; + for (size_t i = 0; i < raw.size(); i += n_spk_) { + push_raw(raw.data() + i); + changed |= observe(raw.data() + i); + } + + const size_t old_size = timeline.size(); + timeline.insert(timeline.end(), raw.begin(), raw.end()); + for (size_t i = old_size; i < timeline.size(); i += n_spk_) relabel(timeline.data() + i); + + if (!changed) + return; + const size_t window = std::min(raw_ring_.size(), timeline.size()); + const size_t ring_offset = raw_ring_.size() - window; + const size_t timeline_offset = timeline.size() - window; + std::copy(raw_ring_.begin() + ring_offset, raw_ring_.end(), timeline.begin() + timeline_offset); + for (size_t i = timeline_offset; i < timeline.size(); i += n_spk_) relabel(timeline.data() + i); +} + +bool +ChannelBirthGate::is_established(int speaker) const { + return speaker >= 0 && speaker < n_spk_ && established_[speaker]; +} + DiarGeometry DiarGeometry::preset(const std::string& name) { if (name == "streaming") diff --git a/src/asr/diar/aosc_state.h b/src/asr/diar/aosc_state.h index 4fd10f3..a7db980 100644 --- a/src/asr/diar/aosc_state.h +++ b/src/asr/diar/aosc_state.h @@ -11,6 +11,7 @@ // the first compression - NeMo's `spkcache_preds is None` sentinel). #pragma once +#include #include #include @@ -39,6 +40,31 @@ struct DiarGeometry { void validate(int n_spk, int sil_frames_per_spk, int pos_emb_max_len) const; }; +// Keeps transient channel redraws from becoming new speaker identities. A +// channel is established only after a short clean or fading handoff; until +// then its probability is folded into the strongest established channel. +class ChannelBirthGate { + public: + explicit ChannelBirthGate(int n_spk); + + void reset(); + void append(const std::vector& raw, std::vector& timeline); + bool is_established(int speaker) const; + + private: + bool observe(const float* probs); + void relabel(float* probs) const; + void push_raw(const float* probs); + + int n_spk_; + int64_t frame_ = 0; + std::vector established_; + std::vector clean_frames_; + std::vector fading_frames_; + std::vector last_win_; + std::vector raw_ring_; +}; + class AoscState { public: AoscState(const DiarGeometry& geo, const DiarScoringConfig& scoring, int n_spk, int emb_dim); diff --git a/src/asr/diar/diar_pipeline.cpp b/src/asr/diar/diar_pipeline.cpp index 7a7d6b3..50eed05 100644 --- a/src/asr/diar/diar_pipeline.cpp +++ b/src/asr/diar/diar_pipeline.cpp @@ -73,7 +73,7 @@ DiarStream::DiarStream(DiarModel& model, const DiarGeometry& geometry) sub_(model.cfg().encoder.subsampling_factor), sec_per_frame_( model.cfg().encoder.subsampling_factor * static_cast(model.cfg().window_stride)), - state_(geo_, model.cfg().scoring, n_spk_, model.cfg().encoder.d_model) { + state_(geo_, model.cfg().scoring, n_spk_, model.cfg().encoder.d_model), birth_gate_(n_spk_) { n_mels_ = m_.fe().n_mels(); geo_.validate( n_spk_, model.cfg().scoring.sil_frames_per_spk, model.cfg().encoder.pos_emb_max_len); @@ -82,6 +82,7 @@ DiarStream::DiarStream(DiarModel& model, const DiarGeometry& geometry) void DiarStream::reset() { state_ = AoscState(geo_, m_.cfg().scoring, n_spk_, m_.cfg().encoder.d_model); + birth_gate_.reset(); audio_buf_.clear(); audio_base_ = 0; mel_buf_.clear(); @@ -183,7 +184,7 @@ DiarStream::run_one_chunk(bool force, bool final_flush) { const int rc_enc = static_cast(std::ceil(rc_mel / static_cast(sub_))); auto emitted = state_.update(out.chunk_embs.data(), out.chunk_frames, out.preds.data(), lc_enc, rc_enc); - probs_.insert(probs_.end(), emitted.begin(), emitted.end()); + birth_gate_.append(emitted, probs_); maybe_compact(); mel_consumed_ = end; @@ -246,6 +247,13 @@ DiarStream::speaker_for_time(double t0, double t1) const { return speaker_for_frames(f0, f1); } +int +DiarStream::speaker_for_word_time(double t0, double t1) const { + const int64_t f0 = static_cast(t0 / sec_per_frame_); + const int64_t f1 = static_cast(std::ceil(t1 / sec_per_frame_)); + return speaker_for_frames(f0, std::min(f1, f0 + 2)); +} + std::vector nemo_speech::asr::diar_segments_from_probs( const float* probs, int64_t n_frames, int n_spk, double sec_per_frame, diff --git a/src/asr/diar/diar_pipeline.h b/src/asr/diar/diar_pipeline.h index 38a733c..cfb3856 100644 --- a/src/asr/diar/diar_pipeline.h +++ b/src/asr/diar/diar_pipeline.h @@ -162,8 +162,11 @@ class DiarStream { // emitted frame (riva's extrapolation for words past the diarized // frontier). Returns -1 when nothing has been emitted yet. int speaker_for_frames(int64_t start_frame, int64_t end_frame) const; - // Same, for a time range in seconds (used for word timestamps). + // Same, for a time range in seconds. int speaker_for_time(double t0, double t1) const; + // Word attribution is onset-anchored because late punctuation emissions + // can extend transducer word spans into the next speaker's turn. + int speaker_for_word_time(double t0, double t1) const; using Segment = DiarSegment; std::vector segments(const DiarSegmentationCfg& cfg = DiarSegmentationCfg()) const; @@ -184,6 +187,7 @@ class DiarStream { double sec_per_frame_; AoscState state_; + ChannelBirthGate birth_gate_; std::vector audio_buf_; size_t audio_base_ = 0; // global sample index of audio_buf_[0] std::vector mel_buf_; diff --git a/src/asr/recognizer.cpp b/src/asr/recognizer.cpp index 253272b..cfdaa9b 100644 --- a/src/asr/recognizer.cpp +++ b/src/asr/recognizer.cpp @@ -422,9 +422,8 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const // Build one Alternative from a decoder hypothesis. Postproc (PnC/ITN/ // profanity) may remap word spans, so it runs before frame->ms conversion; - // word offsets are produced only when requested. Speaker tags: riva - // semantics - mean diarizer frame probability over the word's span, - // argmax, 1-based; only the top alternative is tagged. + // word offsets are produced only when requested. Speaker tags are 1-based; + // only the top alternative is tagged. auto make_alt = [&](const std::string& transcript, float confidence, const std::vector& words_in, bool tag_speakers) { Alternative alt; @@ -446,8 +445,11 @@ RecognitionStream::build_result_(const StreamingUpdate& u, bool is_final) const ww.confidence = w.confidence; ww.language_code = lang0; if (tag_speakers && diar_) { + // Transducer punctuation can extend a word timestamp into + // the next turn. Anchor attribution to the word onset and + // average two diar frames to reject single-frame noise. const int spk = - diar_->speaker_for_time(ww.start_time / 1000.0, ww.end_time / 1000.0); + diar_->speaker_for_word_time(ww.start_time / 1000.0, ww.end_time / 1000.0); ww.speaker_tag = spk >= 0 ? spk + 1 : 0; } alt.words.push_back(std::move(ww)); diff --git a/tests/cpp/asr/CMakeLists.txt b/tests/cpp/asr/CMakeLists.txt index e47508b..f775025 100644 --- a/tests/cpp/asr/CMakeLists.txt +++ b/tests/cpp/asr/CMakeLists.txt @@ -48,6 +48,10 @@ target_link_libraries(test_sortformer_parity PRIVATE nemo_speech_asr) add_executable(test_diar_streaming test_diar_streaming.cpp) target_link_libraries(test_diar_streaming PRIVATE nemo_speech_asr) +add_executable(test_diar_state test_diar_state.cpp) +target_link_libraries(test_diar_state PRIVATE nemo_speech_asr) +add_test(NAME diar_state COMMAND test_diar_state) + add_executable(test_diar_recognizer test_diar_recognizer.cpp) target_link_libraries(test_diar_recognizer PRIVATE nemo_speech_asr) diff --git a/tests/cpp/asr/test_diar_state.cpp b/tests/cpp/asr/test_diar_state.cpp new file mode 100644 index 0000000..fb8d8fa --- /dev/null +++ b/tests/cpp/asr/test_diar_state.cpp @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#include +#include +#include + +#include "aosc_state.h" + +using namespace nemo_speech::asr; + +namespace { + +bool +near(float a, float b) { + return std::fabs(a - b) < 1e-6f; +} + +bool +test_channel_birth_gate() { + ChannelBirthGate gate(4); + std::vector timeline; + + gate.append( + {0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, + 0.01f, 0.01f, 0.01f}, + timeline); + if (!gate.is_established(0)) + return false; + + std::vector redraw; + for (int i = 0; i < 20; i++) redraw.insert(redraw.end(), {0.4f, 0.01f, 0.01f, 0.8f}); + const size_t redraw_offset = timeline.size(); + gate.append(redraw, timeline); + if (gate.is_established(3)) + return false; + for (size_t i = redraw_offset; i < timeline.size(); i += 4) + if (!near(timeline[i], 0.8f) || !near(timeline[i + 3], 0.0f)) + return false; + + gate.append({0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f, 0.01f}, timeline); + const size_t handoff_offset = timeline.size() - 8; + gate.append({0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f, 0.01f}, timeline); + if (!gate.is_established(1)) + return false; + for (size_t i = handoff_offset; i < timeline.size(); i += 4) + if (!near(timeline[i + 1], 0.99f)) + return false; + + const size_t revision_offset = timeline.size(); + gate.append({0.01f, 0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f}, timeline); + if (gate.is_established(2)) + return false; + gate.append({0.01f, 0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f}, timeline); + if (!gate.is_established(2)) + return false; + for (size_t i = revision_offset; i < timeline.size(); i += 4) + if (!near(timeline[i + 2], 0.99f)) + return false; + + gate.append( + {0.01f, 0.01f, 0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f, 0.01f, 0.01f, 0.99f, 0.01f, + 0.01f, 0.01f, 0.99f}, + timeline); + if (!gate.is_established(3)) + return false; + return true; +} + +} // namespace + +int +main() { + if (!test_channel_birth_gate()) { + std::fprintf(stderr, "[FAIL] transient speaker channel was established\n"); + return 1; + } + std::printf("[PASS] transient speaker channels are relabeled\n"); + return 0; +} From 857775bc67c9ad9800bc33b7dbe4905e72a00cde Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 11:09:53 +0000 Subject: [PATCH 08/11] fix(docs): align release guides, API reference, and CLI help --- CONTRIBUTING.md | 23 ++++ README.md | 41 ++++-- app/serve.cpp | 8 +- app/transcribe.cpp | 11 +- config/README.md | 4 +- config/asr.example.yaml | 31 +++-- config/diar.example.yaml | 6 +- config/nmt.example.yaml | 5 +- config/server.example.yaml | 14 +- config/tts.example.yaml | 3 - docs/README.md | 1 + docs/api.md | 62 ++++++--- docs/asr/configuration.md | 161 ++++++++++------------ docs/asr/customization.md | 31 +++-- docs/asr/models.md | 142 +++++++++---------- docs/build.md | 92 +++++++++---- docs/cli.md | 77 ++++++++--- docs/clients.md | 24 +++- docs/development/asr-batching.md | 5 + docs/development/cublas-shim.md | 16 ++- docs/development/diagnostics.md | 16 ++- docs/development/windows-build.md | 89 ++++-------- docs/install.md | 47 +++++-- docs/model-conversion.md | 17 ++- docs/nmt/configuration.md | 22 ++- docs/nmt/models.md | 11 +- docs/server.md | 56 +++++--- docs/troubleshooting.md | 5 +- docs/tts/configuration.md | 50 +++---- docs/tts/models.md | 18 +-- include/nemo_speech/asr.h | 2 +- src/asr/CMakeLists.txt | 2 +- src/tts/magpietts/README.md | 140 +++++++++---------- src/tts/nanocodec/README.md | 28 ++-- src/tts/tokenizer/mandarin_data/README.md | 1 + 35 files changed, 700 insertions(+), 561 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 86937c3..eec61df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,29 @@ We welcome external contributions to NeMo-Speech.cpp. +## Development checks + +Follow the [source-build guide](docs/build.md) for prerequisites and submodules. +For a model-independent CPU ASR test build: + +```bash +git submodule update --init ggml llama.cpp +scripts/configure.sh cpu-asr -DNEMO_SPEECH_BUILD_TESTS=ON +cmake --build --preset cpu-asr +ctest --test-dir build/cpu-asr --output-on-failure +``` + +Install [pre-commit](https://pre-commit.com/) and run the same formatting, +license-header, and static file checks used by CI: + +```bash +pre-commit run --all-files +``` + +Use the closest matching CUDA, Metal, Vulkan, server, or component preset when +the change affects code outside the CPU ASR path. Include the commands and +results relevant to the change in the pull request. + ## Contribution license and provenance Unless a file states otherwise, contributions are submitted under the diff --git a/README.md b/README.md index b231ad2..5522e03 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ # NeMo-Speech.cpp -A lightweight native C++ runtime for running NVIDIA Nemotron Speech model family locally, with broad hardware support. It supports multilingual speech recognition, speaker diarization, translation, and speech synthesis in realtime and batch mode. +A lightweight native C++ runtime for running the NVIDIA Nemotron Speech model family locally, with broad hardware support. It supports multilingual speech recognition, speaker diarization, translation, and speech synthesis in real-time and batch modes. - -NeMo-Speech.cpp is NVIDIA's official local speech inference solution, with day-0 support for our latest speech models. It builds on models from [NVIDIA NeMo Speech](https://github.com/NVIDIA-NeMo/Speech), with native inference powered by [ggml](https://github.com/ggerganov/ggml). +NeMo-Speech.cpp is NVIDIA's official local speech inference solution, with day-0 support for our latest speech models. It builds on models from [NVIDIA NeMo Speech](https://github.com/NVIDIA-NeMo/Speech), with native inference powered by [ggml](https://github.com/ggml-org/ggml). ## Models and applications @@ -45,21 +44,28 @@ On Windows, run from PowerShell: irm https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.ps1 | iex ``` +Open a new PowerShell window after installation so the updated user `PATH` +takes effect. + The installer prefers a verified native release and falls back to a source build when an artifact is unavailable. A source build requires Git, CMake 3.26 -or newer, Ninja, a C++17 compiler, and the selected GPU toolkit. See +or newer, Ninja, a C++17 compiler, SentencePiece development files, and the +toolchain required by the selected backend, if any. See [Installation](docs/install.md) for platform-specific prerequisites and options. ## Quick start -Transcribe the bundled sample. On first use, the CLI downloads the pinned -default Nemotron 3.5 GGUF from Hugging Face and verifies its size and SHA-256: +Transcribe a local WAV file. On first use, the CLI downloads the pinned default +Nemotron 3.5 GGUF from Hugging Face and verifies its size and SHA-256: ```bash -nemo-speech transcribe test_files/asr/wav/test/jfk.wav +nemo-speech transcribe /path/to/audio.wav ``` +Source checkouts can use `test_files/asr/wav/test/jfk.wav` as a smoke-test +input. + The same command can transcribe the default microphone on builds that include live capture: @@ -92,9 +98,9 @@ nemo-speech serve \ --open ``` -The server binds to by default and also provides a -documented OpenAI-compatible audio API subset and realtime WebSocket -transcription. A separately built `riva_server` binary provides the +The server binds to by default. Its transcription and +speech routes expose documented OpenAI-compatible subsets, alongside realtime +WebSocket transcription. A separately built `riva_server` binary provides the Riva-compatible gRPC interface. See the [server guide](docs/server.md) when you are ready to integrate either interface. @@ -114,8 +120,9 @@ gRPC usage. ## Build from source -Requires CMake 3.26 or newer, Ninja, C and C++17 compilers, and a supported -CUDA toolkit. For a CUDA ASR and TTS server with the playground: +Requires CMake 3.26 or newer, Ninja, C and C++17 compilers, SentencePiece +development files, and the toolchain required by the selected backend, if any. +For a CUDA ASR and TTS server with the playground: ```bash git submodule update --init ggml llama.cpp third_party/cpp-httplib @@ -145,9 +152,13 @@ Windows, and container instructions are in ## License -NVIDIA-authored code is released under the [Apache License 2.0](LICENSE), with -the project copyright notice in [NOTICE](NOTICE). Third-party components retain -their respective terms; see [Third-Party Notices](THIRD_PARTY_NOTICES.md). +NVIDIA-authored code is released under the +[Apache License 2.0](https://github.com/NVIDIA/NeMo-Speech.cpp/blob/main/LICENSE), +with the project copyright notice in +[NOTICE](https://github.com/NVIDIA/NeMo-Speech.cpp/blob/main/NOTICE). Third-party +components retain their respective terms; see +[Third-Party Notices](https://github.com/NVIDIA/NeMo-Speech.cpp/blob/main/THIRD_PARTY_NOTICES.md). +Release archives also include these files under `share/licenses/nemo-speech/`. ## Contributing diff --git a/app/serve.cpp b/app/serve.cpp index de4e2ab..f855754 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -521,7 +521,8 @@ run_server(int argc, char** argv) { #endif if (!engines.ready()) throw std::runtime_error( - "no models were loaded; pass --asr-model, --tts-model, --nmt-model, or --config"); + "no models were loaded; pass --asr-model, --diar-model, --tts-model, --nmt-model, " + "or --config"); if (!no_warmup) { nemo_speech::WarmupOptions warmup; #if defined(NEMO_SPEECH_CLI_TTS) @@ -606,8 +607,9 @@ void print_serve_help(const char* program) { std::printf( "Usage: %s serve [options]\n\n" - "Start the OpenAI-compatible HTTP API and browser playground. Models are\n" - "provided as local paths, indexed names, or through a YAML configuration.\n\n" + "Start the local speech HTTP API and browser playground. The transcription\n" + "and speech routes expose OpenAI-compatible subsets. Models are provided as\n" + "local paths, indexed names, or through a YAML configuration.\n\n" "Server:\n" " --host ADDRESS Bind address (default: 127.0.0.1)\n" " --port N HTTP port (default: 8080)\n" diff --git a/app/transcribe.cpp b/app/transcribe.cpp index c219a2a..8084317 100644 --- a/app/transcribe.cpp +++ b/app/transcribe.cpp @@ -502,10 +502,11 @@ print_transcribe_help(const char* program) { " -o, --output PATH Output path for one input\n" " --output-dir DIR Preserve directory layout under DIR\n" " -r, --recursive Recurse into input directories\n" - " --word-times Include word timestamps in diagnostics\n" + " --word-times Compatibility flag; JSON/subtitles imply it\n" " --vad-model PATH Optional Silero VAD GGUF\n" - " --diar-model PATH Tag words with a Sortformer diarizer\n" - " --max-speaker-count N Diarization speaker cap (default 8)\n" + " --vad-masking Mask silence features (requires --vad-model)\n" + " --diarize Tag words with the default Sortformer diarizer\n" + " --diar-model MODEL Tag words with a selected Sortformer diarizer\n" " --itn-model-dir DIR Inverse text normalization grammars\n" " --pnc-model PATH Punctuation/capitalization GGUF\n" #if defined(NEMO_SPEECH_CLI_NMT) @@ -515,7 +516,9 @@ print_transcribe_help(const char* program) { " --no-punctuation Disable automatic punctuation\n" " --verbatim Disable ordinary ITN\n" " --stream Stream chunks from a recorded WAV input\n" - " --max-alternatives N Request N-best hypotheses\n" + " --endpointing Finalize streaming utterances on silence\n" + " --stop-history-eou-ms N Endpoint silence threshold (default 800)\n" + " --max-alternatives N Request N-best; current decoders return one\n" " --speech-context PHRASE Add a decoder boost phrase (repeatable)\n" " --speech-context-boost N Boost applied to speech context phrases\n" " --profanity-filter Mask words from the configured list\n" diff --git a/config/README.md b/config/README.md index 4c181a0..fff378f 100644 --- a/config/README.md +++ b/config/README.md @@ -7,8 +7,8 @@ Use `config/asr.example.yaml` for ASR-only, `config/diar.example.yaml` for standalone diarization, `config/tts.example.yaml` for TTS-only, `config/nmt.example.yaml` for NMT-only, or `config/server.example.yaml` for a combined server. Capabilities are enabled -automatically when their required model paths are present; each `enabled` key -may be set to `true`, `false`, or `auto`. +automatically when their required model paths are present. The `asr.enabled`, +`nmt.enabled`, and `tts.enabled` keys may be set to `true`, `false`, or `auto`. See [Server configuration](../docs/server.md#engine-and-listener-configuration) for YAML, environment-variable, and CLI precedence. diff --git a/config/asr.example.yaml b/config/asr.example.yaml index b32810e..e0d977d 100644 --- a/config/asr.example.yaml +++ b/config/asr.example.yaml @@ -1,20 +1,20 @@ # nemo-speech ASR config. Load with: nemo-speech serve --config this.yaml # -# The nested maps mirror the dotted keys exactly: `asr.vad.masker.onset` below -# is the same setting as `--asr.vad.masker.onset` on the command line. The tree -# is generated from the config structs (RecognizerConfig::Register), so it stays -# in sync automatically; `nemo-speech serve --help` lists common options. +# Nested maps mirror dotted command-line keys: `asr.vad.masker.onset` is the +# same setting as `--asr.vad.masker.onset`. See docs/asr/configuration.md for +# the complete key reference. # # Precedence is: this file < NEMO_SPEECH_* env vars < CLI flags. # Every key is optional - omitted keys keep their built-in defaults. An UNKNOWN # key is a hard error (so typos are caught, not silently ignored). The values -# shown here are the defaults unless noted; paths are placeholders to fill in. +# shown here are the defaults unless noted; the model reference and commented +# paths are examples. asr: backend: gpu: 0 # GPU device index, -1 = CPU model: - path: /models/nemotron-speech-streaming-en-0.6b.q8_0.gguf + path: nemotron-3.5 # indexed name, HF repo ID, or local GGUF # name: # display name; default is derived from the model streaming: @@ -23,6 +23,13 @@ asr: ctc_right_padding: 1.92 # CTC right context (seconds) rnnt_right_context: 1 # cache-aware R: 1 = low-latency preset, -1 = model max + batching: + enabled: true # nemo-speech serve default; disable to minimize single-request latency + max_batch_size: 1024 + max_queue_delay_us: 5000 + state_arena_slots: 16 + offline_bucket_ms: 0 # 0 = do not silence-pad offline inputs into shared buckets + decoder: kind: greedy # greedy | flashlight # Flashlight LM beam (set kind: flashlight, or just lm_path which implies it): @@ -36,9 +43,9 @@ asr: # word_insertion_score: 1.0 vad: - model_path: /models/silero-v6.2.0.gguf # empty = no VAD loaded + # model_path: /models/silero-v6.2.0.gguf # empty = no VAD loaded masker: - mask_enable: false # zero silent mel frames before the encoder + mask_enable: false # mask silence features before the encoder onset: 0.5 # prob > onset -> enter speech offset: 0.3 # prob < offset -> leave speech # pad_onset_ms: 200 @@ -46,10 +53,14 @@ asr: # min_duration_off_ms: 500 endpointing: - enable: true # mid-stream end-of-utterance - vad_based: true # ride the VAD timeline (needs vad.model_path); else token-silence + enable: false # mid-stream end-of-utterance + vad_based: false # ride the VAD timeline (needs vad.model_path); else token-silence stop_history_eou_ms: 800 # trailing silence before EOU (ms) + diar: + # model_path: sortformer # enables word-level speaker tags when requested + preset: streaming # streaming | offline (larger chunks and caches) + postproc: # pnc_model_path: /models/pnc.gguf # punctuation + capitalization # itn_model_dir: /models/sparrowhawk_en # inverse text normalization diff --git a/config/diar.example.yaml b/config/diar.example.yaml index c3e7be0..798ff0e 100644 --- a/config/diar.example.yaml +++ b/config/diar.example.yaml @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Pull the published diarization model first, then use this file with either: +# The indexed diarization model is downloaded on first use. Run with either: # nemo-speech diarize meeting.wav --config config/diar.example.yaml # nemo-speech serve --config config/diar.example.yaml diar: - model_path: nvidia/diar-streaming-sortformer-4spk-v2 - preset: streaming # streaming | offline + model_path: nvidia/diar_streaming_sortformer_4spk-v2 + preset: streaming # streaming | offline (larger chunks and caches) diff --git a/config/nmt.example.yaml b/config/nmt.example.yaml index a3e386e..ca9ab10 100644 --- a/config/nmt.example.yaml +++ b/config/nmt.example.yaml @@ -4,8 +4,8 @@ # nemo-speech serve --config config/nmt.example.yaml # # Requires a build with -DNEMO_SPEECH_BUILD_NMT=ON. The translation model is a -# Riva-Translate GGUF; the language pair is selected per request via the -# source_language / target_language fields of TranslateText. +# Riva-Translate GGUF; the language pair is selected per request with the +# source_language and target_language fields. nmt: enabled: true # true | false | auto backend: @@ -20,4 +20,3 @@ nmt: pool: contexts: 1 # concurrent decode contexts; each adds one # n_ctx-sized KV cache. Raise for concurrency. - verbose: false # enable verbose llama.cpp loader/runtime logs diff --git a/config/server.example.yaml b/config/server.example.yaml index 3e6dbac..35319f0 100644 --- a/config/server.example.yaml +++ b/config/server.example.yaml @@ -3,8 +3,8 @@ # Load with: # nemo-speech serve --config config/server.example.yaml # -# ASR, NMT, and TTS are enabled automatically when their required model paths -# are present. Set `enabled: false` under a section to force that service off. +# ASR, diarization, NMT, and TTS are enabled when their required model paths +# are present. Set `enabled: false` under ASR, NMT, or TTS to force it off. http: enabled: true host: 127.0.0.1 @@ -49,6 +49,11 @@ asr: # itn_model_dir: /models/sparrowhawk_en # profanity_list_path: /models/profanity.txt +# This model also enables word-level speaker tags on ASR responses. +diar: + model_path: nvidia/diar_streaming_sortformer_4spk-v2 + preset: streaming # streaming | offline (larger chunks and caches) + tts: enabled: true # true | false | auto magpie-model: /models/magpie-tts/magpie_tts_multilingual_357m.v2602.f16.gguf @@ -83,8 +88,6 @@ tts: uma-mode: auto # auto | off | on longform: auto # auto | off | on - benchmark: false - verbose: false voice-name: warmup-enabled: true warmup-text: "Hello from Magpie T T S." @@ -98,9 +101,8 @@ nmt: gpu: 0 # GPU device index, -1 = CPU model: path: /models/riva-translate-4b-instruct-v2.q8_0.gguf # f16 also works - n_ctx: 8192 + n_ctx: 1024 # raise for longer inputs; KV cache grows with n_ctx generation: max_new_tokens: 256 pool: contexts: 1 # concurrent decode contexts - verbose: false diff --git a/config/tts.example.yaml b/config/tts.example.yaml index cb10bb2..31148f2 100644 --- a/config/tts.example.yaml +++ b/config/tts.example.yaml @@ -44,9 +44,6 @@ tts: uma-mode: auto # auto | off | on longform: auto # auto | off | on - benchmark: false - verbose: false - # Server defaults and startup warmup: # voice-name: John # warmup-enabled: true diff --git a/docs/README.md b/docs/README.md index 5d628aa..115bae0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,7 @@ Start with: - [Installation](install.md) - [Command-line workflows](cli.md) +- [Configuration examples](../config/README.md) - [Model conversion](model-conversion.md) - [HTTP/realtime and optional gRPC server](server.md) - [HTTP API reference](api.md) diff --git a/docs/api.md b/docs/api.md index 3d6f787..e9d6154 100644 --- a/docs/api.md +++ b/docs/api.md @@ -4,21 +4,30 @@ Complete field reference for the `nemo-speech serve` HTTP API. Every endpoint follows the same conventions: - **Auth**: if the server was started with an API key, send - `Authorization: Bearer ` (WebSocket clients may use `?api_key=`). + `Authorization: Bearer ` to `/v1` routes (WebSocket clients may use + `?api_key=`). The playground, health, readiness, and version routes are + intentionally unauthenticated. - **Errors**: non-2xx responses carry `{"error": {"message": "...", "type": "invalid_request_error" | "server_error"}}`. -- **Audio uploads**: multipart form with the WAV in `file`. Sample rate is read - from the WAV header (8-96 kHz accepted). +- **Audio uploads**: multipart form with the uncompressed RIFF/WAVE file in + `file`. Mono or stereo PCM16 and float32 WAVs at 8-96 kHz are accepted; + stereo is downmixed to mono. -Endpoints whose model is not loaded return an error (`server_error`, or 501 if -compiled out); `GET /v1/models` lists what this server can do. +Endpoints return 501 when their capability is not included in the build and a +`server_error` when its model is not loaded. `GET /v1/models` lists the active +capabilities. + +OpenAI SDK compatibility covers model listing and the documented subsets of +`/v1/audio/transcriptions` and `/v1/audio/speech`; it does not extend to other +OpenAI APIs. Realtime transcription uses the WebSocket contract below, not the +OpenAI Realtime API. ## Service | Method + path | Purpose | |---|---| | `GET /` | bundled playground UI | -| `GET /health`, `GET /ready` | liveness (`{"status": "ok", ...}`) / readiness (`{"ready": true, ...}`) | +| `GET /health`, `GET /ready` | compact health status (`{"status": "ok", ...}`) / detailed readiness (`{"ready": true, ...}`) | | `GET /version` | build version | | `GET /v1/models` | loaded models and capabilities | @@ -29,6 +38,7 @@ Speech-to-text (OpenAI-compatible multipart subset). Alias: | Field | Type | Default | Description | |---|---|---|---| +| `model` | string | loaded ASR model | accepted for client compatibility; this server uses its one loaded ASR model | | `file` | WAV upload | required | audio to transcribe | | `language` | string | model default | language code; prompt-conditioned models select their language prompt | | `response_format` | string | `json` | `json`, `verbose_json` (adds words/timestamps), `text`, `srt`, or `vtt` | @@ -36,7 +46,7 @@ Speech-to-text (OpenAI-compatible multipart subset). Alias: | `verbatim` | bool | `false` | skip inverse text normalization | | `profanity_filter` | bool | `false` | mask words from the configured list | | `diarization` | bool | `false` | tag words with speakers (requires `verbose_json` and a diarizer model) | -| `max_speaker_count` | int | `8` | diarization speaker cap | +| `max_speaker_count` | int | ignored | deprecated compatibility field; Sortformer v2 supports up to four speakers | | `speech_contexts` | JSON array | none | word boosting, `[{"phrases": ["..."], "boost": N}]` - same shape as gRPC; scoring: [word boosting](asr/configuration.md#word-boosting) | | `prompt` | string | none | OpenAI-compat: one boosted phrase at boost 10 | @@ -53,10 +63,10 @@ punctuation-aware cue grouping as the file CLI. ## WebSocket /v1/realtime -Live PCM16 transcription (OpenAI realtime-style events). The server sends -`session.created` on connect. Optionally send one `session.update` JSON event -(rejected once audio has started); then binary little-endian PCM16 frames (or -base64 chunks in `input_audio_buffer.append`); finish with +Live PCM16 transcription using a project-specific event protocol. The server +sends `session.created` on connect. Optionally send one `session.update` JSON +event (rejected once audio has started); then binary little-endian PCM16 frames +(or base64 chunks in `input_audio_buffer.append`); finish with `input_audio_buffer.commit`. `input_audio_buffer.clear` or `response.cancel` discards buffered audio (`input_audio_buffer.cleared`). @@ -70,8 +80,8 @@ discards buffered audio (`input_audio_buffer.cleared`). | `verbatim` | bool | `false` | skip inverse text normalization | | `profanity_filter` | bool | `false` | mask words from the configured list | | `word_timestamps` | bool | `false` | word timings on final events | -| `speaker_diarization` | bool | `false` | tag words with speakers | -| `max_speaker_count` | int | `8` | diarization speaker cap | +| `speaker_diarization` | bool | `false` | tag words with speakers; requires a loaded diarizer | +| `max_speaker_count` | int | ignored | deprecated compatibility field; Sortformer v2 supports up to four speakers | | `endpointing_ms` | number | server default | end-of-utterance silence threshold | | `speech_contexts` | array | none | word boosting, as in `/v1/audio/transcriptions` | | `prompt` | string | none | OpenAI-compat: one boosted phrase at boost 10 | @@ -87,14 +97,23 @@ Text-to-speech (OpenAI-compatible JSON subset). | Field | Type | Default | Description | |---|---|---|---| +| `model` | string | loaded TTS model | accepted for client compatibility; this server uses its one loaded TTS model | | `input` | string | required | text to synthesize | -| `voice` | string | model default | voice name | +| `voice` | string | model default | local voice name, model-qualified voice name, or zero-based speaker index | | `language` | string | model default | language code | | `speed` | number | `1.0` | only `1.0` is accepted; other values return 400 | -| `sample_rate` | int | model default | output sample rate | +| `sample_rate` | int | model default | output sample rate, from 8000 Hz through the model rate (22050 Hz for the supported NanoCodec model) | | `response_format` | string | `wav` | `wav` or `pcm` | -Response: audio bytes with the matching content type. +Response: mono signed PCM16, either in a WAV container or raw little-endian +bytes with the matching content type. The complete audio is buffered before the +HTTP response; streaming synthesis is not part of this compatibility subset. + +Local voice names are case-insensitive and are listed in the `voices` field of +the speech entry returned by `GET /v1/models`. `.` is also +accepted. `default` and supported OpenAI voice aliases such as `alloy` select +the server's configured default local speaker; they do not provide the +corresponding hosted OpenAI voices. An unrecognized local name returns 400. ## POST /v1/translations @@ -134,10 +153,10 @@ fields as `/v1/audio/translations`, except `target_language` is required and |---|---|---|---| | `target_language` | string | required | target language code | | `response_format` | string | `wav` | `wav` or `pcm` | -| `voice` | string | model default | TTS voice for the translated audio | -| `sample_rate` | int | model default | output audio sample rate | +| `voice` | string | model default | TTS voice for the translated audio; follows `/v1/audio/speech` voice rules | +| `sample_rate` | int | model default | output rate, from 8000 Hz through the loaded TTS model rate | -Response: translated audio. +Response: translated mono signed PCM16 audio in the requested container. ## POST /v1/audio/diarizations @@ -146,7 +165,10 @@ Speaker segmentation without transcription. Alias: `/v1/diarizations`. | Field | Type | Default | Description | |---|---|---|---| | `file` | WAV upload | required | audio to segment | -| `mode` | string | `streaming` | `streaming` or `offline` | +| `mode` | string | `streaming` | `streaming` for long-form audio, or full-attention `offline` for recordings up to about 6.6 minutes | Response: `{"segments": [{"start": s, "end": s, "speaker": n}]}` (1-based speaker ids). + +Request `mode=offline` uses full attention. It is distinct from +`diar.preset: offline`, which still uses the streaming path. diff --git a/docs/asr/configuration.md b/docs/asr/configuration.md index 4393678..f5eae25 100644 --- a/docs/asr/configuration.md +++ b/docs/asr/configuration.md @@ -1,8 +1,10 @@ # ASR configuration -Feature-level ASR configuration shared by `nemo-speech serve` and the -separate `riva_server`. For how keys are set (YAML / env / CLI precedence, -`--config`, listeners, service enablement), see +Recognizer configuration shared by `nemo-speech transcribe`, +`nemo-speech serve`, and the separate `riva_server`. The `asr.enabled` switch +is server-only; the remaining keys configure the recognizer itself. For how +keys are set (YAML / env / CLI precedence, `--config`, listeners, service +enablement), see [Server configuration](../server.md#engine-and-listener-configuration). - [Key reference](#key-reference) @@ -11,7 +13,7 @@ separate `riva_server`. For how keys are set (YAML / env / CLI precedence, - [VAD feature masking](#vad-feature-masking) - [Endpointing](#endpointing) - [Postprocessing: profanity, ITN, PnC](#postprocessing-profanity-itn-pnc) -- [Riva parity: known exclusions](#riva-parity-known-exclusions) +- [gRPC compatibility](#grpc-compatibility) ## Key reference @@ -29,12 +31,13 @@ column lists the short flag where one exists - the dotted form | `asr.streaming.ctc_left_padding` | `--left-pad-sec` | `1.92` | CTC left context (s) | | `asr.streaming.ctc_right_padding` | `--right-pad-sec` | `1.92` | CTC right context (s) | | `asr.streaming.rnnt_right_context` | - | `1` | cache-aware R; `-1` = model max | -| `asr.batching.enabled` | - | `false` | opt in to batching compatible neural work | -| `asr.batching.max_batch_size` | - | `1024` | maximum items in one neural microbatch | +| `asr.batching.enabled` | - | surface-dependent | batch compatible neural work; see below | +| `asr.batching.max_batch_size` | - | `1024` | maximum items combined in one neural batch | | `asr.batching.max_queue_delay_us` | - | `5000` | bounded wait for compatible work (µs) | | `asr.batching.max_queue_depth` | - | `2048` | pending-job backpressure per neural stage | -| `asr.batching.ingress_cohort_delay_us` | - | `20000` | streaming ingress-wave alignment window (µs) | +| `asr.batching.ingress_cohort_delay_us` | - | `20000` | maximum wait for aligning concurrent streaming inputs (µs) | | `asr.batching.state_arena_slots` | - | `16` | concurrent RNNT/VAD state rows resident on device | +| `asr.batching.offline_bucket_ms` | - | `0` | silence-pad offline inputs to this duration multiple; `0` disables bucketing | | `asr.decoder.kind` | - | `greedy` | `greedy` / `flashlight` | | `asr.decoder.lm_path` | `--lm-path` | - | KenLM `.bin`/`.arpa` (implies flashlight) | | `asr.decoder.lexicon_path` | `--lexicon` | - | flashlight lexicon TSV | @@ -59,7 +62,7 @@ column lists the short flag where one exists - the dotted form | `asr.vad.masker.stddev_floor` | `--vad-stddev-floor` | `1e-5` | normalization denom floor | | `asr.vad.masker.mask_value` | `--vad-mask-value` | `-16.635` | log-mel fill for masked frames | | `asr.diar.model_path` | `--diar-model` | - | Sortformer diarizer GGUF (empty = diarization unavailable) | -| `asr.diar.preset` | `--diar-preset` | `streaming` | geometry preset: `streaming` \| `offline` (long-form batch mode: 8 s chunks + bigger AOSC caches); replaces the individual keys below | +| `asr.diar.preset` | `--diar-preset` | `streaming` | `streaming` or `offline` (8 s chunks and larger caches; both stream); replaces the individual keys below | | `asr.diar.chunk` | `--diar-chunk` | `20` | chunk length (80 ms frames) | | `asr.diar.right_context` | `--diar-rc` | `0` | chunk right context (frames) | | `asr.diar.left_context` | `--diar-lc` | `0` | chunk left context (frames) | @@ -78,32 +81,34 @@ column lists the short flag where one exists - the dotted form The examples below use short aliases for brevity; each maps to the dotted key and works identically in YAML. -Batching is off by default to preserve local B=1 latency. High-concurrency workloads -can enable it with `--asr.batching.enabled true`; the default 5 ms neural queue -window then coalesces exact-shape CTC, RNNT, TDT, VAD, and PnC work while -preserving bounded backpressure. Concurrent RecognitionStreams use a separate -20 ms ingress cohort window by default so callers reach FE and the acoustic -stage as a small number of useful waves instead of fragmenting the exact-shape -graph cache. This behavior belongs to each `RecognitionStream`, independent of -the interface that created it. Set -`NEMO_SPEECH_INGRESS_COHORT=0` only for diagnostics. Batching also switches -streaming feature extraction to a shared batched graph when CUDA is available. -Without request batching, CUDA streaming still uses the GPU frontend but bypasses -the ingress queue; `NEMO_SPEECH_STREAM_GPU_FE=0` selects the CPU diagnostic -path. Full-utterance CTC, RNNT, and TDT also use GPU feature extraction whenever -CUDA is available. +Batching is off by default for direct library use to preserve single-request +latency. +`nemo-speech serve` enables it by default, while `nemo-speech transcribe` and +`nemo-speech bench` enable and size it automatically only when they run more +than one utterance concurrently. `nemo-speech transcribe --no-batching` +disables that command's automatic policy. The separate gRPC server and direct +library users opt in with `--asr.batching.enabled`. + +With batching enabled, the default 5 ms neural queue window combines compatible +CTC, RNNT, TDT, VAD, and PnC work while preserving bounded backpressure. +`offline_bucket_ms` can silence-pad offline utterances to compatible lengths; +it is disabled by default because padding adds work. + +HTTP and gRPC streaming also coordinate concurrent input streams before +batching. Direct library calls and CLI commands do not add that transport +delay. See [ASR batching](../development/asr-batching.md) for tuning and backend +details. ## CTC decoding: greedy vs flashlight CTC heads run greedy argmax by default. Set `asr.decoder.lm_path` (KenLM `.bin`/`.arpa`) + `asr.decoder.lexicon_path` (flashlight lexicon TSV) to enable -the flashlight LexiconDecoder with n-gram rescoring. Requires a +Flashlight beam search with n-gram rescoring. Requires a `-DNEMO_SPEECH_WITH_FLASHLIGHT=ON` build; greedy is always available. That build dynamically links `libkenlm` on Unix or `kenlm.dll` on Windows. The library must be available on the shared-library search path; the build layouts -stage it beside the other runtime libraries. Flashlight itself remains private -in the ASR library. +stage it beside the other runtime libraries. ```bash riva_server \ @@ -119,27 +124,21 @@ language model, lexicon, and audio domain. ## Word boosting -Per-request word boosting biases the decoder toward caller-supplied phrases -(names, jargon). Sent via `RecognitionConfig.speech_contexts` (`{phrases[], -boost}`); stock clients expose `--boosted_words` + `--boosted_words_score`. -The same name and shape apply on every surface: gRPC, HTTP form field, realtime -`session.update` (`[{"phrases": ["..."], "boost": N}]`), and the CLI -`--speech-context` flags. HTTP `prompt` is an OpenAI-compat shim: one phrase at -boost 10. Requests are portable between CTC and cache-aware RNNT; each clamps -to its safe range. Heads without boosting support (greedy CTC without an LM, -Parakeet TDT) ignore `speech_contexts` with a one-time warning. - -**Tokenizer (both heads):** boost phrases are tokenized with the SentencePiece -tokenizer **embedded in the GGUF** (`asr.tokenizer.spm_model`), which is -guaranteed to match the model's vocab. GGUFs converted before the embed: -re-convert with `convert_model.py`, or (CTC only) pass an external -`asr.decoder.tokenizer_path` as an override. +Per-request word boosting biases recognition toward names, jargon, and other +caller-supplied phrases. Use `RecognitionConfig.speech_contexts` in gRPC, the +`speech_contexts` HTTP/realtime field, or `--speech-context` in the CLI. HTTP +`prompt` supplies one phrase with boost 10. Flashlight CTC and cache-aware RNNT +support boosting; greedy CTC without an LM and Parakeet TDT ignore it. + +**Tokenizer (CTC and RNNT):** boosting requires the SentencePiece tokenizer +embedded in current GGUFs. Reconvert older GGUFs with `convert_model.py`, or +for CTC set `asr.decoder.tokenizer_path` to a matching external tokenizer. **CTC (flashlight, beam)** - requires the flashlight LM decoder (`asr.decoder.lm_path` + `asr.decoder.lexicon_path`); greedy CTC with no LM -ignores `speech_contexts` (warns once). The boost is a per-word bump inside the -beam-search LM score: competing hypotheses push back, so scores run high. -Clamp `asr.decoder.max_boost` (default `10`); typical requests 8-10. +ignores `speech_contexts`. Typical request scores are 8-10 and are capped by +`asr.decoder.max_boost` (default `10`). CTC and RNNT scores are not directly +comparable. ```bash riva_server \ @@ -152,14 +151,11 @@ riva_streaming_asr_client --riva_uri=localhost:50051 \ --boosted_words="nvidia,parakeet,nemotron" --boosted_words_score=8.0 ``` -**Cache-aware RNNT (greedy)** - built in, no flashlight or LM artifacts: -phrases compile into a context-biasing tree (Aho-Corasick shallow fusion, the -approach NeMo uses). Greedy has no competing hypothesis to push back, so -each point of score is far more potent than on CTC - roughly 3x. Clamp `asr.decoder.boosting_max_boost` -(default `5.0`); typical requests 2-3. Tune deployment-wide strength with -`asr.decoder.boosting_tree_alpha` (default `1.0`, `0` disables); -`asr.decoder.boosting_depth_scaling` (default `2.0`) shapes how the score grows -along a phrase match. +**Cache-aware RNNT (greedy)** - built in and requires no LM artifacts. Typical +request scores are 2-3 and are capped by +`asr.decoder.boosting_max_boost` (default `5.0`). Tune overall strength with +`asr.decoder.boosting_tree_alpha` (default `1.0`; `0` disables boosting) and +`asr.decoder.boosting_depth_scaling` (default `2.0`). ```bash riva_server \ @@ -172,20 +168,14 @@ riva_streaming_asr_client --riva_uri=localhost:50051 \ --boosted_words="Kowalczyk,Nemotron" --boosted_words_score=3.0 ``` -A CTC-scale request against an RNNT head is safe: the score clamps to 5.0 and -stays in the stable region (no repeats or insertions). - ## VAD feature masking -Optional Silero VAD floors silence mel frames before the encoder, matching -NVIDIA Riva's VAD feature-masking behavior. **Off by default even with a VAD model loaded** - -the default Riva configuration uses VAD for endpointing without feature masking -(`mask_features=false`). Enable masking with `asr.vad.masker.mask_enable` -(`--vad-masking 1`). Always compiled; composes with greedy or LM decoding on both -CTC and RNNT. +Optional Silero VAD masks silence features before the encoder. It remains off +when a VAD model is loaded unless `asr.vad.masker.mask_enable` +(`--vad-masking`) is set. VAD masking works with greedy or LM decoding on CTC +and RNNT. -The VAD model is a separate GGUF (`general.architecture="vad"`), not bundled into -the ASR model: +The VAD model is a separate GGUF, not part of the ASR model: ```bash pip install "silero-vad==6.2.0" @@ -194,35 +184,27 @@ python3 convert_model.py silero --outfile models/silero-v6.2.0.gguf nemo-speech serve \ --asr.model.path parakeet-ctc-1.1b.q8_0.gguf --gpu 0 \ --lm-path lm.bin --lexicon lexicon.txt \ - --vad-model models/silero-v6.2.0.gguf --vad-masking 1 + --vad-model models/silero-v6.2.0.gguf --vad-masking ``` -See the `asr.vad.masker.*` keys in the [reference](#key-reference) for onset / -offset / padding / mask-value tunables. The decision follows the same -`vad_postprocessor` sequence used by Riva (binarize → pad → merge short -silences) before feature masking. +See the `asr.vad.masker.*` keys in the [reference](#key-reference) for onset, +offset, padding, and mask-value settings. ## Endpointing Off by default, the server emits one final when the client closes the stream. With `asr.endpointing.enable` (`--endpointing`) it detects end-of-utterance -mid-stream and emits a final per utterance (multiple `is_final=true` per stream), -matching Riva. Works on both heads; the per-utterance reset clears -transcript/token buffers. RNNT endpoints flush through the normal EOS path and -then reset encoder and predictor state while preserving the stream timeline, so -delayed tokens cannot leak into the next utterance. +mid-stream and emits a final per utterance (multiple `is_final=true` per stream). +It works with buffered CTC and cache-aware RNNT; the offline-only Parakeet TDT +model does not support streaming endpointing. EOU fires when trailing silence reaches `asr.endpointing.stop_history_eou_ms` -(default 800), then re-arms on the next speech. Both silence signals run on the -**decode clock** (audio actually decoded), never the raw-audio frontier - the -buffered CTC runner decodes ~2 s behind arriving audio, and firing on arrival -time would finalize before the tail is decoded. +(default 800), then re-arms on the next speech. - **token-silence (default)** - gap since the decoder's last non-blank frame; works with greedy and flashlight, no VAD model needed. - **VAD-driven** (`asr.endpointing.vad_based` + a VAD model) - silence from the - Silero timeline (Riva's `vad_based_eou=true`). Falls back to token-silence with - a warning if no VAD model. + Silero timeline. Falls back to token-silence with a warning if no VAD model. ```bash # token-silence EOU (default, no VAD), CTC, 1 s threshold: @@ -231,22 +213,19 @@ nemo-speech serve --asr.model.path parakeet-ctc-1.1b.q8_0.gguf --gpu 0 \ # VAD-driven EOU, RNNT: nemo-speech serve --asr.model.path nemotron-speech-streaming-en-0.6b.q8_0.gguf --gpu 0 \ - --endpointing --vad-based-eou 1 --vad-model models/silero-v6.2.0.gguf + --endpointing --vad-based-eou --vad-model models/silero-v6.2.0.gguf ``` -Masking and endpointing are independent and share the *same* Silero inference, so -you can run masking only, token-silence endpointing only, VAD-driven endpointing -without masking (the default), or both. Per-request Riva knobs: +Masking and endpointing are independent and can be enabled separately or +together. For Riva-compatible gRPC clients, `custom_configuration["stop_history_eou"]` overrides the threshold for a stream; a `runtime_config["force_eou"] = "true"` message finalizes the current utterance -immediately. Onset endpointing (`start_history`/`start_threshold`) and -`residue_tokens` / segment-gap variants are not implemented. +immediately. ## Postprocessing: profanity, ITN, PnC -Postprocessing runs on the final transcript in a fixed order: **profanity → ITN → -PnC**. Each stage is enabled by giving the server its artifact and gated per -request; stages with no artifact are skipped at zero cost. +Postprocessing runs on the final transcript in this order: **profanity → ITN → +PnC**. Each stage requires its configured artifact and request option. | stage | config key | build flag | per-request gate | |---|---|---|---| @@ -280,18 +259,16 @@ nemo-speech serve \ These request gates are independent. -## Riva parity: known exclusions +## gRPC compatibility -Intentional differences a side-by-side with riva-speech will surface: +The Riva-compatible gRPC adapter has these limitations: - **LINEAR_PCM only.** Mono 16-bit PCM from 8-96 kHz is accepted and resampled to the model rate with a streaming anti-alias filter. FLAC, µ-law, A-law, and Opus still require client-side transcoding. - **N-best output is not implemented.** `max_alternatives` is accepted, but current decoders return one alternative per result. -- **No utterance-onset gating** and no `min_duration_on` short-speech deletion: - pre-speech noise Riva suppresses can surface as partials. Empty-transcript - finals are swallowed server-side. +- **No utterance-onset gating** and no `min_duration_on` short-speech deletion. - **Confidence:** interim alternatives report 0.0. Final greedy CTC alternatives report mean token posterior (with per-word minima when timestamps are requested); RNNT and beam-decoded results currently report 1.0. diff --git a/docs/asr/customization.md b/docs/asr/customization.md index b3cb98c..9492c57 100644 --- a/docs/asr/customization.md +++ b/docs/asr/customization.md @@ -6,17 +6,16 @@ helps choose the right mechanism; the exact keys and defaults are in ## Feature matrix -Support depends mostly on the head type. Word boosting works on CTC (flashlight -LM decoder) and cache-aware RNNT (built-in context-biasing tree); -postprocessing and diarization are head-independent. +Support depends mostly on the head type. Word boosting works on CTC with the +Flashlight LM decoder and on cache-aware RNNT; postprocessing and diarization +are head-independent. -| Model | Languages | Word boosting | VAD masking | Endpointing | Profanity | ITN | Auto punctuation | Language ID | Diarization | +| Model | Languages | Word boosting | VAD masking | Endpointing | Profanity | ITN | Auto punctuation | Language metadata | Diarization | |---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Parakeet CTC 1.1B](models.md#parakeet-ctc-11b-offline--buffered-streaming) | en | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | -| [Nemotron-Speech 0.6B](models.md#nemotron-speech-streaming-06b-cache-aware-rnnt) | en | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | | [Nemotron 3.5 0.6B](models.md#nemotron-35-06b-multilingual-prompt-conditioned-rnnt) | 40+ | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | +| [Nemotron-Speech 0.6B](models.md#nemotron-speech-streaming-06b-cache-aware-rnnt) | en | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | | [Parakeet TDT 0.6B v3](models.md#parakeet-tdt-06b-v3-multilingual-offline-transducer) | 25 | No | No | No | Yes | Yes | Yes | No | Yes | - +| [Parakeet CTC 1.1B](models.md#parakeet-ctc-11b-offline--buffered-streaming) | en | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | ## Request-time options @@ -30,10 +29,9 @@ phrases (names, jargon); stock Riva clients expose `--boosted_words` + tokenizer - re-convert pre-embed GGUFs with `convert_model.py` (CTC also accepts an `asr.decoder.tokenizer_path` override). -CTC needs the flashlight LM decoder at startup and takes typical scores of -8-10; cache-aware RNNT boosts during greedy decoding with no extra artifacts, -and each point is ~3x more potent (typical 2-3). Mechanism, clamps, and field -syntax per surface: [word boosting](configuration.md#word-boosting). +CTC needs the Flashlight LM decoder and typically uses scores of 8-10. +Cache-aware RNNT requires no extra artifacts and typically uses 2-3. See +[word boosting](configuration.md#word-boosting) for fields and limits. ### Transcript postprocessing @@ -54,14 +52,17 @@ is requested without one. Word timestamps are enabled automatically. See [ASR configuration](configuration.md#key-reference) and [Sortformer models](models.md#sortformer-speaker-diarization). +Sortformer v2 supports up to four speakers. + For diarization without ASR, use `nemo-speech diarize` or the standalone `nemo_speech_diar_*` C API. ### Language selection Nemotron 3.5 accepts a request language such as `en-US` or `es-ES`, or `auto` -for model-based detection. The selected language is returned on the transcript -and words. Other listed ASR models do not perform language identification. +for model-based detection. Structured results return the selected or detected +language on the transcript and words. Other listed ASR models do not return +language-identification metadata. ### Force an endpoint @@ -87,8 +88,8 @@ These settings affect the loaded engine and therefore require a restart. for accuracy. Parakeet CTC instead uses its chunk and left/right padding settings. -## Compatibility notes +## gRPC compatibility The Riva-compatible gRPC adapter intentionally does not implement every Riva codec and recognition option. See -[Riva parity: known exclusions](configuration.md#riva-parity-known-exclusions). +[gRPC compatibility](configuration.md#grpc-compatibility). diff --git a/docs/asr/models.md b/docs/asr/models.md index 75fcf80..110ae18 100644 --- a/docs/asr/models.md +++ b/docs/asr/models.md @@ -12,85 +12,75 @@ nemo-speech pull nemotron-3.5 `nemotron-3.5` is the default when `--model` is omitted. A short name, full repository ID, or existing local GGUF path can be passed to `--model`. -## Parakeet CTC (1.1B, offline / buffered streaming) +## Nemotron 3.5 (0.6B, multilingual, prompt-conditioned RNNT) -Hugging Face: [nvidia/parakeet-ctc-1.1b](https://huggingface.co/nvidia/parakeet-ctc-1.1b) +A cache-aware FastConformer-RNNT with **language-ID prompt conditioning** across +40+ language-locales. Hugging Face: +[nvidia/nemotron-3.5-asr-streaming-0.6b](https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b) ```bash -nemo-speech pull parakeet-ctc +nemo-speech pull nemotron-3.5 ``` -## Parakeet TDT (0.6B v3, multilingual, offline transducer) - -Token-and-Duration Transducer: the joint predicts each token together with its -frame span. 25 European languages, self-punctuating. Hugging Face: -[nvidia/parakeet-tdt-0.6b-v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) +Select a language such as `en-US` or `es-ES`, or use `auto` for model-based +detection. Structured results include the selected or detected language: ```bash -nemo-speech pull parakeet-tdt +nemo-speech transcribe audio.wav \ + --model nemotron-3.5 \ + --language auto \ + --json ``` -The model is not cache-aware trained: inference is full-utterance only. -Streaming requests are rejected with an error; use offline recognition -(`nemo-speech transcribe`, `POST /v1/audio/transcriptions`, or gRPC -`Recognize`). +When ITN is configured with a parent grammar directory (`en/`, `es/`, ...), +the same explicit or auto-detected language code selects the grammar used for +the final transcript. Unsupported languages remain unchanged. + +The CLI uses this model by default. It supports whole-file recognition, +recorded streaming with `--stream`, and live microphone transcription. ## Nemotron-Speech Streaming (0.6B, cache-aware RNNT) -Hugging Face: [nvidia/nemotron-speech-streaming-en-0.6b](https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b) +English FastConformer-RNNT for whole-file or cache-aware streaming inference. +Hugging Face: +[nvidia/nemotron-speech-streaming-en-0.6b](https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b) ```bash nemo-speech pull nemotron-en ``` -## Nemotron 3.5 (0.6B, multilingual, prompt-conditioned RNNT) - -The same cache-aware FastConformer-RNNT plus **language-ID prompt conditioning** -across 40+ language-locales (`EncDecRNNTBPEModelWithPrompt`). Hugging Face: -[nvidia/nemotron-3.5-asr-streaming-0.6b](https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b) - -```bash -nemo-speech pull nemotron-3.5 -``` +## Parakeet TDT (0.6B v3, multilingual, offline transducer) -The GGUF contains the prompt metadata (`asr.rnnt.num_prompts`, -`asr.rnnt.prompt_dictionary`), and the runtime applies the model's -`prompt_kernel` language fusion ahead of the RNNT joint. Select the language via -the request's `language_code` (`en-US`, `es-ES`, ...) or `auto`; the `` tag -is stripped from the transcript and the detected language is returned on -`SpeechRecognitionAlternative.language_code` and per-word `WordInfo.language_code`: +Token-and-Duration Transducer: the joint predicts each token together with its +frame span. 25 European languages, self-punctuating. Hugging Face: +[nvidia/parakeet-tdt-0.6b-v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) ```bash -riva_server \ - --asr.model.path models/nemotron-3.5-asr-streaming-0.6b.q8_0.gguf \ - --bind 0.0.0.0:50051 - -# In another shell: -riva_streaming_asr_client --riva_uri=localhost:50051 \ - --audio_file=audio.wav --language_code=auto \ - --interim_results=false --word_time_offsets=true +nemo-speech pull parakeet-tdt ``` -When ITN is configured with a parent grammar directory (`en/`, `es/`, ...), -the same explicit or auto-detected language code selects the grammar used for -the final transcript. Unsupported languages remain unchanged. - -## Converting custom ASR checkpoints +The model does not support cache-aware streaming; inference is full-utterance +only. Streaming requests are rejected with an error; use offline recognition +(`nemo-speech transcribe`, `POST /v1/audio/transcriptions`, or gRPC +`Recognize`). -The root [`convert_model.py`](../../convert_model.py) converter accepts a local -`.nemo` archive, an extracted NeMo checkpoint, a local Hugging Face model -directory, or a Hugging Face repository ID. It emits the unified `asr.*` -metadata the runtime expects. The head type (CTC, RNNT, or TDT) is auto-detected -from `model_config.yaml`; override it with `--head-type {ctc,rnnt,tdt}`. +## Parakeet CTC (1.1B, offline / buffered streaming) -Install the conversion dependencies in a virtual environment: +English FastConformer-CTC for whole-file recognition or overlapping buffered +streaming. Hugging Face: +[nvidia/parakeet-ctc-1.1b](https://huggingface.co/nvidia/parakeet-ctc-1.1b) ```bash -pip install -r requirements.txt +nemo-speech pull parakeet-ctc ``` -The converter reads `.nemo` archives directly and does not require -`nemo_toolkit`. Remote checkpoints use the standard Hugging Face cache. +## Converting custom ASR checkpoints + +Use the root [`convert_model.py`](../../convert_model.py) converter for a custom +checkpoint or alternate quantization. Follow the [model conversion +guide](../model-conversion.md) to set up its isolated Python environment and +choose a supported source. ASR head type (CTC, RNNT, or TDT) is auto-detected; +use `--head-type` only when an override is needed. ## Quantization (`--outtype`) @@ -98,9 +88,8 @@ The converter reads `.nemo` archives directly and does not require python3 convert_model.py model.nemo --outfile model.gguf --outtype q8_0 ``` -Applied to Linear weights (encoder MHA + FFN, RNNT LSTM predictor, joint -projections) and the ConformerConv pointwise convs. Conv weights and embeddings -default to F16; norms / biases / positional encodings stay F32. +Quantization applies to linear and pointwise-convolution weights. Other tensors +retain their supported floating-point formats. | `--outtype` | format | bytes/elem | use case | | --- | --- | --- | --- | @@ -109,47 +98,47 @@ default to F16; norms / biases / positional encodings stay F32. | `fp16` | F16 | 2.000 | Apple Silicon, older GPUs | | `q6_k` | Q6_K | 0.820 | smaller artifact, more quantization | | `q5_k` | Q5_K | 0.688 | smaller artifact, more quantization | -| `q4_k` | Q4_K | 0.562 | smallest listed artifact, most quantization | +| `q4_k` | Q4_K | 0.562 | compact K-quant | +| `nvfp4` | NVFP4 | 0.562 | FP4; native acceleration on supported Blackwell GPUs | +| `mxfp4` | MXFP4 | 0.531 | compact FP4; acceleration depends on the backend | `q8_0` is the portable default; pass `--outtype` to choose a different size/precision tradeoff. K-quants (`q4_k`/`q5_k`/`q6_k`) require inner dim divisible by 256; any tensor that fails -alignment falls back to F16 and is reported by the converter. +alignment falls back to F16 and is reported by the converter. NVFP4 and MXFP4 +require inner dim divisible by 64 and use the same fallback. Validate FP4 +accuracy and performance on the target model and backend before deployment. ### CUDA batching: planar Q8 layout -The converter's default Q8 layout is the portable block-interleaved format. The -CUDA backend in this project can instead store all encoder Q8 values and scales -in tensor-wide planes so high-concurrency FastConformer projections enter the -batched skinny-Q8 tensor-core path without a runtime repack: +The default Q8 layout is portable. For high-concurrency CUDA inference, the +converter can instead produce a planar Q8 layout: ```bash python3 convert_model.py model.nemo --outfile model.planar.q8_0.gguf \ --outtype q8_0 --q8-layout planar ``` -The layout flag covers ordinary encoder projections and fused attention QKV. -Planar Q8 is CUDA-only. Keep a block-layout artifact for other backends. +Planar Q8 is CUDA-only. Keep a default-layout artifact for other backends. ## Companion models (optional) -These are separate GGUFs the server loads alongside the ASR model - each is its -own file (own `general.architecture`), not bundled into the ASR GGUF, so they can -be swapped without re-converting the ASR model. Enable them at runtime via their -server flags; see [configuration](configuration.md). +These optional GGUFs are loaded alongside the ASR model and can be changed +without reconverting it. Enable them with their runtime options; see +[configuration](configuration.md). ### Silero VAD Used for [VAD feature masking](configuration.md#vad-feature-masking) and -VAD-driven [endpointing](configuration.md#endpointing). Converted from the public -Silero-VAD package (`general.architecture="vad"`): +VAD-driven [endpointing](configuration.md#endpointing). Convert from the public +Silero VAD package: ```bash pip install "silero-vad==6.2.0" python3 convert_model.py silero --outfile models/silero-v6.2.0.gguf # offline alternative, using an existing whisper.cpp Silero checkpoint: # python3 convert_model.py silero --outfile models/silero-v6.2.0.gguf \ -# --from-whisper-ggml /path/to/for-tests-silero-v6.2.0-ggml.bin +# --from-whisper-ggml /path/to/silero-v6.2.0-ggml.bin ``` Source: [snakers4/silero-vad](https://github.com/snakers4/silero-vad) (the pip @@ -157,12 +146,9 @@ package), or whisper.cpp's bundled checkpoint for the offline path. ### Sortformer speaker diarization -Used for word-level speaker tags (`WordInfo.speaker_tag`, requested via -`diarization_config.enable_speaker_diarization`) and standalone diarization -(`examples/diarize_file` over the `nemo_speech_diar_*` C ABI, streaming or `--offline`). -Converted -from the public streaming Sortformer v2 checkpoint -(`general.architecture="sortformer"`): +Used for ASR speaker tags and standalone `nemo-speech diarize`. Sortformer v2 +supports up to four speakers, with stateful streaming for long recordings and +full-attention inference for short recordings. Convert it with: ```bash python3 convert_model.py nvidia/diar_streaming_sortformer_4spk-v2 \ @@ -172,8 +158,8 @@ python3 convert_model.py nvidia/diar_streaming_sortformer_4spk-v2 \ Enable with `--diar-model models/sortformer-v2-f32.gguf`; streaming geometry comes from `--diar-preset` (see [configuration](configuration.md)). Segment -postprocessing defaults are NeMo's callhome-tuned values and are -dataset-sensitive. +postprocessing defaults follow the checkpoint and may need tuning for your +audio. Source: [nvidia/diar_streaming_sortformer_4spk-v2](https://huggingface.co/nvidia/diar_streaming_sortformer_4spk-v2). @@ -183,7 +169,7 @@ Used for [automatic punctuation](configuration.md#postprocessing-profanity-itn-p - restores casing and `. , ?` for models that emit lowercase unpunctuated text (e.g. Parakeet CTC). Use a compatible PnC GGUF, or convert a local NeMo BERT punctuation-and-capitalization `.nemo` checkpoint directly -(`general.architecture="pnc"`): +with `convert_model.py`: ```bash python3 convert_model.py pnc.nemo --outfile pnc-bert.q8_0.gguf --outtype q8_0 diff --git a/docs/build.md b/docs/build.md index ed50e98..e54e7c0 100644 --- a/docs/build.md +++ b/docs/build.md @@ -8,12 +8,14 @@ platform without a suitable release artifact. - Git; - CMake 3.26 or newer and Ninja; -- C and C++17 compilers (GCC 13 or newer on Linux); and -- the toolkit required by the selected GPU backend. +- a C compiler and a C++17-capable compiler compatible with the selected + backend and, when applicable, its toolkit; +- SentencePiece development files for ASR and diarization; and +- any toolkit required by the selected backend. -CUDA 12 and 13 are supported. gRPC builds additionally need compatible gRPC, -Protobuf compiler/runtime, and Abseil development packages from a mutually -compatible package set. +CUDA 12 and 13 are supported. Outside the Windows build driver, gRPC builds +additionally need compatible gRPC, Protobuf compiler/runtime, and Abseil +development packages from a mutually compatible package set. ### Install the basic tools @@ -21,20 +23,22 @@ Ubuntu/Debian: ```bash sudo apt-get update -sudo apt-get install -y build-essential cmake ninja-build git pkg-config +sudo apt-get install -y build-essential cmake ninja-build git pkg-config \ + libsentencepiece-dev ``` -Fedora/RHEL-family: +Fedora: ```bash -sudo dnf install -y gcc gcc-c++ cmake ninja-build git pkgconf-pkg-config +sudo dnf install -y gcc gcc-c++ cmake ninja-build git pkgconf-pkg-config \ + sentencepiece-devel ``` macOS with Homebrew: ```bash xcode-select --install # skip if the Command Line Tools are already installed -brew install cmake ninja +brew install cmake ninja sentencepiece ``` Windows from an elevated PowerShell with Chocolatey: @@ -48,6 +52,19 @@ that ship an older CMake require a newer package before configuration. CUDA, Vulkan, Metal, gRPC, and optional language frontend dependencies are installed only when selecting those features; see the platform sections below. +### Backend toolkits + +- CPU builds need no accelerator toolkit. +- CUDA builds need a supported CUDA 12 or 13 toolkit (including `nvcc`) at + build time and a compatible NVIDIA driver at run time. Set + `CMAKE_CUDA_ARCHITECTURES` when the output must run on GPUs other than the + build host. +- Vulkan builds need Vulkan headers and loader development files, `glslc`, and + SPIR-V headers at build time, plus a vendor Vulkan driver at run time. On + Ubuntu these are available as `libvulkan-dev`, `glslc`, and `spirv-headers`. +- Metal builds require macOS on Apple Silicon and the Xcode Command Line Tools; + no separate Metal SDK is needed. + ## Prepare the checkout Initialize the submodules needed by the selected components: @@ -57,10 +74,16 @@ git submodule update --init ggml git submodule update --init third_party/cpp-httplib # HTTP server only git submodule update --init llama.cpp # ASR live capture or NMT git submodule update --init proto/riva-common # gRPC only +git submodule update --init third_party/flashlight-text third_party/kenlm # Flashlight only +git submodule update --init third_party/open_jtalk # Japanese TTS only +git submodule update --init --recursive third_party/cppjieba # Mandarin TTS only ``` `scripts/configure.sh` checks required submodules before CMake runs. CUDA -presets also apply the pinned patches from `ggml-patches/` in order. +presets also apply the pinned patches from `ggml-patches/` in order. Mandarin +TTS also requires the Git LFS files under `src/tts/tokenizer/mandarin_data/`; +the helper reports any files that are still LFS pointers. Materialize them with +`git lfs pull --include='src/tts/tokenizer/mandarin_data/*'`. ## Configure and build @@ -83,13 +106,12 @@ landmarks are: | `metal-nmt` | Metal-enabled NMT build | | `vulkan-diar` | Vulkan standalone diarization build | | `-server` | ASR, diarization, NMT, TTS, HTTP API, realtime WebSocket, and playground | -| `cuda-full` | CUDA server plus normalization, Flashlight, and language frontends | +| `cuda-full` | CUDA server plus normalization, Flashlight, and language frontends (prepare the [optional dependencies](#optional-dependencies) first) | | `developer` | CPU speech components plus HTTP, gRPC, examples, tests, and tools | -The `-server` presets are used by the source installer and release -builders. They include NMT but not protobuf or gRPC. Use `cuda-full`, -`developer`, or explicit CMake options when the Riva-compatible adapters and -separate `riva_server` executable are needed. +The `-server` presets include NMT but not protobuf or gRPC. Use +`cuda-full`, `developer`, or explicit CMake options when the Riva-compatible +adapters and separate `riva_server` executable are needed. The preset selects which components and ggml backend are compiled. @@ -139,21 +161,39 @@ frontends are selected. ## Optional dependencies -The HTTP server uses the `third_party/cpp-httplib` submodule and OpenSSL for -optional TLS. CLI-only builds require neither dependency. +The HTTP server uses the `third_party/cpp-httplib` submodule and OpenSSL +development files for optional TLS (`libssl-dev` on Ubuntu/Debian). CLI-only +builds require neither dependency. + +On Ubuntu/Debian, the optional gRPC adapters can use one compatible system +package set: + +```bash +sudo apt-get install -y libgrpc++-dev libprotobuf-dev protobuf-compiler \ + protobuf-compiler-grpc libabsl-dev +``` Flashlight decoding requires the `third_party/flashlight-text` and `third_party/kenlm` submodules. KenLM is built as a replaceable dynamic library (`libkenlm` or `kenlm.dll`). -ITN and TN dependencies install into a project-local prefix without `sudo`: +Linux ITN/TN builds additionally require Autotools, Protobuf headers and +`protoc`, RE2 development files, and GCC 12. For example, on Ubuntu: + +```bash +sudo apt-get install -y autoconf automake bison libtool gcc-12 g++-12 \ + libprotobuf-dev protobuf-compiler libre2-dev +``` + +Build the pinned OpenFST and Sparrowhawk dependencies into the project-local +prefix without `sudo`: ```bash -scripts/build_itn_deps.sh +CC=gcc-12 CXX=g++-12 scripts/build_itn_deps.sh ``` -When combining normalization with Flashlight on Linux, build the private static -SentencePiece dependency as well: +On Linux, normalization builds also require the static SentencePiece +dependency: ```bash scripts/build_sentencepiece_static.sh @@ -197,7 +237,8 @@ notes. ## Containers `docker/Dockerfile` provides a minimal `runtime` target and a `builder` -target containing the toolchain and sources: +target containing the toolchain and sources. Put an ASR GGUF in +`$PWD/models` before starting the model-free runtime image: ```bash docker build --platform=linux/amd64 -f docker/Dockerfile --target runtime \ @@ -205,7 +246,8 @@ docker build --platform=linux/amd64 -f docker/Dockerfile --target runtime \ docker run --gpus all -p 8080:8080 \ -v "$PWD/models:/models:ro" nemo-speech-runtime:x86_64 \ - serve --host 0.0.0.0 --config /models/server.yaml + serve --host 0.0.0.0 \ + --asr-model /models/nemotron-3.5-asr-streaming-0.6b.q8_0.gguf ``` The runtime image entry point is `nemo-speech`; models remain outside the @@ -227,5 +269,5 @@ Selected artifacts are written under the configured build directory's `bin/`: | `transcribe_file`, `diarize_file` | Native ASR and diarization examples | | `translate_text`, `synthesize_text` | Native NMT and TTS examples | -Tests and developer tools are emitted only when their corresponding build -options are enabled. +Examples, tests, and developer tools are emitted only when their corresponding +build options are enabled. diff --git a/docs/cli.md b/docs/cli.md index 9cc0689..f4dcc09 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -41,7 +41,7 @@ The cache location is platform-specific: | Platform | Default cache | |---|---| | macOS | `~/Library/Caches/NeMoSpeech/models` | -| Linux | `${XDG_CACHE_HOME:-~/.cache}/nemo-speech/models` | +| Linux | `$XDG_CACHE_HOME/nemo-speech/models`, or `~/.cache/nemo-speech/models` when unset | | Windows | `%LOCALAPPDATA%\NeMoSpeech\models` | Set `NEMO_SPEECH_MODEL_DIR` to use another location. Passing an existing local @@ -56,8 +56,14 @@ Transcribe one WAV file: nemo-speech transcribe recording.wav nemo-speech transcribe recording.wav --model nemotron-en nemo-speech transcribe recording.wav --model ./models/asr.q8_0.gguf +nemo-speech transcribe recording.wav --stream ``` +File transcription submits the complete recording as one offline request by +default. Add `--stream` to feed a recorded WAV through the streaming recognizer +in 160 ms input chunks. `--live` also uses the streaming recognizer. +Offline-only models such as Parakeet TDT reject `--stream` and `--live`. + The file CLI accepts mono or stereo PCM16 and float32 WAV input from 8-96 kHz. It downmixes and resamples to the model rate. Unsupported containers or codecs produce an error with a conversion command. @@ -66,14 +72,16 @@ produce an error with a conversion command. ```bash nemo-speech transcribe --live \ - --backend auto + --backend auto \ + --endpointing ``` -The command captures the system's default microphone and prints interim and -endpointed transcripts to stderr while you speak. Press Ctrl-C once to stop; -the stream is flushed and the complete final transcript is written to stdout. -Use `--output transcript.txt` to write it to a file, or select `json`, `srt`, -or `vtt` with `--format`. +The command captures the system's default microphone and prints interim +transcripts to stderr while you speak. With `--endpointing`, trailing silence +also finalizes utterances without ending the capture. Press Ctrl-C once to +stop; the stream is flushed and the complete final transcript is written to +stdout. Use `--output transcript.txt` to write it to a file, or select `json`, +`srt`, or `vtt` with `--format`. Live capture is compiled directly into the CLI through miniaudio and uses the native host audio API: CoreAudio on macOS, WASAPI on Windows, and ALSA or @@ -86,18 +94,23 @@ shell running `nemo-speech`. ```bash nemo-speech transcribe recording.wav --format srt --output recording.srt nemo-speech transcribe recording.wav --format vtt --output recording.vtt -nemo-speech transcribe recording.wav --json --word-times +nemo-speech transcribe recording.wav --json ``` +JSON, SRT, and WebVTT output request word timestamps automatically. Plain-text +output remains transcript-only. `--word-times` is retained for compatibility +but does not change the rendered output of any current CLI format. + SRT and WebVTT cues prefer sentence, clause, and pause boundaries. Cues use up to two lines, target 37 characters per line, and allow small whole-word overflow within the common 42-character subtitle limit. Plain results are written to stdout. Progress and diagnostics are written to stderr so output can be redirected safely. Global `--json`, `--quiet`, and -`--verbose` options work across commands. The default output keeps command -lifecycle, effective inference configuration, results, warnings, and errors. -Model-loader, backend, ggml, and llama.cpp diagnostics require `--verbose`. +`--verbose` options work across commands. Inference and server commands show +their lifecycle, effective configuration, results, warnings, and errors by +default. Model-loader, backend, ggml, and llama.cpp diagnostics require +`--verbose`. ### Transcribe a directory @@ -120,7 +133,8 @@ one pass: ```bash nemo-speech transcribe meeting.wav \ --vad-model silero.gguf \ - --diar-model sortformer.gguf \ + --vad-masking \ + --diarize \ --pnc-model punctuation.gguf \ --itn-model-dir grammars/en-US \ --nmt-model translate.q8_0.gguf \ @@ -128,7 +142,20 @@ nemo-speech transcribe meeting.wav \ --json ``` -Use only the companion models needed by the workflow. +Loading a VAD model alone does not alter recognition; the example enables VAD +feature masking explicitly. `--diarize` downloads and uses the indexed default +Sortformer model, while `--diar-model MODEL` selects a different one and also +enables speaker tags. Use only the companion models needed by the workflow. + +For ASR plus speaker labels without the other stages: + +```bash +nemo-speech transcribe meeting.wav --diarize --json +``` + +Sortformer v2 supports up to four speakers. Diarization enables +word timestamps automatically and places a 1-based `speaker` value on each +word in JSON output. ## Diarize audio @@ -142,11 +169,17 @@ nemo-speech diarize recordings/ \ ``` Directory inputs load one shared model and dynamically batch compatible steps. -Relative paths are preserved. Streaming geometry is the default; use -`--offline` for full-attention processing of short recordings. Segmentation -thresholds are dataset-dependent; use `--onset`, `--offset`, `--pad-onset`, -`--pad-offset`, `--min-duration-on`, and `--min-duration-off` when applying a -checkpoint's published postprocessing configuration. +Relative paths are preserved. A stateful streaming pass is the default and is +appropriate for long recordings. `--preset offline` selects larger streaming +chunks and caches; it does not enable full attention. Use `--offline` for one +full-attention pass over a short recording. The indexed model's positional +table limits that path to about 6.6 minutes, so use the default streaming pass +for longer audio. + +Sortformer v2 supports up to four speakers. Segmentation thresholds +are dataset-dependent; use `--onset`, `--offset`, `--pad-onset`, `--pad-offset`, +`--min-duration-on`, and `--min-duration-off` when applying a checkpoint's +published postprocessing configuration. ## Translate text @@ -181,14 +214,18 @@ Run `nemo-speech doctor` to see the compiled backends and detected devices. ## Convert and inspect models -The built-in index covers the published ready-to-run GGUFs. Use the converter -when working with a custom checkpoint or producing a different quantization: +The built-in index covers the published ready-to-run GGUFs. From a source +checkout, use the Python converter when working with a custom checkpoint or +producing a different quantization: ```bash python convert_model.py custom-model.nemo --outfile custom-model.q8_0.gguf nemo-speech model info custom-model.q8_0.gguf ``` +Conversion tools are not included in the native binary archives; the installed +runtime itself does not require Python. + The converter can also resolve Hugging Face repository IDs through the standard cache. See [model conversion](model-conversion.md) for the isolated Python environment and supported model families. Custom files remain local; pass diff --git a/docs/clients.md b/docs/clients.md index b9eb69e..2ef84ff 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -3,14 +3,26 @@ Start a local server with the models needed by your application: ```bash -nemo-speech serve --asr-model models/asr.q8_0.gguf +nemo-speech serve --asr-model nemotron-3.5 ``` -The HTTP server implements the OpenAI audio API subset documented in the -[HTTP API reference](api.md). +Model listing, transcription, and speech expose the OpenAI-compatible subsets +documented in the [HTTP API reference](api.md). Other OpenAI APIs are not +implemented. Realtime transcription uses the project's WebSocket protocol +rather than the OpenAI Realtime API. An API key is only required when the server was started with `--api-key`; SDKs still require a nonempty placeholder locally. +OpenAI SDKs also require a `model` argument. NeMo-Speech.cpp currently loads +one model per capability, so this compatibility field does not switch models; +use `GET /v1/models` to inspect the active model IDs. + +For speech, use a local voice from the speech model's `voices` list. The +`default` and supported OpenAI voice aliases such as `alloy` select the +configured default local speaker; they do not select hosted OpenAI voices. +Local names are case-insensitive and can also be written as +`.` or as a zero-based speaker index. + ## OpenAI Python SDK ```python @@ -44,13 +56,17 @@ than placing an API key in a public page. ## curl +The speech example requires a TTS model. Start a TTS-only server with +`nemo-speech serve --tts-model magpie`, or add `--tts-model magpie` to the ASR +server command above. + ```bash curl -s http://127.0.0.1:8080/v1/audio/transcriptions \ -F file=@recording.wav -F model=default -F response_format=verbose_json curl -s http://127.0.0.1:8080/v1/audio/speech \ -H 'Content-Type: application/json' \ - -d '{"model":"default","voice":"default","input":"Hello","response_format":"wav"}' \ + -d '{"model":"default","voice":"alloy","input":"Hello","response_format":"wav"}' \ -o hello.wav ``` diff --git a/docs/development/asr-batching.md b/docs/development/asr-batching.md index 4809d36..a6b9141 100644 --- a/docs/development/asr-batching.md +++ b/docs/development/asr-batching.md @@ -62,6 +62,7 @@ asr: max_queue_depth: 2048 ingress_cohort_delay_us: 20000 state_arena_slots: 32 + offline_bucket_ms: 1000 ``` | Key | Tuning effect | @@ -72,10 +73,14 @@ asr: | `max_queue_depth` | Bounds pending work and provides backpressure. | | `ingress_cohort_delay_us` | Aligns streaming audio arrivals before frontend and encoder work. | | `state_arena_slots` | Reserves recurrent/cache state rows; provision at least the maximum concurrent stateful streams. | +| `offline_bucket_ms` | Silence-pads offline utterances to a duration multiple so similar lengths can share graph shapes; `0` disables bucketing. | More streams than `max_batch_size` are processed in multiple waves. Increasing the cap or either delay does not guarantee better throughput; tune them on the target GPU with the expected request cadence and audio chunk size. +Offline bucketing is useful only for concurrent offline workloads, and its +padding cost grows with the bucket size. Start with a modest value such as +`1000` ms and measure it on the expected duration distribution. The gRPC and HTTP streaming adapters opt into ingress coordination. Direct library streams and benchmark calls do not, so they avoid the transport diff --git a/docs/development/cublas-shim.md b/docs/development/cublas-shim.md index b37f623..4e6a02a 100644 --- a/docs/development/cublas-shim.md +++ b/docs/development/cublas-shim.md @@ -14,19 +14,25 @@ the same ABI from `cublas64_.dll`. The target inherits `CMAKE_CUDA_ARCHITECTURES` when set and falls back to JIT-portable `compute_80` PTX for ad-hoc builds. Dropping real cuBLAS and cuBLASLt is the bulk of the package size. The shim is built separately from ggml. +Prebuilt releases include Turing (SM75) code. Local Turing builds must set +`CMAKE_CUDA_ARCHITECTURES=75` or `native`, because `compute_80` PTX cannot run +on SM75. It's an optional CMake target, **`NEMO_SPEECH_CUBLAS_SHIM` (default `OFF`)**, built when explicitly enabled with `GGML_CUDA` (a no-op for Metal, Vulkan, -and CPU builds). Normal source builds link the CUDA toolkit's cuBLAS and -cuBLASLt. Portable container and release-archive builds enable the shim and -omit those libraries from their runtime closure. Linux uses a matching SONAME -and symbol version; Windows uses the matching versioned DLL name. +and CPU builds). Normal shared-library source builds link the CUDA toolkit's +cuBLAS; static ggml builds may also link cuBLASLt. Portable container and +release-archive builds enable the shim and do not require those libraries at +run time. Linux uses a matching SONAME and symbol version; Windows uses the +matching versioned DLL name. To build and exercise the container GEMM path outside the container, enable the shim and put its output directory first on the loader path: ```bash -scripts/configure.sh cuda-asr -DNEMO_SPEECH_CUBLAS_SHIM=ON +scripts/configure.sh cuda-asr \ + -DNEMO_SPEECH_CUBLAS_SHIM=ON \ + -DCMAKE_CUDA_ARCHITECTURES=native cmake --build --preset cuda-asr LD_LIBRARY_PATH=$PWD/build/cuda-asr/bin \ ./build/cuda-asr/bin/nemo-speech transcribe audio.wav --model model.gguf diff --git a/docs/development/diagnostics.md b/docs/development/diagnostics.md index 536e5ac..d4c378a 100644 --- a/docs/development/diagnostics.md +++ b/docs/development/diagnostics.md @@ -1,10 +1,13 @@ # Backend coverage diagnostic -`check_backend_coverage` loads an ASR GGUF, runs one inference step through every -Session (encoder, RNNT predictor + joint, cache-aware encoder), and prints the -per-op backend assignment. Use it to catch **silent CPU fallbacks** when -enabling a new backend (Vulkan / Metal / CPU) - a single fallback op -mid-graph is a GPU↔CPU roundtrip per audio chunk, which kills streaming latency. +`check_backend_coverage` loads an ASR GGUF and exercises the frontend and +encoder Sessions used by its CTC or streaming-transducer path, including the +compact CTC head, RNNT/TDT predictor and joint, and cache-aware encoder when +applicable. It then prints their per-op backend assignment. +Use it to catch **silent CPU fallbacks** when enabling a new GPU backend - a +single fallback op mid-graph adds a GPU↔CPU roundtrip per audio chunk and can +significantly increase streaming latency. The lazy offline transducer path is +outside this diagnostic's coverage. ```bash scripts/configure.sh cuda-asr -DNEMO_SPEECH_BUILD_TOOLS=ON @@ -14,6 +17,9 @@ build/cuda-asr/bin/check_backend_coverage \ # --gpu N GPU device index (default 0). -1 forces CPU. ``` +Append `--diar diar_streaming_sortformer_4spk-v2.q8_0.gguf` to exercise the +optional Sortformer Session in the same run. + Sample output: ``` diff --git a/docs/development/windows-build.md b/docs/development/windows-build.md index a7435de..8993ca9 100644 --- a/docs/development/windows-build.md +++ b/docs/development/windows-build.md @@ -1,4 +1,4 @@ -# Building on Windows (CUDA/Vulkan) +# Building on Windows Native Windows build with **MSVC + Ninja**, covering the CUDA, Vulkan, and CPU backends plus the optional Riva-compatible gRPC server. For other platforms, @@ -6,30 +6,17 @@ see [Build from source](../build.md). ## Toolchain -One policy across Windows architectures. Visual Studio 2022 Build Tools are -required in every configuration (`cl.exe` is `nvcc`'s only supported CUDA host -compiler on Windows). +Visual Studio 2022 Build Tools are required in every configuration (`cl.exe` is +`nvcc`'s supported CUDA host compiler on Windows). | Host arch | C/C++ compiler | CUDA host compiler | |---|---|---| | x64 | `cl` (default; `clang-cl` selectable) | `cl` | | ARM64 (e.g. Tegra) | `clang-cl` (required - ggml's ARM CPU backend rejects MSVC) | `cl` | -`build.ps1` picks this automatically (`-Compiler auto`); override with -`-Compiler msvc|clang-cl`. Notes: - -- `clang-cl` targets the same MSVC ABI (same STL/CRT/linker), so the Windows - handling in the tree (DLL export via `WINDOWS_EXPORT_ALL_SYMBOLS`, no POSIX - APIs) applies to both compilers. Install it via the VS "C++ Clang tools" - component or `choco install llvm`. MinGW (gcc/clang in GNU mode) is not - supported - it cannot drive `nvcc`. -- Warning flags are scoped to C/C++ compilation - (`$`) so nvcc's `cl` host never receives them - with - clang-cl as CXX they would otherwise leak into the CUDA compile (`D8021`). -- **ARM64 specifics.** CUDA 13.4+/CCCL 3.4 requires the `cuda/iterator` include - fix present in the current ggml pin. On integrated GPUs (e.g. Tegra), Vulkan - devices enumerate as iGPU (`GGML_BACKEND_DEVICE_TYPE_IGPU`), which the ASR - backend picker accepts. +`build.ps1` selects the compiler with `-Compiler auto`; override it with +`-Compiler msvc|clang-cl`. `clang-cl` uses the MSVC ABI. MinGW is not supported +and cannot drive `nvcc`. ## Prerequisites @@ -64,19 +51,16 @@ defaults. ```powershell git submodule update --init ggml # required (all backends) git submodule update --init proto/riva-common # gRPC server -git submodule update --init llama.cpp # NMT (-DNEMO_SPEECH_BUILD_NMT=ON) +git submodule update --init llama.cpp # ASR live capture or NMT git submodule update --init third_party/flashlight-text third_party/kenlm # only for LM-fused CTC decoding git submodule update --init third_party/open_jtalk # optional TTS JA tokenizer (-TtsJa) git submodule update --init --recursive third_party/cppjieba # optional TTS ZH tokenizer (-TtsZh) # or: git submodule update --init --recursive ``` -The JA/ZH TTS tokenizers are gated by `NEMO_SPEECH_TTS_WITH_JA` / -`NEMO_SPEECH_TTS_WITH_ZH` (both default OFF). JA builds Open JTalk/MeCab from -the `third_party/open_jtalk` submodule and compiles its dictionary during the build -(`build-\open_jtalk_dic`); ZH uses the header-only `third_party/cppjieba` submodule -(recursive - it nests `limonp`). Pass `-TtsJa` / `-TtsZh` to `build.ps1`, or -enable the corresponding CMake options directly. +The JA/ZH TTS tokenizers are disabled by default. Enable them with `-TtsJa` or +`-TtsZh`; the Mandarin dependency must be initialized recursively as shown +above. ## Build with `build.ps1` (recommended) @@ -116,7 +100,7 @@ Key parameters: `-Backend cuda|vulkan|cpu`, `-Architecture auto|x64|arm64`, `-Grpc`, `-Nmt`, `-AsrOnly`, `-Http`, `-HttpTls`, `-Flashlight`, `-TtsJa`, `-TtsZh`, -`-Config Release|RelWithDebInfo|Debug`, `-CudaArch `, +`-Config Release|RelWithDebInfo|Debug`, `-CudaArch `, `-VcpkgRoot C:\vcpkg`, `-VcpkgTriplet `, `-BuildDir `, `-Jobs N`. Binaries land in `build-[-][-]\bin`; the default `core` and `auto` suffixes are omitted. @@ -154,29 +138,15 @@ cmake --build build-vulkan --parallel app-local `cublas64_.dll` that avoids shipping cuBLAS and cuBLASLt. - **ggml patches are CUDA-only.** A Vulkan/CPU build uses stock ggml; pass `-DNEMO_SPEECH_GGML_PATCHED=OFF` (the encoder uses the portable op path). -- DLLs export their symbols via `WINDOWS_EXPORT_ALL_SYMBOLS` (the C ABI libs use - `__declspec(dllexport)`); at runtime, dependent DLLs must be next to the `.exe` - or on `PATH` (Ninja places them together in `build-\bin`). -- Flashlight builds the replaceable `kenlm.dll` from a runtime-only source - allowlist. Flashlight and its static-md vcpkg dependencies remain private in - the ASR DLL. -- **`NOMINMAX` is defined globally on Windows** so `` (pulled in by the - CUDA headers when `GGML_CUDA=ON`) doesn't clobber `std::min`/`std::max`. -- **ggml patch files are forced to LF** via `.gitattributes`. On a CRLF checkout, - `git apply` rejects some hunks as "corrupt patch"; the Windows patch script - (`scripts/windows/apply-ggml-patches.ps1`) also strips all CR bytes defensively. -- **Open JTalk/MeCab** (TTS tokenizer, vendored via the `third_party/open_jtalk` submodule) - builds on MSVC with adjusted flags in `src/tts/tokenizer/CMakeLists.txt`: the - POSIX `HAVE_*` defines are swapped for `HAVE_WINDOWS_H` (mecab's own Win32 - mmap/dirent paths), and `` is force-included with - `_HAS_AUTO_PTR_ETC=1` so `std::binary_function` (removed in C++17) resolves. - No vendored sources are modified. - -### After pulling new changes - -If `apply-ggml-patches` reports a patch "does NOT apply cleanly" after a pull -(commonly a mixed line-ending or half-applied state in the ggml worktree), reset -the submodule to pristine and re-apply: +- Dependent DLLs must be next to the executable or on `PATH`. Ninja places them + together in `build-\bin`. +- Flashlight builds install the replaceable `kenlm.dll` alongside the runtime + libraries. + +### Reset a partially patched ggml checkout + +If `apply-ggml-patches` reports that a patch does not apply cleanly, reset the +submodule and re-apply it: ```powershell git -C ggml reset -q @@ -193,23 +163,15 @@ powershell -ExecutionPolicy Bypass -File scripts\windows\apply-ggml-patches.ps1 | **CUDA** | ✅ Supported | ✅ Supported | ✅ Supported | | **Vulkan** | ✅ Supported | ✅ Supported | ✅ Supported | -NMT runs through llama.cpp linked against the in-tree patched ggml, so it inherits the build's -backend. TTS uses generic ggml graphs on Vulkan; CUDA builds additionally use -specialized sampling and local-transformer kernels. - Use the unified `nemo-speech` CLI for local ASR, NMT, and TTS commands; see the [CLI guide](../cli.md). Stock Riva clients work against `riva_server` when the build includes `-Grpc`. A CPU-only build selects the CPU automatically, or you can pass `--device cpu` explicitly. -**Vulkan graph-optimization workaround.** ggml-vulkan's graph-optimization pass -reorders graph nodes in a way that breaks this runtime's **in-place persistent -cache tensors** (the streaming FastConformer/RNNT K/V/conv cache): the decode -degenerates into a single repeated token after the first chunk. The runtime -auto-sets `GGML_VK_DISABLE_GRAPH_OPTIMIZE` for Vulkan builds -(`src/runtime/ggml/backend.cpp`), which fixes it at a small Vulkan-perf cost. -This is a platform-independent ggml-vulkan issue, not a Windows one. An explicit -user-set value of the env var takes precedence over the auto-set. +**Vulkan graph optimization.** The runtime disables ggml-vulkan graph +optimization because it is incompatible with the persistent caches used by +streaming ASR. An explicit user value for `GGML_VK_DISABLE_GRAPH_OPTIMIZE` +takes precedence; enabling the pass can produce incorrect streaming output. ### Optional features (flashlight, ITN) @@ -218,9 +180,6 @@ user-set value of the env var takes precedence over the auto-set. | **Flashlight** (`-DNEMO_SPEECH_WITH_FLASHLIGHT=ON`) | ✅ Builds replaceable `kenlm.dll`; SentencePiece and compression libraries are provisioned automatically. | | **ITN/TN** (`-DNEMO_SPEECH_WITH_NORM=ON`) | ❌ Not supported on Windows. Requires the OpenFST 1.8 / Sparrowhawk WFST stack, which `scripts/build_itn_deps.sh` builds via Linux autotools (neither is in vcpkg). | -Automatic dependencies use one architecture-matched `*-windows-static-md` -triplet. - ## Next steps Model conversion and runtime commands are platform-neutral. Continue with: diff --git a/docs/install.md b/docs/install.md index 17711fa..70af9b8 100644 --- a/docs/install.md +++ b/docs/install.md @@ -10,7 +10,9 @@ cache](cli.md#models-and-cache). ## Linux and macOS -Inspect [`scripts/install.sh`](../scripts/install.sh), then run: +Inspect +[`scripts/install.sh`](https://github.com/NVIDIA/NeMo-Speech.cpp/blob/main/scripts/install.sh), +then run: ```bash curl -fsSL https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.sh | sh @@ -22,6 +24,13 @@ With no version argument, the installer reads the current release identifier from the repository's `VERSION` file, including prerelease identifiers. Native Linux archives require glibc 2.31 or newer (Ubuntu 20.04 or equivalent). +Prebuilt CPU archives require no GPU toolkit. Linux CUDA archives include the +required user-space CUDA libraries but still need a compatible NVIDIA driver. +Vulkan archives use the host's Vulkan loader and vendor driver. The Linux +x86_64 CUDA archive supports Turing-class GPUs (compute capability 7.5, +including RTX 20-series) and newer. On an older GPU, select `--backend cpu` or +`--backend vulkan`, or build from source with a compatible CUDA toolkit. + The installer selects CUDA when `nvidia-smi` is available, Metal on Apple Silicon, and CPU otherwise. Override the backend or force a source build: @@ -43,11 +52,15 @@ verified against their published SHA-256 files; a present archive with an invalid or mismatched checksum always fails rather than falling back to source. The source fallback requires Git, CMake 3.26 or newer, Ninja, a C++17 compiler, -and the toolkit for the selected GPU backend. It clones only the submodules -needed by the CLI and playground. When run from a checkout without an -explicit version, it builds that checkout's current branch. Override the source -for a fork or local mirror with `NEMO_SPEECH_SOURCE_URL` and -`NEMO_SPEECH_SOURCE_REF`. +SentencePiece development files, and any toolkit required by the selected +backend. On Ubuntu/Debian install `libsentencepiece-dev`; on Fedora install +`sentencepiece-devel`; on macOS install `sentencepiece` with Homebrew. + +Source installs use `main` by default. To install a fork or the current +checkout, set `NEMO_SPEECH_SOURCE_URL` to its URL or path (`$PWD` on Linux or +macOS, or `(Get-Location).Path` in PowerShell). Set +`NEMO_SPEECH_SOURCE_REF` to select another branch or tag. Local installs clone +the committed branch state and do not include uncommitted changes. Automatic model pulls require the `curl` executable. The Linux/macOS installer also uses it for release downloads; the Windows installer uses PowerShell's @@ -58,28 +71,36 @@ paths and already cached models still work if `curl` later becomes unavailable. ## Windows -Inspect [`scripts/install.ps1`](../scripts/install.ps1), then run from -PowerShell: +Inspect +[`scripts/install.ps1`](https://github.com/NVIDIA/NeMo-Speech.cpp/blob/main/scripts/install.ps1), +then run from PowerShell: ```powershell irm https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.ps1 | iex -nemo-speech --version ``` +The installer updates the current user's `PATH`. Open a new PowerShell window, +then run `nemo-speech --version`. + Select a backend explicitly when needed: ```powershell -.\scripts\install.ps1 -Source -Backend cuda +irm https://github.com/NVIDIA/NeMo-Speech.cpp/raw/main/scripts/install.ps1 ` + -OutFile .\install-nemo-speech.ps1 +powershell -ExecutionPolicy Bypass -File .\install-nemo-speech.ps1 ` + -Source -Backend cuda ``` Select the components to install: ```powershell # ASR and diarization only -.\scripts\install.ps1 -Source -Backend cpu -Profile asr +powershell -ExecutionPolicy Bypass -File .\install-nemo-speech.ps1 ` + -Source -Backend cpu -Profile asr # Full runtime profile (add -HttpTls for TLS) -.\scripts\install.ps1 -Source -Backend cuda -Profile full +powershell -ExecutionPolicy Bypass -File .\install-nemo-speech.ps1 ` + -Source -Backend cuda -Profile full ``` | Profile | Components | @@ -135,5 +156,5 @@ entry from the shell startup file if the installer added it. On Windows, remove `%LOCALAPPDATA%\Programs\NeMoSpeech` (or the selected prefix) and that prefix's `bin` directory from the current-user PATH. The model cache is stored separately and is not removed with the runtime: `~/Library/Caches/NeMoSpeech/models` -on macOS, `${XDG_CACHE_HOME:-~/.cache}/nemo-speech/models` on Linux, and +on macOS, `${XDG_CACHE_HOME:-$HOME/.cache}/nemo-speech/models` on Linux, and `%LOCALAPPDATA%\NeMoSpeech\models` on Windows. diff --git a/docs/model-conversion.md b/docs/model-conversion.md index d40ea87..dc52dcd 100644 --- a/docs/model-conversion.md +++ b/docs/model-conversion.md @@ -3,15 +3,22 @@ Published NeMo-Speech.cpp models provide ready-to-run GGUF files on their Hugging Face pages; see the [ASR](asr/models.md) and [TTS](tts/models.md) model guides. Use the converter for compatible custom checkpoints, alternate -quantization choices, and optional sidecar models that do not publish a GGUF. +quantization choices, and supporting models that do not publish a GGUF. -All model families use the same root entry point: +The converters are Python source tools and are not included in the native +release archives. Run them from a source checkout in a virtual environment; +the C++ runtime itself does not require Python. All model families use the same +root entry point: ```bash -pip install -r requirements.txt -python3 convert_model.py SOURCE --outfile MODEL.gguf +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -r requirements.txt +python convert_model.py SOURCE --outfile MODEL.gguf ``` +On Windows PowerShell, activate with `.\.venv\Scripts\Activate.ps1`. + `SOURCE` may be a local `.nemo` archive, an extracted NeMo checkpoint, a local Hugging Face model directory, or a Hugging Face repository ID. For NeMo model repositories, the converter downloads only the `.nemo` checkpoint through the @@ -46,7 +53,7 @@ install it only when converting an NMT model: ```bash git submodule update --init llama.cpp -pip install -r llama.cpp/requirements/requirements-convert_hf_to_gguf.txt +python -m pip install -r llama.cpp/requirements/requirements-convert_hf_to_gguf.txt python3 convert_model.py nvidia/Riva-Translate-4B-Instruct-v2 \ --outfile models/translate.q8_0.gguf --outtype q8_0 ``` diff --git a/docs/nmt/configuration.md b/docs/nmt/configuration.md index 9555a84..123f703 100644 --- a/docs/nmt/configuration.md +++ b/docs/nmt/configuration.md @@ -41,13 +41,13 @@ NMT auto-enables when `nmt.model.path` is set; force with `nmt.enabled`. | `nmt.model.n_ctx` | `1024` | decode context length in tokens (model max `8192`) | | `nmt.generation.max_new_tokens` | `256` | cap per input text | | `nmt.pool.contexts` | `1` | concurrent decode contexts (one request per context) | -| `nmt.verbose` | `false` | enable verbose llama.cpp loader/runtime logs | +| `nmt.verbose` | `false` | verbose llama.cpp logs for direct/gRPC use; use global `--verbose` with `nemo-speech` | ## Memory and concurrency Each decode context holds one KV cache sized by `n_ctx` (about `0.13 MiB/token`, so `~136 MiB` at the default `1024`). The defaults target the common case: -sentence-level translation, one request per context (`bs=1`, like ASR/TTS). +sentence-level translation with one request per context. - **Longer inputs:** raise `nmt.model.n_ctx` (up to the model's `8192`). The model translates a sentence/short paragraph well but degrades on inputs far @@ -60,24 +60,22 @@ sentence-level translation, one request per context (`bs=1`, like ASR/TTS). ## RPCs -The optional `riva_server` adapter implements `RivaTranslation.TranslateText` -and `ListSupportedLanguagePairs`. +The optional `riva_server` supports `RivaTranslation.TranslateText` and +`ListSupportedLanguagePairs`. `TranslateText` takes a batch of `texts` plus `source_language` and `target_language` and returns one `Translation` per input. The pair is resolved to the model's tag (`en-de`, `en-zh-cn`, ...); pass the two codes -(`source_language: en`, `target_language: de`) or the full tag in either field. -Unsupported pairs return `INVALID_ARGUMENT`. The server wraps each text in the -model's chat-style translation prompt (System/User turns naming the source and -target languages), so the client sends plain text only. +(`source_language: en`, `target_language: de`) or put the full tag in either +field and leave the other empty. Every supported pair has English on one side; +unsupported pairs return `INVALID_ARGUMENT`. Send plain text. `ListSupportedLanguagePairs` returns the supported `source -> target` pairs keyed by model name. -When ASR is loaded, the same adapter also exposes -`StreamingTranslateSpeechToText`; loading TTS enables -`StreamingTranslateSpeechToSpeech`. Both use the shared `SpeechTranslator` -composition rather than separate transport-owned inference paths. +When ASR is loaded, `riva_server` also exposes `StreamingTranslateSpeechToText`; +loading TTS enables +`StreamingTranslateSpeechToSpeech`. See [Client integration](../clients.md) for HTTP and Riva-compatible gRPC examples. diff --git a/docs/nmt/models.md b/docs/nmt/models.md index cbd2d0e..f20c614 100644 --- a/docs/nmt/models.md +++ b/docs/nmt/models.md @@ -8,11 +8,13 @@ pinned llama.cpp converter. Hugging Face: [nvidia/Riva-Translate-4B-Instruct-v2](https://huggingface.co/nvidia/Riva-Translate-4B-Instruct-v2) Unlike the NeMo model converters, NMT conversion requires the pinned llama.cpp -submodule and its additional Python dependencies: +submodule and its additional Python dependencies. Conversion is run from a +source checkout; the Python tools are not included in native release archives. +See [Model conversion](../model-conversion.md) for the base environment setup. ```bash git submodule update --init llama.cpp -pip install -r llama.cpp/requirements/requirements-convert_hf_to_gguf.txt +python3 -m pip install -r llama.cpp/requirements/requirements-convert_hf_to_gguf.txt python3 convert_model.py nvidia/Riva-Translate-4B-Instruct-v2 \ --outfile riva-translate-4b-instruct-v2.q8_0.gguf --outtype q8_0 ``` @@ -24,11 +26,6 @@ python3 convert_model.py nvidia/Riva-Translate-4B-Instruct-v2 \ - The underlying llama.cpp converter may print a `fix_mistral_regex` warning while reading the Hugging Face tokenizer. It is benign for this model. -## llama.cpp and ggml compatibility - -The build reuses the root `ggml` target instead of compiling llama.cpp's -bundled copy, so ASR, TTS, and NMT share one backend at runtime. - ## Precision The converter accepts `f32`, `f16`, `bf16`, and `q8_0`; f16 and q8_0 are the diff --git a/docs/server.md b/docs/server.md index 2baa2bc..b99f959 100644 --- a/docs/server.md +++ b/docs/server.md @@ -3,14 +3,14 @@ The project provides two server executables over the same core C++ engines: - `nemo-speech serve` hosts HTTP, realtime WebSocket, and the browser - playground. It loads configured models once into an `EngineRegistry`. + playground. It loads each configured model once. - `riva_server` hosts the Riva-compatible gRPC services. -They are separate processes and do not share loaded model instances. The source -installer and `*-server` presets build the HTTP executable for ASR, diarization, -NMT, and TTS without the gRPC dependency chain. `cuda-full`, `developer`, or -explicit component options add optional language frontends and `riva_server` -(presets: [build guide](build.md)). +They are separate processes and do not share loaded models. The `*-server` +presets include HTTP support for ASR, diarization, NMT, and TTS without gRPC. +`cuda-full` adds gRPC, text normalization, and optional TTS language frontends; +`developer` also adds examples, tests, and tools. Individual features can be +selected explicitly (presets: [build guide](build.md)). ```bash nemo-speech serve \ @@ -23,6 +23,9 @@ nemo-speech serve \ # Open the playground after the listener is ready. nemo-speech serve --asr-model models/asr.q8_0.gguf --open +# Serve standalone speaker diarization (downloads the indexed model if needed). +nemo-speech serve --diar-model sortformer + # Start the separate Riva-compatible gRPC server. riva_server --asr.model.path models/asr.q8_0.gguf --bind 0.0.0.0:50051 ``` @@ -31,6 +34,8 @@ riva_server --asr.model.path models/asr.q8_0.gguf --bind 0.0.0.0:50051 Both servers accept the same dotted engine keys, such as `asr.vad.masker.onset`, through YAML, environment variables, or CLI options. +The HTTP server additionally accepts top-level `diar.*` keys for standalone +diarization; use `asr.diar.*` to add speaker labels to ASR results. Settings are applied in this order: ```text @@ -45,7 +50,7 @@ Nested YAML maps mirror the dotted keys. Start from a checked-in example: | [`config/diar.example.yaml`](../config/diar.example.yaml) | standalone diarization | | [`config/tts.example.yaml`](../config/tts.example.yaml) | TTS-only server | | [`config/nmt.example.yaml`](../config/nmt.example.yaml) | NMT-only server | -| [`config/server.example.yaml`](../config/server.example.yaml) | combined HTTP speech server | +| [`config/server.example.yaml`](../config/server.example.yaml) | combined ASR, diarization, NMT, and TTS server | ```bash nemo-speech serve --config config/asr.example.yaml @@ -76,7 +81,7 @@ HTTP listener settings use the same precedence: | `--host` / `http.host` | `127.0.0.1` | HTTP bind address | | `--port` / `http.port` | `8080` | HTTP port | | `--api-key` / `http.api-key` | none | require `Authorization: Bearer ` on API routes | -| `--tls-cert` / `http.tls-cert` | none | TLS certificate path (with `--tls-key`, enables HTTPS) | +| `--tls-cert` / `http.tls-cert` | none | TLS certificate path (with `--tls-key`; requires a build with `NEMO_SPEECH_HTTP_TLS=ON`) | | `--tls-key` / `http.tls-key` | none | TLS private-key path | | `--cors-origin` / `http.cors-origin` | none | allowed browser origin | | `--no-ui` / `http.playground` | playground on | disable the embedded playground | @@ -99,12 +104,14 @@ The complete engine key references are in [ASR configuration](asr/configuration. [NMT configuration](nmt/configuration.md). The default loopback binding is intentional. For remote access, set -`--host 0.0.0.0`, TLS (`--tls-cert` + `--tls-key`), and an API key explicitly. Prefer the -`NEMO_SPEECH_HTTP_API_KEY` environment variable over placing a secret in -command-line arguments. API keys require `Authorization: Bearer `; -browser WebSockets may supply `?api_key=`. NVIDIA NIM is the supported -production deployment path; this server is intended for local use and direct -integration. +`--host 0.0.0.0`, TLS (`--tls-cert` + `--tls-key`), and an API key explicitly. +TLS must be enabled at source-build time with `NEMO_SPEECH_HTTP_TLS=ON` and is +not part of the default server presets. Prefer the `NEMO_SPEECH_HTTP_API_KEY` +environment variable over placing a secret in command-line arguments. API +routes require `Authorization: Bearer `; browser WebSockets may supply +`?api_key=`. The playground, health, readiness, and version routes remain +unauthenticated. NVIDIA NIM is the supported production deployment path; this +server is intended for local use and direct integration. Cross-origin browser access is disabled by default. `--cors-origin ORIGIN` allows one explicit origin; use `*` only for an intentionally public API. @@ -124,10 +131,10 @@ nemo-speech --json serve --access-log --asr-model models/asr.q8_0.gguf ## Health and readiness -`GET /health` returns the engine status and runtime version. `GET /ready` +`GET /health` returns a compact engine status and runtime version. `GET /ready` returns readiness, selected device, and loaded capabilities. Both return HTTP -503 until the configured engines are ready. The CLI can check either endpoint; -it uses `/ready` by default: +503 when no engine is ready. The CLI can check either endpoint; it uses +`/ready` by default: ```bash nemo-speech health --url http://127.0.0.1:8080/ready @@ -138,7 +145,8 @@ nemo-speech health --url http://127.0.0.1:8080/ready Full request/response field reference: **[HTTP API reference](api.md)**. - `GET /`, `GET /health`, `GET /ready`, and `GET /version` -- `GET /v1/models` +- `GET /v1/models` - loaded model inventory (OpenAI SDK-compatible, with + capability metadata) - `POST /v1/audio/transcriptions` - speech-to-text (OpenAI-compatible subset) - `POST /v1/audio/speech` - text-to-speech (OpenAI-compatible subset) - `POST /v1/translations` - text translation @@ -147,6 +155,10 @@ Full request/response field reference: **[HTTP API reference](api.md)**. - `POST /v1/audio/diarizations` - speaker segments (`/v1/diarizations` alias) - WebSocket `/v1/realtime` - live PCM16 transcription +OpenAI SDK compatibility is limited to model listing and the documented +transcription and speech subsets. The translation and diarization routes are +project extensions, and the realtime socket is not the OpenAI Realtime API. + The realtime socket accepts binary little-endian PCM16 frames; an optional `session.update` JSON event before the first frame sets session options. See the [API reference](api.md#websocket-v1realtime) for session fields and the @@ -168,10 +180,10 @@ Uploads are capped at 512 MiB by default (`--max-upload-mb`); the same limit applies to cumulative audio on a realtime WebSocket stream. Socket reads and writes time out after 30 seconds by default (`--read-timeout` and `--write-timeout`); inference work runs on a bounded worker pool (`--threads`). -The NMT context pool remains an engine setting (`nmt.pool.contexts`) and is not -silently expanded to match HTTP workers. SIGINT/SIGTERM stops HTTP admission and -releases its loaded models. `--no-warmup` is available for HTTP diagnostics but -is not recommended when startup readiness matters. +`--threads` does not change the NMT context pool (`nmt.pool.contexts`). +SIGINT/SIGTERM stops HTTP admission and releases loaded models. `--no-warmup` +is available for diagnostics but is not recommended when startup readiness +matters. The separate `riva_server` accepts messages up to gRPC's signed 32-bit limit and drains active RPCs for up to 10 seconds on SIGINT/SIGTERM. It currently uses diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ad548ad..9f4237d 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -29,10 +29,11 @@ runtime version, compiled capabilities, backend devices, and driver status. | Symptom | Check | Resolution | |---|---|---| | Command is absent | `nemo-speech --help`; `doctor` features | Install/build an archive containing that component. | -| No suitable model | `model info FILE` | Download the model and pass its local GGUF path. | +| No suitable model | `model list`; `model info FILE` | Use an indexed model name, run `model pull NAME`, or pass a local GGUF path. | +| Automatic model download is unavailable | `doctor --json` `model_download` field; `curl --version` | Install `curl` and ensure it is on `PATH`, or use an existing local model. | | Missing companion model | command error | Download the reported component and pass it explicitly or set it in YAML. | | GPU requested but unavailable | `doctor --json` devices | Install the matching backend build/driver or use `--device cpu`. | -| Server is live but not ready | `/ready`; server stderr | Correct the reported model/component path or compatibility error. | +| Server exits before listening | server stderr | Correct the reported model/component path or compatibility error. | | HTTP 401 | request `Authorization` header | Send `Authorization: Bearer $NEMO_SPEECH_HTTP_API_KEY`. | | HTTP 413 | upload size | Raise `--max-upload-mb` intentionally or split the input. | | gRPC UNAVAILABLE | `riva_server` listener and port | Start `riva_server --bind 0.0.0.0:50051`, verify binding/firewall, and use plaintext unless TLS is terminated externally. | diff --git a/docs/tts/configuration.md b/docs/tts/configuration.md index 2496706..17839ad 100644 --- a/docs/tts/configuration.md +++ b/docs/tts/configuration.md @@ -21,7 +21,7 @@ nemo-speech serve \ --tts.codec-model models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ --tts.tokenizer-model-dir models/magpie-tts/extracted \ --host 127.0.0.1 --port 8080 \ - --tts.language-code en-US --tts.voice-name John --tts.benchmark true + --tts.language-code en-US --tts.voice-name John ``` For Riva-compatible gRPC, use the same engine options with `riva_server` and @@ -39,33 +39,43 @@ as `en/`, `fr/`, and `vi/`, each with those two FARs; `post_process.far` is used when present. The older split layout (`classify/tokenize_and_classify.far`, `verbalize/verbalize.far`) remains supported. -The optional `riva_server` adapter implements `RivaSpeechSynthesis.Synthesize`, +The optional `riva_server` supports `RivaSpeechSynthesis.Synthesize`, `SynthesizeOnline`, and `GetRivaSynthesisConfig`. It takes plain text in `SynthesizeSpeechRequest.text`, supports native Magpie tokenizers for `en`, `es`, `de`, `fr`, `it`, `vi`, `zh`, `hi`, and `ja`, and returns `LINEAR_PCM` s16le at the NanoCodec sample rate. Japanese and Mandarin are included when `NEMO_SPEECH_TTS_WITH_JA` and `NEMO_SPEECH_TTS_WITH_ZH`, respectively, are -enabled at build time; both default to `OFF`. Native tokenizers are cached by -language. Mandarin uses bundled Jieba and pypinyin-compatible data together -with the model's pinyin-to-phoneme dictionary. Set `MAGPIE_MANDARIN_G2P_DIR` -only to override the bundled Mandarin data directory. +enabled at build time; both default to `OFF`. Mandarin uses bundled Jieba and +pypinyin-compatible data together with the model's pinyin-to-phoneme +dictionary. Set `MAGPIE_MANDARIN_G2P_DIR` only to override the bundled Mandarin +data directory. -`GetRivaSynthesisConfig` advertises every compiled-in TTS language in -`language_code` and exposes the per-language dotted voice names in -`voices_by_language`. The legacy `voice_name`, `subvoices`, and `voices` -parameters remain available for clients that assemble voice names themselves. +`GetRivaSynthesisConfig` advertises the compiled-in TTS languages and the +dotted voice names accepted by synthesis requests. TTS auto-enables when `tts.magpie-model`, `tts.codec-model`, and `tts.tokenizer-model-dir` are all set; force with `tts.enabled`. +## Voices + +Voice names are case-insensitive. The runtime accepts a local speaker name, a +zero-based speaker index, or a model-qualified name such as +`magpietts.John`. `tts.voice-name` selects the default; otherwise +`tts.speaker` is used. + +The HTTP model inventory lists the available local names. On +`/v1/audio/speech`, `default` and supported OpenAI voice aliases such as +`alloy` select that configured local default; they are not additional voices. See the +[HTTP API reference](../api.md#post-v1audiospeech). + ## Text normalization Install the shared Sparrowhawk/OpenFST normalizer and enable it in the build: ```bash scripts/build_itn_deps.sh -scripts/configure.sh cpu -DNEMO_SPEECH_WITH_NORM=ON -cmake --build build -j +scripts/configure.sh cpu-tts -DNEMO_SPEECH_WITH_NORM=ON +cmake --build --preset cpu-tts ``` Pass the grammar directory to the CLI or server: @@ -129,7 +139,7 @@ All keys nest under `tts.`. Defaults shown; CLI alias listed where one exists. | `tts.tokenizer-model-dir` | - | - | extracted Magpie `.nemo` dir (required) | | `tts.tokenizer.sentence-limit.` | - | per language (`en` 45 ... `ja` 40) | sentence-chunking threshold in words (characters for `zh`/`ja`); subkeys `en`, `es`, `fr`, `vi`, `it`, `de`, `zh`, `hi`, `ja` | | `tts.tn-model-dir` | - | - | enables Sparrowhawk TN with this grammar dir; requires `NEMO_SPEECH_WITH_NORM=ON` | -| `tts.language-code` | - | `en-US` | default Riva language code | +| `tts.language-code` | - | `en-US` | default text language code | | `tts.voice-name` | - | - | default voice name or speaker index | | `tts.speaker` | - | `0` | default baked speaker index | @@ -163,7 +173,7 @@ All keys nest under `tts.`. Defaults shown; CLI alias listed where one exists. | key | CLI alias | default | meaning | |---|---|---|---| -| `tts.threads` | `--threads` | `4` | CPU threads for Magpie + codec | +| `tts.threads` | `--threads` | `4` | CPU threads for Magpie + codec; use the dotted key with HTTP, where `--threads` controls request workers | | `tts.codec-threads` | - | `0` | codec CPU threads; `0` = use `threads` | | `tts.lt-backend` | - | `auto` | local-transformer backend: `auto`/`cpu`/`cuda` | | `tts.lt-fp32` | `--tts.local-transformer-fp32` | `false` | run the local transformer in FP32 | @@ -175,16 +185,8 @@ All keys nest under `tts.`. Defaults shown; CLI alias listed where one exists. | key | CLI alias | default | meaning | |---|---|---|---| -| `tts.benchmark` | `--benchmark` | `false` | print tokenizer/runtime timing per request | -| `tts.verbose` | `--verbose` | `false` | detailed Magpie/NanoCodec chunk logs | +| `tts.benchmark` | `--benchmark` | `false` | emit per-request metrics from `riva_server` | +| `tts.verbose` | `--verbose` | `false` | detailed Magpie/NanoCodec logs; use global `--verbose` with `nemo-speech` | | `tts.warmup-enabled` / `tts.no-warmup` | - | on | startup tokenizer/runtime warmup | | `tts.warmup-text` | - | (built-in) | text used for startup warmup | | `tts.warmup-steps` | - | `8` | decoder frames used for startup warmup | - -## Notes - -- With CUDA Graphs enabled, the shared TTS host defaults - `GGML_CUDA_GRAPH_EVICT_AFTER_MS=0` so captured graphs stay resident across idle - request gaps; set that env var before launch to restore eviction. -- CUDA builds require the ggml patches applied before compiling - see - [ggml patches](../development/ggml-patches.md). diff --git a/docs/tts/models.md b/docs/tts/models.md index 3957613..5ce2050 100644 --- a/docs/tts/models.md +++ b/docs/tts/models.md @@ -62,22 +62,16 @@ nemo-speech synthesize "Hello from Magpie Multilingual." --output output.wav The unified [`convert_model.py`](../../convert_model.py) entry point accepts compatible local `.nemo` archives and extracted NeMo checkpoints. It defaults to `--outtype f16` for MagpieTTS and NanoCodec; pass `--outtype f32` to retain -full precision. +full precision. The converter is a source-tree Python tool and is not included +in native release archives; see [Model conversion](../model-conversion.md) for +environment setup. ```bash -pip install -r requirements.txt python3 convert_model.py custom-magpie.nemo --outfile custom-magpie.f16.gguf ``` -The converter reads `.nemo` archives directly with PyTorch and does not require -`nemo_toolkit`. The optional `scripts/tts/tokenize-magpietts.py` debugging -helper does use NeMo's Python tokenizer implementation. +Conversion does not require `nemo_toolkit`. The optional +`scripts/tts/tokenize-magpietts.py` debugging helper does. -## Notes - -For CUDA builds, the MagpieTTS and NanoCodec operations require the ggml -patches applied by `scripts/configure.sh`; see -[ggml patches](../development/ggml-patches.md). - -Once converted, point the server at them - see +Once converted, point the server at them; see [TTS configuration](configuration.md). diff --git a/include/nemo_speech/asr.h b/include/nemo_speech/asr.h index 03d5548..746655d 100644 --- a/include/nemo_speech/asr.h +++ b/include/nemo_speech/asr.h @@ -185,7 +185,7 @@ typedef struct nemo_speech_asr_recognition_options { // (nemo_speech_asr_result_word_speaker_tag). Requires the recognizer to have been // created with a diar model (nemo_speech_asr_diar_config.model_path), else the // request fails with INVALID_ARGUMENT. max_speaker_count is accepted for - // riva compatibility; the Sortformer model's capacity (4) caps it. + // compatibility but is ignored; Sortformer v2 supports up to four speakers. bool enable_speaker_diarization; int32_t max_speaker_count; } nemo_speech_asr_recognition_options; diff --git a/src/asr/CMakeLists.txt b/src/asr/CMakeLists.txt index 333db6e..0474889 100644 --- a/src/asr/CMakeLists.txt +++ b/src/asr/CMakeLists.txt @@ -95,7 +95,7 @@ if(SENTENCEPIECE_STATIC_LIB) endif() elseif(NEMO_SPEECH_WITH_NORM AND UNIX AND NOT APPLE) message(FATAL_ERROR - "ITN + Flashlight requires private static SentencePiece; " + "normalization requires private static SentencePiece; " "run scripts/build_sentencepiece_static.sh") else() # vcpkg's imported target carries its platform-specific usage requirements. diff --git a/src/tts/magpietts/README.md b/src/tts/magpietts/README.md index f2e1fe1..2ea5918 100644 --- a/src/tts/magpietts/README.md +++ b/src/tts/magpietts/README.md @@ -1,10 +1,12 @@ # MagpieTTS Runtime -This directory contains the GGML/GGUF MagpieTTS runtime used by two public +This directory contains the GGML/GGUF MagpieTTS runtime used by the public surfaces: -- `synthesize_text`: standalone C-ABI text or token-ID to WAV example. -- `riva_server`: Riva-compatible gRPC server that accepts real text for TTS. +- `nemo-speech synthesize`: local text-to-speech CLI. +- `nemo-speech serve`: HTTP API and browser playground. +- `synthesize_text`: optional stable-C-ABI example with text and token-ID input. +- `riva_server`: optional Riva-compatible gRPC server. MagpieTTS generates codec tokens autoregressively. The public multilingual 357M checkpoint uses the separate NeMo NanoCodec decoder @@ -16,77 +18,80 @@ into 22050 Hz mono PCM audio. The examples below assume these files are available: ```text -models/magpie_tts_multilingual_357m/magpie_tts_multilingual_357m.f16.gguf -models/magpie_tts_multilingual_357m/extracted -models/nemo_nano_codec_22khz_1.89kbps_21.5fps/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf +models/magpie-tts/magpie_tts_multilingual_357m.v2602.f16.gguf +models/magpie-tts/extracted +models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf ``` -`magpie_tts_multilingual_357m.f16.gguf` is the MagpieTTS autoregressive model. -The `extracted` directory is the unpacked MagpieTTS `.nemo` checkpoint and is -needed by the tokenizer. The NanoCodec GGUF is the token-to-audio decoder. +`magpie_tts_multilingual_357m.v2602.f16.gguf` is the MagpieTTS autoregressive +model. The `extracted` directory is the unpacked MagpieTTS `.nemo` checkpoint +and is needed by the tokenizer. The NanoCodec GGUF is the token-to-audio +decoder. ## Build -For CUDA builds, apply the local ggml patch before compiling MagpieTTS or -NanoCodec targets: +Use a supported preset. The configure helper validates dependencies and applies +the pinned ggml patches for CUDA builds: ```bash -scripts/apply-ggml-patches.sh -cmake -S . -B build \ - -DGGML_CUDA=ON \ - -DGGML_CUDA_GRAPHS=ON \ - -DNEMO_SPEECH_BUILD_GRPC=ON -cmake --build build --target synthesize_text riva_server -j$(nproc) +scripts/configure.sh cuda-tts +cmake --build --preset cuda-tts ``` -`GGML_CUDA=ON` enables the CUDA backend. `GGML_CUDA_GRAPHS=ON` reduces CUDA -graph-launch overhead. `NEMO_SPEECH_BUILD_GRPC=ON` is required for -`riva_server`. For a CPU-only standalone build, omit the CUDA flags; for a -standalone-only build, the gRPC flag is not required. - -At runtime, set `GGML_CUDA_DISABLE_GRAPHS=1` to compare CUDA behavior without -CUDA Graphs. The ggml patch also exposes `GGML_CUDA_GRAPH_EVICT_AFTER_MS` and -`GGML_CUDA_GRAPH_SWEEP_MS`; the default eviction policy is 10 seconds with a -5 second sweep interval, and `GGML_CUDA_GRAPH_EVICT_AFTER_MS=0` disables -eviction. +Use `cpu-tts` for a CPU build. To build the optional gRPC server, use +`cuda-full` after installing the optional dependencies described in the +[`build guide`](../../../docs/build.md). To build `synthesize_text`, add +`-DNEMO_SPEECH_BUILD_EXAMPLES=ON` while configuring. ## Convert To GGUF Convert the MagpieTTS `.nemo` checkpoint or extracted checkpoint directory: ```bash -python convert_model.py models/magpie_tts_multilingual_357m/extracted \ - --outfile models/magpie_tts_multilingual_357m/magpie_tts_multilingual_357m.f16.gguf \ +python convert_model.py models/magpie-tts/extracted \ + --outfile models/magpie-tts/magpie_tts_multilingual_357m.v2602.f16.gguf \ --outtype f16 \ - --metadata-json models/magpie_tts_multilingual_357m/magpie_tts_multilingual_357m.gguf.json + --metadata-json models/magpie-tts/magpie_tts_multilingual_357m.gguf.json ``` Convert the NanoCodec decoder separately: ```bash -python convert_model.py models/nemo_nano_codec_22khz_1.89kbps_21.5fps/extracted \ - --outfile models/nemo_nano_codec_22khz_1.89kbps_21.5fps/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ +python convert_model.py models/nano-codec/extracted \ + --outfile models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ --outtype f16 ``` +The converters are source-tree Python tools; see +[`docs/model-conversion.md`](../../../docs/model-conversion.md) for setup. + +## Unified CLI + +The default command downloads the pinned MagpieTTS, tokenizer, and NanoCodec +artifacts when needed: + +```bash +build/cuda-tts/bin/nemo-speech synthesize "Hello world." --output magpie.wav +``` + ## Standalone Example `synthesize_text` uses the same stable C ABI available to external applications. It accepts text by default and also supports pre-tokenized IDs for diagnostics: ```bash -build/bin/synthesize_text \ - --tts.magpie-model models/magpie_tts_multilingual_357m/magpie_tts_multilingual_357m.f16.gguf \ - --tts.codec-model models/nemo_nano_codec_22khz_1.89kbps_21.5fps/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ - --tts.tokenizer-model-dir models/magpie_tts_multilingual_357m/extracted \ +build/cuda-tts/bin/synthesize_text \ + --tts.magpie-model models/magpie-tts/magpie_tts_multilingual_357m.v2602.f16.gguf \ + --tts.codec-model models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ + --tts.tokenizer-model-dir models/magpie-tts/extracted \ --tts.text "Hello world." \ --tts.speaker 0 \ --tts.steps 64 \ --tts.wav-out /tmp/magpietts.wav ``` -Run `build/bin/synthesize_text --help` for text-file, token-ID, resampling, and -generation options. +Run `build/cuda-tts/bin/synthesize_text --help` for text-file, token-ID, +resampling, and generation options. ## Riva TTS Server @@ -97,10 +102,10 @@ and tokenizes it internally with the native C++ tokenizer. Launch the server: ```bash -build/bin/riva_server \ - --tts.magpie-model models/magpie_tts_multilingual_357m/magpie_tts_multilingual_357m.f16.gguf \ - --tts.codec-model models/nemo_nano_codec_22khz_1.89kbps_21.5fps/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ - --tts.tokenizer-model-dir models/magpie_tts_multilingual_357m/extracted \ +build/cuda-full/bin/riva_server \ + --tts.magpie-model models/magpie-tts/magpie_tts_multilingual_357m.v2602.f16.gguf \ + --tts.codec-model models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ + --tts.tokenizer-model-dir models/magpie-tts/extracted \ --bind 0.0.0.0:50051 \ --tts.language-code en-US \ --tts.voice-name John \ @@ -112,22 +117,16 @@ Send requests with a Riva-compatible client; see The server implements `Synthesize`, `SynthesizeOnline`, and `GetRivaSynthesisConfig`. It returns raw `LINEAR_PCM` s16le audio at the -NanoCodec sample rate. Native tokenization currently supports `en`, `es`, `de`, -`fr`, `it`, `vi`, `zh`, `hi`, and `ja`; tokenizers are loaded once and cached -by language. - -`GetRivaSynthesisConfig` exposes speakers using Riva's `voice_name` and -`subvoices` parameters. The resulting dotted names, such as `magpietts.John`, -and the original short names, such as `John`, are both accepted in synthesis -requests. The config response advertises every compiled-in TTS language in -`language_code` and includes a `voices_by_language` JSON parameter with the -dotted Magpie voice names for each language. - -By default, the server runs a short discarded startup warmup request to -initialize tokenizer, runtime state, and local-transformer graph capture. -`riva_server` also defaults `GGML_CUDA_GRAPH_EVICT_AFTER_MS=0` before -model load unless the environment variable is already set, so CUDA Graphs stay -resident across idle request gaps. Use: +NanoCodec sample rate. Native tokenization supports `en`, `es`, `de`, `fr`, +`it`, `vi`, and `hi` by default; `zh` and `ja` require builds with +`NEMO_SPEECH_TTS_WITH_ZH=ON` and `NEMO_SPEECH_TTS_WITH_JA=ON`, respectively. +Tokenizers are loaded once and cached by language. + +`GetRivaSynthesisConfig` advertises every compiled-in TTS language and the +dotted voice names accepted by synthesis requests. Bare local voice names are +also accepted. + +By default, the server runs a short discarded startup warmup request. Use: - `--tts.no-warmup`: skip startup warmup. - `--tts.warmup-text TEXT`: choose the warmup text. @@ -146,22 +145,23 @@ Useful server options: `--tts.no-cfg`, `--tts.no-local-transformer`, `--tts.no-kv-cache`, and `--tts.no-stateful-codec`: use the same runtime controls as the standalone path. -- `--seed`, `--steps`, `--temperature`, `--top-k`, and `--cfg-scale`: set - default generation controls. The Python client can override these per - request through `custom_configuration`. +- `--tts.seed`, `--tts.steps`, `--tts.temperature`, `--tts.top-k`, and + `--tts.cfg-scale`: set default generation controls. The Python client can + override these per request through `custom_configuration`. ## Troubleshooting -- If CUDA TTS targets fail to compile, run - `scripts/apply_ggml_magpietts_patch.sh` before rebuilding. -- If the standalone runner rejects text input, check that - `--tokenizer-model-dir` points at the extracted MagpieTTS `.nemo` directory - and that `--language-code` is supported. For tokenizer debugging, generate - token IDs with `scripts/tts/tokenize-magpietts.py` and pass them with - `--tokens` or `--tokens-file`. +- If CUDA TTS targets fail to compile after a ggml update, rerun + `scripts/configure.sh cuda-tts` before rebuilding. +- If `nemo-speech synthesize` rejects text input, check that `--tokenizer-dir` + points at the extracted MagpieTTS `.nemo` directory and that `--language` is + supported. The `synthesize_text` equivalents are + `--tts.tokenizer-model-dir` and `--tts.language-code`. For tokenizer + debugging, generate token IDs with `scripts/tts/tokenize-magpietts.py` and + pass them with `--tts.tokens` or `--tts.tokens-file`. - If the server rejects a request, check `language_code`, `voice_name`, and `sample_rate_hz`. The client may omit `sample_rate_hz` for native 22050 Hz output or request downsampling to any integer rate from 8000 through 22050 Hz, including 11025 and 16000 Hz. -- If first-request latency matters, keep startup warmup enabled. For controlled - cold-start experiments, use `--no-warmup`. +- If first-request latency matters, keep startup warmup enabled. Disable it with + `--no-warmup` in the unified CLI or `--tts.no-warmup` in `riva_server`. diff --git a/src/tts/nanocodec/README.md b/src/tts/nanocodec/README.md index 64161a7..ffbaa3b 100644 --- a/src/tts/nanocodec/README.md +++ b/src/tts/nanocodec/README.md @@ -1,38 +1,38 @@ # NeMo NanoCodec GGUF -This runner converts the NVIDIA NeMo NanoCodec 22 kHz checkpoint to a GGUF -decoder and runs the token-to-audio path used by MagpieTTS. +This directory covers converting the NVIDIA NeMo NanoCodec 22 kHz checkpoint +to a GGUF decoder and running the token-to-audio path used by MagpieTTS. Model: ## Convert +Run the converter from a source checkout after following +[`docs/model-conversion.md`](../../../docs/model-conversion.md): + ```bash -python convert_model.py models/nemo_nano_codec_22khz_1.89kbps_21.5fps/extracted \ - --outfile models/nemo_nano_codec_22khz_1.89kbps_21.5fps/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ - --metadata-json models/nemo_nano_codec_22khz_1.89kbps_21.5fps/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.gguf.json +python convert_model.py models/nano-codec/extracted \ + --outfile models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ + --metadata-json models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.gguf.json ``` -The converter writes only the inference decoder needed for codec tokens to -audio. It folds PyTorch weight norm into plain convolution weights, stores -Conv1d tensors in GGML layout, expands grouped ConvTranspose1d kernels into -GGML-compatible dense kernels, and records the deterministic FSQ quantizer -metadata/codebook. +The converter writes the inference decoder and FSQ codebook metadata required +to turn codec tokens into audio. ## Decode Build the decoder: ```bash -cmake -S . -B build -DNEMO_SPEECH_BUILD_TOOLS=ON -cmake --build build --target nanocodec -j +scripts/configure.sh cpu-tts -DNEMO_SPEECH_BUILD_TOOLS=ON +cmake --build --preset cpu-tts --target nanocodec ``` Provide a text file containing codec tokens, then run: ```bash -build/bin/nanocodec \ - -m models/nemo_nano_codec_22khz_1.89kbps_21.5fps/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ +build/cpu-tts/bin/nanocodec \ + -m models/nano-codec/nemo_nano_codec_22khz_1.89kbps_21.5fps.decoder.f16.gguf \ --codes magpie_codes.txt \ -o magpie.wav ``` diff --git a/src/tts/tokenizer/mandarin_data/README.md b/src/tts/tokenizer/mandarin_data/README.md index dba7258..4881c25 100644 --- a/src/tts/tokenizer/mandarin_data/README.md +++ b/src/tts/tokenizer/mandarin_data/README.md @@ -14,6 +14,7 @@ recorded in `manifest.json`. Regenerate the runtime tables with the pinned Python packages installed: ```bash +python3 -m pip install jieba==0.42.1 pypinyin==0.55.0 pypinyin-dict==0.9.0 python3 scripts/tts/generate_mandarin_tokenizer_data.py \ --output-dir src/tts/tokenizer/mandarin_data ``` From 3948338f8b603965dc3ac2cf51847eb55865fa71 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 16:44:13 +0530 Subject: [PATCH 09/11] install(mac): ship sentencepiece license in releases --- THIRD_PARTY_NOTICES.md | 6 +++--- src/asr/CMakeLists.txt | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index f543c65..a7314ab 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -161,9 +161,9 @@ The command-line microphone capture layer compiles miniaudio directly into - Copyright 2018 Google Inc. - License: Apache License 2.0 -Default Windows ASR and Linux release builds statically link the SentencePiece -runtime and its bundled Abseil, protobuf-lite, and Darts-clone components. Their -Apache 2.0 and BSD license texts are installed under +Default Windows ASR, macOS, and Linux release builds statically link the +SentencePiece runtime and its bundled Abseil, protobuf-lite, and Darts-clone +components. Their Apache 2.0 and BSD license texts are installed under `share/licenses/nemo-speech/third_party/sentencepiece/`. ### whisper.cpp sample audio diff --git a/src/asr/CMakeLists.txt b/src/asr/CMakeLists.txt index 0474889..ad71691 100644 --- a/src/asr/CMakeLists.txt +++ b/src/asr/CMakeLists.txt @@ -87,12 +87,6 @@ if(SENTENCEPIECE_STATIC_LIB) target_include_directories(nemo_speech_asr PRIVATE ${SENTENCEPIECE_INCLUDE_DIR}) target_link_options( nemo_speech_asr PRIVATE "LINKER:--exclude-libs,libsentencepiece.a") - set(_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR - "${NEMO_SPEECH_DEPENDENCY_PREFIX}/sentencepiece/share/licenses/nemo-speech/third_party/sentencepiece") - if(EXISTS "${_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR}/LICENSE") - install(DIRECTORY "${_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR}/" - DESTINATION "${NEMO_SPEECH_LICENSE_INSTALL_DIR}/third_party/sentencepiece") - endif() elseif(NEMO_SPEECH_WITH_NORM AND UNIX AND NOT APPLE) message(FATAL_ERROR "normalization requires private static SentencePiece; " @@ -110,6 +104,12 @@ else() target_include_directories(nemo_speech_asr PRIVATE ${SENTENCEPIECE_INCLUDE_DIR}) endif() endif() +set(_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR + "${NEMO_SPEECH_DEPENDENCY_PREFIX}/sentencepiece/share/licenses/nemo-speech/third_party/sentencepiece") +if(EXISTS "${_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR}/LICENSE") + install(DIRECTORY "${_NEMO_SPEECH_SENTENCEPIECE_LICENSE_DIR}/" + DESTINATION "${NEMO_SPEECH_LICENSE_INSTALL_DIR}/third_party/sentencepiece") +endif() # SentencePiece (vcpkg) uses absl::flat_hash_map + protobuf internally, but its # config target doesn't propagate those link deps. On Windows/vcpkg, add them # explicitly; elsewhere SentencePiece links its own deps (apt's -dev package). From dbdd465f6163909ddcef3aa49b43fc475a8b7359 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 12:32:57 +0000 Subject: [PATCH 10/11] fix: coderabbit review comments --- THIRD_PARTY_NOTICES.md | 8 ++- app/microphone_capture.cpp | 51 ++++++++----- app/model_store.cpp | 114 ++++++++++++++++++++++++++---- scripts/install.ps1 | 4 +- tests/cli/model_store_test.py | 15 ++++ tests/install/install_ps1_test.py | 11 +++ third_party/miniaudio/LICENSE | 33 ++++++++- 7 files changed, 199 insertions(+), 37 deletions(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index a7314ab..e15abb7 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -144,10 +144,12 @@ applicable agreement is installed with the archive under ### miniaudio - Source: [`mackron/miniaudio`](https://github.com/mackron/miniaudio), version - 0.11.25, vendored by the pinned llama.cpp checkout + 0.11.25 at commit `9634bedb5b5a2ca38c1ee7108a9358a4e233f14d`, vendored by + the pinned llama.cpp checkout - Path: `llama.cpp/vendor/miniaudio/miniaudio.h` -- Copyright 2026 David Reid -- License: MIT No Attribution (MIT-0); upstream text is reproduced at +- Copyright 2025 David Reid +- License: Public Domain (Unlicense) or MIT No Attribution (MIT-0); upstream + text is reproduced at [`third_party/miniaudio/LICENSE`](third_party/miniaudio/LICENSE) The command-line microphone capture layer compiles miniaudio directly into diff --git a/app/microphone_capture.cpp b/app/microphone_capture.cpp index 77b228e..7c88c52 100644 --- a/app/microphone_capture.cpp +++ b/app/microphone_capture.cpp @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include "microphone_capture.h" +#include +#include #include -#include +#include #include // miniaudio is already vendored by the repository. Compiling its capture-only @@ -23,13 +25,14 @@ namespace nemo_speech::cli { struct MicrophoneCapture::Impl { static constexpr int kSampleRate = 16000; + static constexpr size_t kQueueCapacity = kSampleRate * 2; ma_device device{}; bool initialized = false; bool running = false; - std::atomic callback_failed{false}; - std::mutex mutex; - std::vector pending; + std::array pending{}; + std::atomic read_position{0}; + std::atomic write_position{0}; std::string name = "default microphone"; static void data_callback( @@ -38,14 +41,17 @@ struct MicrophoneCapture::Impl { return; auto* self = static_cast(device->pUserData); const auto* samples = static_cast(input); - try { - std::lock_guard lock(self->mutex); - self->pending.insert(self->pending.end(), samples, samples + frame_count); - } - catch (...) { - // Exceptions must never escape the operating system's audio callback. - self->callback_failed.store(true, std::memory_order_release); - } + const size_t write = self->write_position.load(std::memory_order_relaxed); + const size_t read = self->read_position.load(std::memory_order_acquire); + const size_t count = std::min(frame_count, kQueueCapacity - (write - read)); + + // Preserve queued audio and drop newest samples on overflow. The callback + // never waits for space or allocates memory. + const size_t offset = write % kQueueCapacity; + const size_t first = std::min(count, kQueueCapacity - offset); + std::memcpy(self->pending.data() + offset, samples, first * sizeof(float)); + std::memcpy(self->pending.data(), samples + first, (count - first) * sizeof(float)); + self->write_position.store(write + count, std::memory_order_release); } void dispose() noexcept { @@ -78,8 +84,8 @@ MicrophoneCapture::start() { config.periodSizeInMilliseconds = 40; config.dataCallback = Impl::data_callback; config.pUserData = impl_.get(); - impl_->pending.reserve(Impl::kSampleRate * 2); - impl_->callback_failed.store(false, std::memory_order_release); + impl_->read_position.store(0, std::memory_order_relaxed); + impl_->write_position.store(0, std::memory_order_relaxed); ma_result result = ma_device_init(nullptr, &config, &impl_->device); if (result != MA_SUCCESS) @@ -106,11 +112,18 @@ MicrophoneCapture::stop() { std::vector MicrophoneCapture::drain() { - if (impl_->callback_failed.load(std::memory_order_acquire)) - throw std::runtime_error("microphone capture ran out of buffer memory"); - std::vector samples; - std::lock_guard lock(impl_->mutex); - samples.swap(impl_->pending); + const size_t read = impl_->read_position.load(std::memory_order_relaxed); + const size_t write = impl_->write_position.load(std::memory_order_acquire); + const size_t count = write - read; + if (count == 0) + return {}; + std::vector samples(count); + + const size_t offset = read % Impl::kQueueCapacity; + const size_t first = std::min(count, Impl::kQueueCapacity - offset); + std::memcpy(samples.data(), impl_->pending.data() + offset, first * sizeof(float)); + std::memcpy(samples.data() + first, impl_->pending.data(), (count - first) * sizeof(float)); + impl_->read_position.store(read + count, std::memory_order_release); return samples; } diff --git a/app/model_store.cpp b/app/model_store.cpp index 66d1c2c..ff93a1e 100644 --- a/app/model_store.cpp +++ b/app/model_store.cpp @@ -628,17 +628,17 @@ run_curl(const std::vector& arguments) { const intptr_t status = _wspawnv(_P_WAIT, executable.c_str(), argv.data()); return status < 0 ? 127 : static_cast(status); #else + const std::string executable_string = executable.string(); + std::vector argv; + argv.reserve(arguments.size() + 2); + argv.push_back(const_cast(executable_string.c_str())); + for (const auto& argument : arguments) argv.push_back(const_cast(argument.c_str())); + argv.push_back(nullptr); const pid_t child = fork(); if (child < 0) throw std::runtime_error("could not start curl"); if (child == 0) { - std::vector argv; - argv.reserve(arguments.size() + 2); - const std::string executable_string = executable.string(); - argv.push_back(const_cast(executable_string.c_str())); - for (const auto& argument : arguments) argv.push_back(const_cast(argument.c_str())); - argv.push_back(nullptr); - execv(executable.c_str(), argv.data()); + execv(executable_string.c_str(), argv.data()); _exit(127); } int status = 0; @@ -749,13 +749,100 @@ download(const Model& model, const Artifact& artifact, const fs::path& output) { fs::remove(curl_errors, error); } +struct FileState { + uint64_t size; + fs::file_time_type modified; +}; + +bool +file_state(const fs::path& path, uint64_t expected_size, FileState& state) { + std::error_code error; + if (!fs::is_regular_file(path, error) || error) + return false; + const uintmax_t size = fs::file_size(path, error); + if (error || size != expected_size) + return false; + const fs::file_time_type modified = fs::last_write_time(path, error); + if (error) + return false; + state = {static_cast(size), modified}; + return true; +} + +fs::path +verification_marker_path(const fs::path& path) { + fs::path marker = path; + marker += ".verified"; + return marker; +} + +std::string +file_time_string(fs::file_time_type value) { + std::ostringstream output; + output << value.time_since_epoch().count(); + return output.str(); +} + bool -valid_file(const fs::path& path, const Artifact& artifact) { +verification_marker_matches( + const fs::path& path, const Artifact& artifact, const FileState& state) { + const fs::path marker = verification_marker_path(path); std::error_code error; - if (!fs::is_regular_file(path, error) || error || fs::file_size(path, error) != artifact.size || - error) + if (!fs::is_regular_file(marker, error) || error || fs::file_size(marker, error) > 256 || error) + return false; + std::ifstream input(marker, std::ios::binary); + std::array lines; + for (auto& line : lines) + if (!std::getline(input, line)) + return false; + std::string extra; + if (std::getline(input, extra)) + return false; + return lines[0] == "sha256=" + artifact.sha256 && + lines[1] == "size=" + std::to_string(state.size) && + lines[2] == "mtime=" + file_time_string(state.modified); +} + +void +write_verification_marker(const fs::path& path, const Artifact& artifact, const FileState& state) { + const fs::path marker = verification_marker_path(path); + fs::path temporary = marker; + temporary += ".tmp"; + { + std::ofstream output(temporary, std::ios::binary | std::ios::trunc); + output << "sha256=" << artifact.sha256 << '\n' + << "size=" << state.size << '\n' + << "mtime=" << file_time_string(state.modified) << '\n'; + if (!output) { + std::error_code error; + fs::remove(temporary, error); + return; + } + } + std::error_code error; + fs::remove(marker, error); + error.clear(); + fs::rename(temporary, marker, error); + if (error) + fs::remove(temporary, error); +} + +bool +valid_file(const fs::path& path, const Artifact& artifact, bool cache_hit = false) { + FileState before{}; + if (!file_state(path, artifact.size, before)) + return false; + if (cache_hit && verification_marker_matches(path, artifact, before)) + return true; + if (sha256_file(path) != artifact.sha256) + return false; + FileState after{}; + if (!file_state(path, artifact.size, after) || before.size != after.size || + before.modified != after.modified) return false; - return sha256_file(path) == artifact.sha256; + if (cache_hit) + write_verification_marker(path, artifact, after); + return true; } bool @@ -900,7 +987,7 @@ materialize(const Model& model, const Artifact& artifact) { fs::path lock_path = destination; lock_path += ".lock"; ArtifactLock lock(lock_path); - if (artifact.type == "file" && valid_file(destination, artifact)) { + if (artifact.type == "file" && valid_file(destination, artifact, true)) { if (cli_verbose()) std::fprintf(stderr, "[model] cached: %s\n", path_utf8(destination).c_str()); return {model.repo, artifact.role, destination, true}; @@ -954,6 +1041,9 @@ materialize(const Model& model, const Artifact& artifact) { fs::rename(partial, destination, error); if (error) throw std::runtime_error("cannot install model artifact: " + error.message()); + FileState state{}; + if (file_state(destination, artifact.size, state)) + write_verification_marker(destination, artifact, state); } else { const fs::path extracting = directory / (artifact.directory + ".extracting"); fs::remove_all(extracting, error); diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 6e39455..181f65d 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -184,7 +184,9 @@ $sourceRef = if ($env:NEMO_SPEECH_SOURCE_REF) { $env:NEMO_SPEECH_SOURCE_REF } elseif ($releaseVersion -eq 'source' -and (Get-Command git -ErrorAction SilentlyContinue) -and - (Test-Path (Join-Path $sourceUrl '.git'))) { + ($sourceUrl -notmatch '^[A-Za-z][A-Za-z0-9+.-]*://') -and + (Test-Path -LiteralPath $sourceUrl -PathType Container) -and + (Test-Path -LiteralPath (Join-Path $sourceUrl '.git'))) { $localSourceRef = (& git -C $sourceUrl symbolic-ref --quiet --short HEAD 2>$null) if ($LASTEXITCODE -eq 0 -and $localSourceRef) { $localSourceRef } else { 'main' } } elseif ($releaseVersion -in @("nightly", "source")) { diff --git a/tests/cli/model_store_test.py b/tests/cli/model_store_test.py index a36242a..0f4e5ec 100644 --- a/tests/cli/model_store_test.py +++ b/tests/cli/model_store_test.py @@ -213,6 +213,11 @@ def main() -> None: assert artifact["repo"] == "acme/tiny-asr" assert artifact["cached"] is False assert destination.read_bytes() == PAYLOAD + marker = pathlib.Path(f"{destination}.verified") + marker_lines = marker.read_text(encoding="utf-8").splitlines() + assert marker_lines[0] == f"sha256={hashlib.sha256(PAYLOAD).hexdigest()}" + assert marker_lines[1] == f"size={len(PAYLOAD)}" + assert marker_lines[2].startswith("mtime=") requests_after_pull = ArtifactHandler.requests cached = run(binary, environment, "--json", "model", "pull", "acme/tiny-asr") @@ -220,6 +225,13 @@ def main() -> None: assert json.loads(cached.stdout)["artifacts"][0]["cached"] is True assert ArtifactHandler.requests == requests_after_pull + marker.write_text("invalid\n", encoding="utf-8") + refreshed = run(binary, environment, "--json", "pull", "tiny-asr") + assert refreshed.returncode == 0, refreshed.stderr + assert json.loads(refreshed.stdout)["artifacts"][0]["cached"] is True + assert ArtifactHandler.requests == requests_after_pull + assert marker.read_text(encoding="utf-8").splitlines() == marker_lines + tts = run(binary, environment, "--json", "pull", "tiny-tts") assert tts.returncode == 0, tts.stderr tts_artifacts = json.loads(tts.stdout)["artifacts"] @@ -242,11 +254,14 @@ def main() -> None: assert ArtifactHandler.request_counts["tiny-tts.nemo"] == 1 assert ArtifactHandler.request_counts["tiny-codec.gguf"] == 1 + previous_mtime = destination.stat().st_mtime_ns destination.write_bytes(b"x" * len(PAYLOAD)) + os.utime(destination, ns=(previous_mtime + 2_000_000_000,) * 2) repaired = run(binary, environment, "--json", "pull", "tiny-asr") assert repaired.returncode == 0, repaired.stderr assert json.loads(repaired.stdout)["artifacts"][0]["cached"] is False assert destination.read_bytes() == PAYLOAD + assert marker.read_text(encoding="utf-8").splitlines() != marker_lines unknown = expect_json_error( run(binary, environment, "--json", "pull", "unknown/repository"), 3 diff --git a/tests/install/install_ps1_test.py b/tests/install/install_ps1_test.py index 2b7ee7f..47705fa 100644 --- a/tests/install/install_ps1_test.py +++ b/tests/install/install_ps1_test.py @@ -188,6 +188,17 @@ def run(*arguments: str, ok: bool = True) -> subprocess.CompletedProcess[str]: raise RuntimeError(f"installer unexpectedly succeeded:\n{result.stdout}") return result + remote_source = "https://github.com/NVIDIA/NeMo-Speech.cpp.git" + environment["NEMO_SPEECH_SOURCE_URL"] = remote_source + environment.pop("NEMO_SPEECH_SOURCE_REF", None) + remote_plan = run("-Source", "-Backend", "cpu", "-DryRun") + require(f"{remote_source}#main" in remote_plan.stdout, "remote source ref") + environment["NEMO_SPEECH_SOURCE_REF"] = "review-test" + override_plan = run("-Source", "-Backend", "cpu", "-DryRun") + require(f"{remote_source}#review-test" in override_plan.stdout, "source ref override") + environment["NEMO_SPEECH_SOURCE_URL"] = str(source) + environment.pop("NEMO_SPEECH_SOURCE_REF") + run("-Prefix", str(prefix), "-Backend", "cpu", "-NoModifyPath") metadata = prefix / ".nemo-speech-install" require( diff --git a/third_party/miniaudio/LICENSE b/third_party/miniaudio/LICENSE index d88e4f3..6d30b9d 100644 --- a/third_party/miniaudio/LICENSE +++ b/third_party/miniaudio/LICENSE @@ -1,6 +1,35 @@ -MIT No Attribution +This software is available as a choice of the following licenses. Choose +whichever you prefer. -Copyright 2026 David Reid +=============================================================================== +ALTERNATIVE 1 - Public Domain (www.unlicense.org) +=============================================================================== +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +=============================================================================== +ALTERNATIVE 2 - MIT No Attribution +=============================================================================== +Copyright 2025 David Reid Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in From 070e51bd907f7139952b7382e91422a89713e9cc Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Wed, 19 Aug 2026 21:17:59 +0530 Subject: [PATCH 11/11] fix: timestamp portability --- app/model_store.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/model_store.cpp b/app/model_store.cpp index ff93a1e..5a99f8a 100644 --- a/app/model_store.cpp +++ b/app/model_store.cpp @@ -778,9 +778,9 @@ verification_marker_path(const fs::path& path) { std::string file_time_string(fs::file_time_type value) { - std::ostringstream output; - output << value.time_since_epoch().count(); - return output.str(); + const auto nanoseconds = + std::chrono::duration_cast(value.time_since_epoch()); + return std::to_string(nanoseconds.count()); } bool