From 8a921724e7fc90aa7d4c3fe82411b0865a312773 Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:25:38 -0300 Subject: [PATCH 01/10] test: prepare reproducible ASR fixture tooling --- .gitignore | 5 +++ scripts/windows/benchmark-pascal-wav.ps1 | 52 ++++++++++++++++++++++++ scripts/windows/build-pascal.ps1 | 10 +++++ scripts/windows/test-http-wav.ps1 | 36 ++++++++++++++++ scripts/windows/test-pascal-wav.ps1 | 20 +++++++++ test_files/fork/asr/README.md | 38 +++++++++++++++++ test_files/fork/asr/teste-en.txt | 1 + 7 files changed, 162 insertions(+) create mode 100644 scripts/windows/benchmark-pascal-wav.ps1 create mode 100644 scripts/windows/build-pascal.ps1 create mode 100644 scripts/windows/test-http-wav.ps1 create mode 100644 scripts/windows/test-pascal-wav.ps1 create mode 100644 test_files/fork/asr/README.md create mode 100644 test_files/fork/asr/teste-en.txt diff --git a/.gitignore b/.gitignore index fb2505a..9e5b5db 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ __pycache__/ core.* client_logs/ .venv/ +.tools/ +benchmark-results/ +models/ +results/ +*.gguf diff --git a/scripts/windows/benchmark-pascal-wav.ps1 b/scripts/windows/benchmark-pascal-wav.ps1 new file mode 100644 index 0000000..394195e --- /dev/null +++ b/scripts/windows/benchmark-pascal-wav.ps1 @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string]$Model, + [ValidateRange(1, 10000)] [int]$Runs = 10, + [string]$Executable, + [ValidatePattern('^(cpu|cuda(:[0-9]+)?)$')] [string]$Device = 'cuda:0' +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$Wav = Join-Path $RepoRoot 'test_files\fork\asr\teste-en.wav' +if (-not $Executable) { $Executable = Join-Path $RepoRoot 'build-pascal-cuda-http\bin\nemo-speech.exe' } +foreach ($item in @(@{ Name = 'Model'; Path = $Model }, @{ Name = 'WAV fixture'; Path = $Wav }, @{ Name = 'Executable'; Path = $Executable })) { + if (-not (Test-Path -LiteralPath $item.Path -PathType Leaf)) { throw "$($item.Name) not found: $($item.Path)" } +} +$arguments = @('transcribe', $Wav, '--model', $Model, '--device', $Device, '--skinny-q8', 'auto', '--suppress-cuda-graph-log', '--format', 'json') +$commandText = '& "{0}" {1}' -f $Executable, (($arguments | ForEach-Object { '"{0}"' -f $_ }) -join ' ') +function Invoke-Measurement { + $watch = [System.Diagnostics.Stopwatch]::StartNew() + $null = & $Executable @arguments 2>&1 + $exitCode = $LASTEXITCODE + $watch.Stop() + if ($exitCode -ne 0) { throw "nemo-speech exited with $exitCode" } + return [Math]::Round($watch.Elapsed.TotalMilliseconds, 3) +} +function Get-Percentile([double[]]$Values, [double]$Percentile) { + $ordered = @($Values | Sort-Object); $index = ($ordered.Count - 1) * $Percentile + $lower = [Math]::Floor($index); $upper = [Math]::Ceiling($index) + if ($lower -eq $upper) { return $ordered[$lower] } + return $ordered[$lower] + (($ordered[$upper] - $ordered[$lower]) * ($index - $lower)) +} + +Write-Host "Warm-up: $commandText" -ForegroundColor Cyan +$warmupMs = Invoke-Measurement +Write-Host ("Warm-up completed in {0:N3} ms (excluded)." -f $warmupMs) +$measurements = [System.Collections.Generic.List[double]]::new() +for ($i = 1; $i -le $Runs; $i++) { $elapsed = Invoke-Measurement; $measurements.Add($elapsed); Write-Host ("Run {0}/{1}: {2:N3} ms" -f $i, $Runs, $elapsed) } + +$values = [double[]]$measurements.ToArray(); $timestamp = Get-Date -Format 'yyyyMMdd-HHmmss' +$resultsDir = Join-Path $RepoRoot 'benchmark-results'; New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null +$jsonPath = Join-Path $resultsDir "pascal-wav-$timestamp.json"; $markdownPath = Join-Path $resultsDir "pascal-wav-$timestamp.md" +$gpu = @(Get-CimInstance Win32_VideoController -ErrorAction SilentlyContinue | ForEach-Object { $_.Name }) +$cpu = (Get-CimInstance Win32_Processor -ErrorAction SilentlyContinue | Select-Object -First 1 -ExpandProperty Name) +$commit = (& git -C $RepoRoot rev-parse HEAD 2>$null).Trim() +$stats = [ordered]@{ minimum_ms = [Math]::Round(($values | Measure-Object -Minimum).Minimum, 3); maximum_ms = [Math]::Round(($values | Measure-Object -Maximum).Maximum, 3); mean_ms = [Math]::Round(($values | Measure-Object -Average).Average, 3); median_ms = [Math]::Round((Get-Percentile $values 0.5), 3); p95_ms = [Math]::Round((Get-Percentile $values 0.95), 3) } +$result = [ordered]@{ timestamp = (Get-Date).ToString('o'); repository_commit = $commit; model = $Model; wav = $Wav; executable = $Executable; device = $Device; command = $commandText; warmup_ms = $warmupMs; runs_ms = $values; statistics = $stats; hardware = [ordered]@{ cpu = $cpu; gpu = $gpu; os = [Environment]::OSVersion.VersionString } } +$result | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $jsonPath -Encoding utf8 +$rows = $values | ForEach-Object -Begin { $number = 0 } -Process { $number++; "| $number | $([Math]::Round($_, 3)) |" } +@('# Pascal WAV benchmark', '', "- Timestamp: $($result.timestamp)", "- Commit: $commit", "- Model: ``$Model``", "- Device: ``$Device``", "- WAV: ``$Wav``", "- Command: ``$commandText``", "- Warm-up excluded: $warmupMs ms", "- CPU: $cpu", "- GPU: $($gpu -join '; ')", '', '## Summary', '', '| Minimum (ms) | Maximum (ms) | Mean (ms) | Median (ms) | P95 (ms) |', '| ---: | ---: | ---: | ---: | ---: |', "| $($stats.minimum_ms) | $($stats.maximum_ms) | $($stats.mean_ms) | $($stats.median_ms) | $($stats.p95_ms) |", '', '## Runs', '', '| Run | Wall time (ms) |', '| ---: | ---:|') + $rows | Set-Content -LiteralPath $markdownPath -Encoding utf8 +Write-Host "Saved JSON: $jsonPath" -ForegroundColor Green +Write-Host "Saved Markdown: $markdownPath" -ForegroundColor Green diff --git a/scripts/windows/build-pascal.ps1 b/scripts/windows/build-pascal.ps1 new file mode 100644 index 0000000..9c69236 --- /dev/null +++ b/scripts/windows/build-pascal.ps1 @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +[CmdletBinding()] +param( + [string]$BuildDir = (Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'build-pascal-cuda-http'), + [int]$Jobs = 0 +) + +$ErrorActionPreference = 'Stop' +& (Join-Path $PSScriptRoot 'build.ps1') -Backend cuda -CudaArch 61 -AsrOnly -Http -BuildDir $BuildDir -Jobs $Jobs +exit $LASTEXITCODE diff --git a/scripts/windows/test-http-wav.ps1 b/scripts/windows/test-http-wav.ps1 new file mode 100644 index 0000000..e795ab2 --- /dev/null +++ b/scripts/windows/test-http-wav.ps1 @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +[CmdletBinding()] +param( + [string]$Url = 'http://127.0.0.1:8081/v1/audio/transcriptions', + [ValidateRange(1, 600)] [int]$TimeoutSeconds = 120 +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$Wav = Join-Path $RepoRoot 'test_files\fork\asr\teste-en.wav' +$ExpectedPath = Join-Path $RepoRoot 'test_files\fork\asr\teste-en.txt' +foreach ($item in @(@{ Name = 'WAV fixture'; Path = $Wav }, @{ Name = 'Expected transcript'; Path = $ExpectedPath })) { + if (-not (Test-Path -LiteralPath $item.Path -PathType Leaf)) { throw "$($item.Name) not found: $($item.Path)" } +} +if (-not (Get-Command curl.exe -ErrorAction SilentlyContinue)) { throw 'curl.exe is required but was not found on PATH.' } +$baseUrl = ([uri]$Url).GetLeftPart([System.UriPartial]::Authority) +$ready = "$baseUrl/ready" +Write-Host "Checking readiness: $ready" -ForegroundColor Cyan +& curl.exe --silent --show-error --fail --max-time $TimeoutSeconds $ready | Out-Host +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +$responseFile = New-TemporaryFile +try { + $metric = & curl.exe --silent --show-error --output $responseFile --write-out '%{http_code}|%{time_total}' --max-time $TimeoutSeconds --form "file=@$Wav;type=audio/wav" --form 'model=default' --form 'response_format=verbose_json' $Url + $exitCode = $LASTEXITCODE + $parts = (($metric | Out-String).Trim()) -split '\|', 2 + if ($exitCode -ne 0) { exit $exitCode } + if ($parts.Count -ne 2 -or [int]$parts[0] -lt 200 -or [int]$parts[0] -ge 300) { throw "HTTP request failed (curl result: $($parts -join '|')). Response: $(Get-Content -LiteralPath $responseFile -Raw)" } + $raw = Get-Content -LiteralPath $responseFile -Raw + try { $payload = $raw | ConvertFrom-Json } catch { throw "Server response was not JSON: $raw" } + $payload | ConvertTo-Json -Depth 20 + $expected = (Get-Content -LiteralPath $ExpectedPath -Raw).Trim(); $actual = ([string]$payload.text).Trim() + Write-Host ("`nHTTP + inference: {0:N2} ms" -f (([double]$parts[1]) * 1000)) -ForegroundColor Green + Write-Host "Expected: $expected"; Write-Host "Received: $actual" + if ($actual -eq $expected) { Write-Host 'Transcript comparison: exact match.' -ForegroundColor Green } else { Write-Host 'Transcript comparison: visually review the expected and received text above.' -ForegroundColor Yellow } +} +finally { Remove-Item -LiteralPath $responseFile -Force -ErrorAction SilentlyContinue } diff --git a/scripts/windows/test-pascal-wav.ps1 b/scripts/windows/test-pascal-wav.ps1 new file mode 100644 index 0000000..ebaf265 --- /dev/null +++ b/scripts/windows/test-pascal-wav.ps1 @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string]$Model, + [string]$Executable, + [ValidatePattern('^(cpu|cuda(:[0-9]+)?)$')] [string]$Device = 'cuda:0' +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$Wav = Join-Path $RepoRoot 'test_files\fork\asr\teste-en.wav' +if (-not $Executable) { $Executable = Join-Path $RepoRoot 'build-pascal-cuda-http\bin\nemo-speech.exe' } +foreach ($item in @(@{ Name = 'Model'; Path = $Model }, @{ Name = 'WAV fixture'; Path = $Wav }, @{ Name = 'Executable'; Path = $Executable })) { + if (-not (Test-Path -LiteralPath $item.Path -PathType Leaf)) { throw "$($item.Name) not found: $($item.Path)" } +} +$arguments = @('transcribe', $Wav, '--model', $Model, '--device', $Device, '--skinny-q8', 'auto', '--suppress-cuda-graph-log', '--format', 'json') +Write-Host 'Running:' -ForegroundColor Cyan +Write-Host ('& "{0}" {1}' -f $Executable, (($arguments | ForEach-Object { '"{0}"' -f $_ }) -join ' ')) +& $Executable @arguments +exit $LASTEXITCODE diff --git a/test_files/fork/asr/README.md b/test_files/fork/asr/README.md new file mode 100644 index 0000000..a186d42 --- /dev/null +++ b/test_files/fork/asr/README.md @@ -0,0 +1,38 @@ +# Pascal fork WAV fixture + +This directory reserves `teste-en.wav` for a small English ASR functional test and a repeatable +file-based benchmark. Its expected transcript is stored in `teste-en.txt`: + +```text +Ask not what your country can do for you. Ask what you can do for your country. +``` + +## Publication status: pending license review + +The proposed source fixture is a maintainer-local WAV. Its original source and redistribution +license could not be confirmed from the available repository history or file metadata. Therefore +`teste-en.wav` is deliberately **not included** in this fork and must not be committed, released, +or represented as redistributable until a manual license review has approved it. + +The scripts already expect this path: + +```text +test_files/fork/asr/teste-en.wav +``` + +After approval, copy the original file byte-for-byte and compare SHA-256 values before adding it. +The maintainer-local candidate measured as follows; these values are provided for manual review, +not as a redistribution grant: + +| Property | Candidate value | +| --- | --- | +| WAV encoding | PCM signed 16-bit little-endian | +| Sample rate | 24,000 Hz | +| Channels | 1 (mono) | +| Duration | 3.845083 s | +| SHA-256 | `148B936B43CE7C546A866E64DA059F0458AEE2D65E617F16E9D94F06E8D99ED6` | +| Origin | Maintainer-local candidate; source/license pending review | + +It is intended only for functional testing and reproducible benchmarking once its legal status is +verified. Do not invent a license for this recording. A reviewer should document the actual source, +license, and any attribution requirements before it is added to the repository. diff --git a/test_files/fork/asr/teste-en.txt b/test_files/fork/asr/teste-en.txt new file mode 100644 index 0000000..0e644f4 --- /dev/null +++ b/test_files/fork/asr/teste-en.txt @@ -0,0 +1 @@ +Ask not what your country can do for you. Ask what you can do for your country. From 0186287b2699d3400d3d60c50773b56d0e6e3828 Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:25:43 -0300 Subject: [PATCH 02/10] feat(examples): add persistent HTTP microphone client --- examples/python/microphone_http.py | 229 ++++++++++++++++++++ examples/python/requirements-microphone.txt | 3 + scripts/windows/run-pascal-server.ps1 | 28 +++ scripts/windows/setup-microphone-client.ps1 | 24 ++ 4 files changed, 284 insertions(+) create mode 100644 examples/python/microphone_http.py create mode 100644 examples/python/requirements-microphone.txt create mode 100644 scripts/windows/run-pascal-server.ps1 create mode 100644 scripts/windows/setup-microphone-client.ps1 diff --git a/examples/python/microphone_http.py b/examples/python/microphone_http.py new file mode 100644 index 0000000..53071c8 --- /dev/null +++ b/examples/python/microphone_http.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Record microphone audio and send it to a persistent NeMo-Speech.cpp HTTP server.""" + +from __future__ import annotations + +import argparse +import io +import json +import queue +import sys +import time +import wave +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +import numpy as np +import requests +import sounddevice as sd + +SAMPLE_RATE = 16_000 +CHANNELS = 1 +MIN_DURATION_SECONDS = 0.08 +SILENCE_RMS_THRESHOLD = 1e-5 + + +@dataclass(slots=True) +class RecordedAudio: + samples: np.ndarray + duration_seconds: float + rms: float + capture_ms: float + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="http://127.0.0.1:8081/v1/audio/transcriptions") + parser.add_argument("--language", default="en") + parser.add_argument("--device", help="Input-device index or a case-insensitive part of its name.") + parser.add_argument("--list-devices", action="store_true", help="List input devices and exit.") + parser.add_argument("--show-words", action="store_true", help="Print word timestamps when present.") + parser.add_argument("--timeout", type=float, default=120.0, help="HTTP timeout in seconds.") + return parser + + +def input_devices() -> list[tuple[int, dict[str, Any]]]: + return [ + (index, dict(device)) + for index, device in enumerate(sd.query_devices()) + if int(device["max_input_channels"]) > 0 + ] + + +def list_devices() -> None: + for index, device in input_devices(): + print(f"{index}: {device['name']} ({device['max_input_channels']} input channel(s))") + + +def resolve_device(selector: str | None) -> tuple[int | None, str]: + if selector is None: + return None, "default" + devices = input_devices() + try: + index = int(selector) + except ValueError: + matches = [(i, d) for i, d in devices if selector.casefold() in str(d["name"]).casefold()] + if not matches: + raise ValueError(f"No input device contains {selector!r}. Use --list-devices.") + if len(matches) > 1: + choices = ", ".join(f"{i}: {d['name']}" for i, d in matches) + raise ValueError(f"More than one input device matches {selector!r}: {choices}. Use its index.") + index, device = matches[0] + return index, str(device["name"]) + for available_index, device in devices: + if available_index == index: + return index, str(device["name"]) + raise ValueError(f"Input device index {index} is unavailable. Use --list-devices.") + + +def calculate_rms(samples: np.ndarray) -> float: + return float(np.sqrt(np.mean(np.square(samples, dtype=np.float64)))) if samples.size else 0.0 + + +def record_until_enter(device: int | None) -> RecordedAudio: + blocks: queue.Queue[np.ndarray] = queue.Queue() + + def callback(input_data: np.ndarray, frames: int, time_info: Any, status: sd.CallbackFlags) -> None: + del frames, time_info + if status: + print(f"Microphone warning: {status}", file=sys.stderr) + blocks.put(input_data[:, 0].copy()) + + input("Press Enter to start recording...") + started = time.perf_counter() + with sd.InputStream(samplerate=SAMPLE_RATE, channels=CHANNELS, dtype="float32", device=device, callback=callback): + input("Recording. Press Enter again to stop...") + capture_ms = (time.perf_counter() - started) * 1000 + captured = [blocks.get_nowait() for _ in range(blocks.qsize())] + samples = np.ascontiguousarray(np.concatenate(captured), dtype=np.float32) if captured else np.empty(0, dtype=np.float32) + return RecordedAudio(samples, samples.size / SAMPLE_RATE, calculate_rms(samples), capture_ms) + + +def encode_wav(samples: np.ndarray) -> tuple[bytes, float]: + started = time.perf_counter() + safe = np.nan_to_num(samples, nan=0.0, posinf=0.0, neginf=0.0) + peak = float(np.max(np.abs(safe))) if safe.size else 0.0 + if peak > 1.0: + safe = safe / peak + pcm16 = np.clip(safe * 32767.0, -32768, 32767).astype(" str: + parts = urlsplit(transcription_url) + return urlunsplit((parts.scheme, parts.netloc, "/ready", "", "")) + + +def check_server(url: str, timeout: float) -> None: + response = requests.get(ready_url(url), timeout=min(timeout, 5.0)) + response.raise_for_status() + + +def request_transcription(session: requests.Session, url: str, wav_data: bytes, language: str, timeout: float) -> tuple[dict[str, Any], float]: + started = time.perf_counter() + response = session.post(url, files={"file": ("microphone.wav", wav_data, "audio/wav")}, data={"model": "default", "language": language, "response_format": "verbose_json"}, timeout=timeout) + elapsed_ms = (time.perf_counter() - started) * 1000 + response.raise_for_status() + try: + return response.json(), elapsed_ms + except json.JSONDecodeError as error: + raise RuntimeError(f"Server response was not JSON: {response.text}") from error + + +def result_words(payload: dict[str, Any]) -> list[dict[str, Any]]: + words = payload.get("words") + if isinstance(words, list): + return [word for word in words if isinstance(word, dict)] + collected: list[dict[str, Any]] = [] + for segment in payload.get("segments", []): + if isinstance(segment, dict) and isinstance(segment.get("words"), list): + collected.extend(word for word in segment["words"] if isinstance(word, dict)) + return collected + + +def print_result(audio: RecordedAudio, preparation_ms: float, request_ms: float, payload: dict[str, Any], show_words: bool) -> None: + text = str(payload.get("text", "")).strip() + print("\n" + "=" * 64) + print(f'Text: "{text}"') + print("\nTimings:") + print(f" Capture: {audio.capture_ms:.2f} ms") + print(f" In-memory preparation: {preparation_ms:.2f} ms") + print(f" HTTP + inference: {request_ms:.2f} ms") + print(f" Total request: {request_ms:.2f} ms") + if audio.duration_seconds and request_ms: + rtf = (request_ms / 1000) / audio.duration_seconds + print(f" RTF: {rtf:.4f}") + print(f" Speed: {1 / rtf:.3f}x realtime") + if payload.get("duration") is not None: + print(f" Server duration: {float(payload['duration']):.3f} s") + if payload.get("language"): + print(f" Returned language: {payload['language']}") + if show_words: + words = result_words(payload) + if words: + print("\nWords:") + for word in words: + token = str(word.get("word", word.get("text", ""))).strip() + start, end = word.get("start"), word.get("end") + timing = f"{float(start):.3f}–{float(end):.3f}" if start is not None and end is not None else "unknown time" + confidence = f" | confidence={word['confidence']}" if word.get("confidence") is not None else "" + print(f" {timing} {token}{confidence}") + else: + print("\nWords: not returned by the server.") + print("=" * 64 + "\n") + + +def main() -> int: + args = build_parser().parse_args() + if args.list_devices: + list_devices() + return 0 + try: + device, device_name = resolve_device(args.device) + check_server(args.url, args.timeout) + except (ValueError, requests.RequestException) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print("NeMo-Speech.cpp — HTTP microphone client") + print(f"Server: {args.url}") + print(f"Language: {args.language}") + print(f"Microphone: {device_name}") + print("The model remains loaded in the separate persistent server.") + print("Press Ctrl+C to exit.\n") + session = requests.Session() + try: + while True: + audio = record_until_enter(device) + print(f"\nCaptured duration: {audio.duration_seconds:.3f} s") + print(f"RMS: {audio.rms:.8f}") + if audio.duration_seconds < MIN_DURATION_SECONDS: + print("Audio is too short; ignored.\n") + continue + if audio.rms < SILENCE_RMS_THRESHOLD: + print("Silence detected; ignored.\n") + continue + wav_data, preparation_ms = encode_wav(audio.samples) + try: + payload, request_ms = request_transcription(session, args.url, wav_data, args.language, args.timeout) + print_result(audio, preparation_ms, request_ms, payload, args.show_words) + except requests.HTTPError as error: + body = error.response.text if error.response is not None else str(error) + print(f"HTTP ERROR: {body}\n", file=sys.stderr) + except (RuntimeError, requests.RequestException) as error: + print(f"ERROR: {error}\n", file=sys.stderr) + except KeyboardInterrupt: + print("\nStopped.") + return 0 + finally: + session.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/python/requirements-microphone.txt b/examples/python/requirements-microphone.txt new file mode 100644 index 0000000..5fd460f --- /dev/null +++ b/examples/python/requirements-microphone.txt @@ -0,0 +1,3 @@ +numpy +requests +sounddevice diff --git a/scripts/windows/run-pascal-server.ps1 b/scripts/windows/run-pascal-server.ps1 new file mode 100644 index 0000000..e68c3d4 --- /dev/null +++ b/scripts/windows/run-pascal-server.ps1 @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string]$Model, + [string]$Executable, + [ValidatePattern('^cuda(:[0-9]+)?$')] [string]$Device = 'cuda:0', + [ValidateRange(1, 65535)] [int]$Port = 8081 +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +if (-not $Executable) { $Executable = Join-Path $RepoRoot 'build-pascal-cuda-http\bin\nemo-speech.exe' } +foreach ($item in @(@{ Name = 'Model'; Path = $Model }, @{ Name = 'Executable'; Path = $Executable })) { + if (-not (Test-Path -LiteralPath $item.Path -PathType Leaf)) { throw "$($item.Name) not found: $($item.Path)" } +} +if (Get-NetTCPConnection -LocalPort $Port -State Listen -ErrorAction SilentlyContinue) { throw "Port $Port is already in use. No process was stopped." } +$baseUrl = "http://127.0.0.1:$Port" +$arguments = @('serve', '--asr-model', $Model, '--device', $Device, '--host', '127.0.0.1', '--port', $Port, '--skinny-q8', 'auto', '--suppress-cuda-graph-log') +Write-Host "Server:`n$baseUrl`nReady:`n$baseUrl/ready`nTranscriptions:`n$baseUrl/v1/audio/transcriptions" -ForegroundColor Cyan +Write-Host "`nRun this in another PowerShell after preparing the client:" -ForegroundColor Yellow +Write-Host '& ".\.tools\microphone-client-venv\Scripts\python.exe" `' +Write-Host ' ".\examples\python\microphone_http.py" `' +Write-Host (' --url "{0}/v1/audio/transcriptions" `' -f $baseUrl) +Write-Host ' --language en `' +Write-Host ' --show-words' +Write-Host ('`nRunning: & "{0}" {1}' -f $Executable, (($arguments | ForEach-Object { '"{0}"' -f $_ }) -join ' ')) +& $Executable @arguments +exit $LASTEXITCODE diff --git a/scripts/windows/setup-microphone-client.ps1 b/scripts/windows/setup-microphone-client.ps1 new file mode 100644 index 0000000..b082273 --- /dev/null +++ b/scripts/windows/setup-microphone-client.ps1 @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +[CmdletBinding()] +param([string]$Python) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$Requirements = Join-Path $RepoRoot 'examples\python\requirements-microphone.txt' +$Venv = Join-Path $RepoRoot '.tools\microphone-client-venv' +if (-not $Python) { $Python = (Get-Command python -ErrorAction SilentlyContinue | Select-Object -First 1 -ExpandProperty Source) } +if (-not $Python -or -not (Test-Path -LiteralPath $Python -PathType Leaf)) { throw 'Python 3.11 or newer was not found. Pass -Python with its executable path.' } +& $Python -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)" +if ($LASTEXITCODE -ne 0) { throw 'Python 3.11 or newer is required.' } +& $Python -m venv $Venv +if ($LASTEXITCODE -ne 0) { throw 'Unable to create the virtual environment.' } +$VenvPython = Join-Path $Venv 'Scripts\python.exe' +& $VenvPython -m pip install --upgrade pip +if ($LASTEXITCODE -ne 0) { throw 'Unable to upgrade pip.' } +& $VenvPython -m pip install -r $Requirements +if ($LASTEXITCODE -ne 0) { throw 'Unable to install microphone client dependencies.' } +Write-Host 'Microphone client environment is ready.' -ForegroundColor Green +Write-Host '& ".\.tools\microphone-client-venv\Scripts\python.exe" `' +Write-Host ' ".\examples\python\microphone_http.py" `' +Write-Host ' --language en `' +Write-Host ' --show-words' From 012307bc68b4020dae3e69c05a728c3ca6b6015d Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:25:49 -0300 Subject: [PATCH 03/10] docs: document Pascal audio and microphone testing --- README.md | 113 +++++++++++++ docs/pascal-fork-overview.md | 190 ++++++++++++++++++++++ docs/testing-with-audio-and-microphone.md | 122 ++++++++++++++ 3 files changed, 425 insertions(+) create mode 100644 docs/pascal-fork-overview.md create mode 100644 docs/testing-with-audio-and-microphone.md diff --git a/README.md b/README.md index 1e63d9d..d12290a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,123 @@ # NeMo-Speech.cpp +> [!NOTE] +> This is an unofficial community fork of NeMo-Speech.cpp. It is not affiliated with, +> maintained by, or officially supported by NVIDIA. + +The original code belongs to the [NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp) +project. This fork preserves the original notices, credits, and licenses; its fork-specific +credits apply only to the modifications, tests, scripts, and documentation added here. + +## About this fork + +This fork was started to improve compatibility, configuration, and day-to-day usability of +NeMo-Speech.cpp on NVIDIA Pascal GPUs, especially the GeForce GTX 10 series. Initial development +and validation used an NVIDIA GeForce GTX 1060 6 GB (Compute Capability 6.1). + +Its initial target family includes GTX 1050, GTX 1050 Ti, GTX 1060, GTX 1070, GTX 1080, and +GTX 1080 Ti. So far, practical testing has been performed only on the GTX 1060 6 GB; community +validation is required before claiming support for the other Pascal GPUs. + +It adds safe runtime controls and clearer diagnostics. It does **not** yet include a new kernel +optimized specifically for Pascal. + +### Fork-specific author and maintenance + +The fork-specific changes, tests, and documentation were made by: + +- **GitHub:** [UNDER192103](https://github.com/UNDER192103) +- **Name/project:** Under Nouzen + +This attribution does not apply to the original NeMo-Speech.cpp codebase. + +### Initial fork changes + +- `--skinny-q8 auto|on|off` runtime control. +- Automatic CUDA Compute Capability detection. +- Safe fallback for GPUs below SM 8.0, plus a controlled error if an incompatible Skinny Q8 + mode is forced. +- `--suppress-cuda-graph-log` to selectively hide the repeated CUDA Graph architecture message. +- Windows build and execution scripts, plus Pascal/GTX 1060 documentation. + +`--skinny-q8 auto` does not add a Pascal kernel: it disables the incompatible Skinny Q8 path and +uses the existing CUDA fallback. `--suppress-cuda-graph-log` does not enable CUDA Graphs and does +not make inference faster. Neither change alters model precision, model contents, or transcription +math. + +### Tested environment and preliminary observations + +- OS: Windows 11 +- GPU: NVIDIA GeForce GTX 1060 6 GB (Pascal, Compute Capability 6.1) +- CPU: Intel Xeon E5-2660 v2; RAM: 32 GB; CUDA Toolkit: 12.6 +- Model: Nemotron 3.5 ASR Streaming 0.6B Q8 GGUF +- Mode: persistent HTTP server + +Manual observations with the microphone client and persistent HTTP server included complete +phrases at 27.035x and 27.220x realtime, and shorter captures from 9.601x to 20.497x realtime. +They were verified manually by UNDER192103 / Under Nouzen. See +[Pascal fork overview](docs/pascal-fork-overview.md) for the complete results table, context, +limitations, and reproduction notes. + +> [!IMPORTANT] +> These are preliminary observations, not a proven percentage improvement over upstream. The old +> and current runs did not necessarily use the same WAV input, duration, or controlled benchmark +> protocol. A repeatable comparison with warm-up, multiple runs, median, and P95 is still needed. +> The persistent server and CUDA execution are the main causes of the observed low latency; the new +> flags primarily improve compatibility, safety, diagnostics, usability, and log readability. + +### Current status + +Validated on the GTX 1060 6 GB: CUDA SM 6.1 build, file transcription, persistent HTTP server, +`/ready`, `/v1/audio/transcriptions`, CPU execution, CUDA execution, automatic Skinny Q8 fallback, +the controlled `--skinny-q8 on` error, and selective CUDA Graph log suppression. + +Not yet implemented: a Pascal-specific Q8 kernel, DP4A optimization, CUDA Graphs on Pascal, +testing on other GTX 10 GPUs, or a controlled reproducible upstream-versus-fork benchmark. + +Example server command (paths are intentionally generic): + +```powershell +.\build\bin\nemo-speech.exe serve ` + --asr-model "C:\Models\nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" ` + --gpu 0 ` + --host 127.0.0.1 ` + --port 8081 ` + --skinny-q8 auto ` + --suppress-cuda-graph-log +``` + +Community testing on Pascal GPUs is welcome. Please report GPU model, Compute Capability, operating +system, CUDA Toolkit, build command, GGUF model, audio duration, latency, logs, and transcription +result. Do not publish licensed models or protected audio. + +## Reproducible test audio + +The fork test fixture is intended to live at `test_files/fork/asr/teste-en.wav`, with the expected +transcript in [`test_files/fork/asr/teste-en.txt`](test_files/fork/asr/teste-en.txt): + +```text +Ask not what your country can do for you. Ask what you can do for your country. +``` + +The WAV itself is currently **not included** because its redistribution license still needs manual +review. Do not publish it until that review is complete. Once a reviewed copy is present, the three +main commands are: + +```powershell +.\scripts\windows\test-pascal-wav.ps1 -Model "C:\Models\nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" +.\scripts\windows\run-pascal-server.ps1 -Model "C:\Models\nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" +.\scripts\windows\test-http-wav.ps1 +``` + +For the microphone client, setup, WAV metadata, license-review checklist, and complete testing +workflow, see [Testing with audio and microphone](docs/testing-with-audio-and-microphone.md). + A lightweight native C++ runtime for NVIDIA Nemotron Speech models built on ggml. Runs speech models in realtime and in batch mode across platforms/backends. ## Contents - [Installation](#installation) +- [About this fork](#about-this-fork) - [Quick start](#quick-start) - [Command line](#command-line) - [Local server and playground](#local-server-and-playground) diff --git a/docs/pascal-fork-overview.md b/docs/pascal-fork-overview.md new file mode 100644 index 0000000..25abbf4 --- /dev/null +++ b/docs/pascal-fork-overview.md @@ -0,0 +1,190 @@ +# Pascal community fork overview + +> [!NOTE] +> This is an unofficial community fork of NeMo-Speech.cpp. It is not affiliated with, +> maintained by, or officially supported by NVIDIA. + +The original code, authorship, notices, credits, and license remain those of the +[NeMo-Speech.cpp project](https://github.com/NVIDIA/NeMo-Speech.cpp). The attribution below is +limited to the modifications, tests, scripts, and documentation introduced by this fork. + +## Fork-specific author and maintenance + +The fork-specific work was carried out by: + +- **GitHub:** [UNDER192103](https://github.com/UNDER192103) +- **Name/project:** Under Nouzen + +## Objective + +This fork was started to improve the compatibility, configuration, and user experience of +NeMo-Speech.cpp on NVIDIA Pascal GPUs. It is initially aimed at the GeForce GTX 10 series: + +```text +GTX 1050 +GTX 1050 Ti +GTX 1060 +GTX 1070 +GTX 1080 +GTX 1080 Ti +``` + +The first hardware used for development and validation was an NVIDIA GeForce GTX 1060 6 GB, +Compute Capability 6.1. Practical tests have only been performed on that GTX 1060. Community +validation is still required before claiming operation on any other Pascal GPU. + +The first stage adds safe execution controls, automatic architecture detection, and better +diagnostics. It does not contain a newly designed Pascal-specific kernel. + +## Initial changes + +- Added `--skinny-q8 auto|on|off`. +- Added automatic CUDA Compute Capability detection. +- Added a safe fallback for GPUs below SM 8.0. +- Added a controlled error when Skinny Q8 is forced on incompatible hardware. +- Added `--suppress-cuda-graph-log`. +- Added selective suppression of the repeated CUDA Graph architecture message. +- Preserved other logs, warnings, and errors. +- Added Windows build and execution scripts. +- Added GTX 1060/Pascal-specific documentation. + +### Runtime flags in detail + +`--skinny-q8` is a global CLI option and may appear before or after a subcommand. The value is +validated and converted into an internal process environment setting before the backend is created. + +| Mode | Behavior on a GPU below SM 8.0 | +| --- | --- | +| `auto` | Sets the existing `GGML_SKINNY_Q8=0` fallback and continues. | +| `off` | Explicitly sets `GGML_SKINNY_Q8=0` and continues. | +| `on` | Stops before inference with an actionable compatibility error. | + +For SM 8.0 or newer, `auto` leaves Skinny Q8 available and `on` explicitly enables it. The code +queries CUDA through `ggml_backend_cuda_get_device_compute_capability()` and compares the GGML +architecture identifier with `800`; a GTX 1060 reports `610` (SM 6.1). + +`--skinny-q8 auto` **does not** implement a Pascal Q8 kernel. It selects the existing compatible +CUDA fallback on Pascal. It does not modify the model, precision, or transcription mathematics. + +`--suppress-cuda-graph-log` sets a process-local control that hides only the repeated debug +message explaining why CUDA Graphs are disabled on pre-Ampere GPUs. It does not enable CUDA Graphs, +change kernels, affect memory use, or improve inference speed. + +## Tested environment + +| Item | Value | +| --- | --- | +| Operating system | Windows 11 | +| GPU | NVIDIA GeForce GTX 1060 6 GB | +| Architecture | Pascal | +| Compute Capability | 6.1 | +| CPU | Intel Xeon E5-2660 v2 | +| RAM | 32 GB | +| CUDA Toolkit | 12.6 | +| Model | Nemotron 3.5 ASR Streaming 0.6B Q8 GGUF | +| Mode | Persistent HTTP server | + +## Preliminary observed results + +The following runs were manually verified by **UNDER192103 / Under Nouzen**, using the microphone +client and a persistent HTTP server. + +| Observed runtime/test | Audio duration | HTTP + inference | RTF | Speed | +| --- | ---: | ---: | ---: | ---: | +| Previous configuration | approximately 2.2 s | approximately 114–116 ms | not recorded | approximately 19–20x | +| Custom — complete phrase | 2.054 s | 75.98 ms | 0.0370 | 27.035x | +| Custom — complete phrase | 1.976 s | 72.59 ms | 0.0367 | 27.220x | +| Custom — short speech | 1.040 s | 50.74 ms | 0.0488 | 20.497x | +| Custom — very short capture | 0.650 s | 67.70 ms | 0.1042 | 9.601x | + +Recognized text in the complete-phrase tests: + +```text +Olá! Como que você está hoje? +``` + +> [!IMPORTANT] +> These are real observations from development, but the previous and current tests did not +> necessarily use the same WAV input, audio duration, or identical formal test suite. The +> difference between approximately 114–116 ms and approximately 72–76 ms must therefore be treated +> as preliminary, not as a proven percentage improvement. +> +> A controlled benchmark using the same audio file, warm-up, multiple repetitions, median, and P95 +> will be published separately. Do not describe this fork as “40% faster” until that experiment +> exists. + +The main contributors to low observed latency are CUDA execution and keeping the model loaded in a +persistent server. The new flags are primarily improvements to compatibility, safety, diagnostics, +ease of execution, and log readability. + +## Reproduction + +Build a CUDA configuration compatible with the target hardware, then run a persistent local +server. This public example deliberately uses generic paths: + +```powershell +.\build\bin\nemo-speech.exe serve ` + --asr-model "C:\Models\nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" ` + --gpu 0 ` + --host 127.0.0.1 ` + --port 8081 ` + --skinny-q8 auto ` + --suppress-cuda-graph-log +``` + +With the server running, send audio to the local transcription endpoint: + +```text +http://127.0.0.1:8081/v1/audio/transcriptions +``` + +Use `--skinny-q8 auto` for the portable, safe choice. On the tested GTX 1060, expected startup +diagnostics include a CUDA backend selection and the automatic `skinny-q8=off` fallback due to +Compute Capability 6.1. + +## Current status + +Validated on the GTX 1060 6 GB: + +- CUDA SM 6.1 build; +- file transcription; +- persistent HTTP server; +- `/ready` endpoint; +- `/v1/audio/transcriptions` endpoint; +- CPU execution; +- CUDA execution; +- automatic Skinny Q8 fallback; +- controlled error for `--skinny-q8 on`; +- selective CUDA Graph log suppression. + +Not yet implemented: + +- Pascal-specific Q8 kernel; +- DP4A-based optimization; +- CUDA Graphs for Pascal; +- tests on other GTX 10 GPUs; +- formal reproducible benchmark between upstream and this fork. + +## Community contributions + +Pascal users are invited to test and report: + +- GPU model and Compute Capability; +- operating system and CUDA Toolkit version; +- build command; +- GGUF model; +- audio duration and measured latency; +- relevant logs; and +- transcription result. + +Please do not publish models or audio protected by licenses. A report that includes the same test +audio, warm-up conditions, repeat count, median, and P95 is especially useful for the planned +controlled benchmark. + +## Next steps + +1. Create a reproducible upstream-versus-fork benchmark protocol. +2. Gather community validation across the GTX 10 family. +3. Investigate Pascal-appropriate optimizations, including DP4A, only after measurement identifies + a meaningful bottleneck. +4. Keep all compatibility behavior explicit and safe for unsupported GPU architectures. diff --git a/docs/testing-with-audio-and-microphone.md b/docs/testing-with-audio-and-microphone.md new file mode 100644 index 0000000..5f1f93f --- /dev/null +++ b/docs/testing-with-audio-and-microphone.md @@ -0,0 +1,122 @@ +# Testing with audio and microphone + +This guide covers the Pascal-oriented Windows workflow: build the runtime, test one fixed WAV, +start a persistent HTTP server, and use the separate microphone client. The microphone client does +not load a model; the server owns the loaded model. + +## Test-fixture licensing status + +The expected fixture path is `test_files/fork/asr/teste-en.wav`, but the WAV is intentionally not +present yet. Its candidate source and redistribution license require manual review. Do not add, +commit, release, or claim redistribution rights for that audio until the review documents the real +source and license. The expected transcript is already present in `teste-en.txt`. + +When an approved copy is available, copy it without modifying its bytes and compare SHA-256 before +adding it. The proposed candidate hash is +`148B936B43CE7C546A866E64DA059F0458AEE2D65E617F16E9D94F06E8D99ED6`; see the fixture +[README](../test_files/fork/asr/README.md) for its measured format and the pending-review notice. + +## 1. Compilar + +```powershell +.\scripts\windows\build-pascal.ps1 +``` + +This creates `build-pascal-cuda-http\bin\nemo-speech.exe` with CUDA architecture 6.1 and the HTTP +server. It requires the Windows build prerequisites documented in `scripts/windows/build.ps1`. + +## 2. Testar por arquivo + +```powershell +.\scripts\windows\test-pascal-wav.ps1 ` + -Model "C:\Models\nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" +``` + +The script uses the real CLI options `--device cuda:0`, `--skinny-q8 auto`, and +`--suppress-cuda-graph-log`. Use `-Device cpu` for a CPU run or `-Executable` to point to another +compiled binary. + +## 3. Preparar cliente de microfone + +```powershell +.\scripts\windows\setup-microphone-client.ps1 +``` + +It creates `.tools\microphone-client-venv` and installs only `numpy`, `requests`, and +`sounddevice`. It does not install PyTorch, NeMo, or CUDA. + +## 4. Iniciar servidor + +```powershell +.\scripts\windows\run-pascal-server.ps1 ` + -Model "C:\Models\nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" +``` + +Leave this PowerShell open. The process starts a local server at `http://127.0.0.1:8081`, exposes +`/ready` and `/v1/audio/transcriptions`, and keeps the model loaded between requests. + +## 5. Testar o WAV via HTTP + +```powershell +.\scripts\windows\test-http-wav.ps1 +``` + +The script checks `/ready`, posts the WAV as multipart form data with `response_format=verbose_json`, +formats the response, measures HTTP-plus-inference time, and prints the expected and returned text +for visual comparison. It uses `curl.exe` and does not require Python. + +## 6. Testar microfone em outro PowerShell + +```powershell +& ".\.tools\microphone-client-venv\Scripts\python.exe" ` + ".\examples\python\microphone_http.py" ` + --language en ` + --show-words +``` + +Press Enter to start capture and Enter again to stop. The client captures 16 kHz mono audio, creates +an in-memory PCM16 WAV, posts it to the persistent server, and prints capture, preparation, +HTTP-plus-inference, RTF, and realtime-speed measurements. It does not save microphone audio by default. + +## 7. Listar microfones + +```powershell +& ".\.tools\microphone-client-venv\Scripts\python.exe" ` + ".\examples\python\microphone_http.py" ` + --list-devices +``` + +## 8. Escolher dispositivo + +```powershell +& ".\.tools\microphone-client-venv\Scripts\python.exe" ` + ".\examples\python\microphone_http.py" ` + --device "Microphone" ` + --language en ` + --show-words +``` + +`--device` accepts an input-device index or a case-insensitive part of its name. If the name matches +more than one device, the client asks for an index. Other client controls are `--url` and +`--timeout` (default: 120 seconds). + +## File benchmark + +After the reviewed WAV exists, run: + +```powershell +.\scripts\windows\benchmark-pascal-wav.ps1 ` + -Model "C:\Models\nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" ` + -Runs 10 +``` + +It performs an excluded warm-up, records wall-clock time for each subsequent run, and saves JSON +and Markdown with minimum, maximum, mean, median, P95, hardware, model, commit, and command in +`benchmark-results/`. That directory is ignored by Git. + +## Publishing checklist + +Before publishing the WAV, confirm its actual source and redistribution license, preserve its bytes, +compare the SHA-256 hash, and document any required attribution. Do not add models, builds, +executables, virtual environments, caches, benchmark output, personal paths, or additional audio to +the repository. From 4a22bcd41529bb3585c38b5f86c8ccdf73d6ad1c Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:47:43 -0300 Subject: [PATCH 04/10] docs: document Pascal short and long latency observations --- README.md | 32 ++++-- docs/README.md | 2 + docs/pascal-fork-overview.md | 46 +++----- docs/pascal-performance-observations.md | 133 ++++++++++++++++++++++ docs/testing-with-audio-and-microphone.md | 12 ++ scripts/windows/benchmark-short-wav.ps1 | 100 ++++++++++++++++ test_files/fork/asr/README.md | 7 ++ 7 files changed, 288 insertions(+), 44 deletions(-) create mode 100644 docs/pascal-performance-observations.md create mode 100644 scripts/windows/benchmark-short-wav.ps1 diff --git a/README.md b/README.md index d12290a..6dc27c6 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ uses the existing CUDA fallback. `--suppress-cuda-graph-log` does not enable CUD not make inference faster. Neither change alters model precision, model contents, or transcription math. -### Tested environment and preliminary observations +### Tested environment - OS: Windows 11 - GPU: NVIDIA GeForce GTX 1060 6 GB (Pascal, Compute Capability 6.1) @@ -52,18 +52,25 @@ math. - Model: Nemotron 3.5 ASR Streaming 0.6B Q8 GGUF - Mode: persistent HTTP server -Manual observations with the microphone client and persistent HTTP server included complete -phrases at 27.035x and 27.220x realtime, and shorter captures from 9.601x to 20.497x realtime. -They were verified manually by UNDER192103 / Under Nouzen. See -[Pascal fork overview](docs/pascal-fork-overview.md) for the complete results table, context, -limitations, and reproduction notes. +## Pascal performance observations -> [!IMPORTANT] -> These are preliminary observations, not a proven percentage improvement over upstream. The old -> and current runs did not necessarily use the same WAV input, duration, or controlled benchmark -> protocol. A repeatable comparison with warm-up, multiple runs, median, and P95 is still needed. -> The persistent server and CUDA execution are the main causes of the observed low latency; the new -> flags primarily improve compatibility, safety, diagnostics, usability, and log readability. +On the tested GTX 1060 6 GB, the custom runtime showed performance similar to the default runtime +for the included 11-second JFK sample. + +For manually recorded short requests around two to three seconds, the custom runtime showed +substantially lower latency in the local test environment, typically around 72–80 ms, while the +default runtime was generally above 130 ms and showed larger latency spikes. + +These short-request results are preliminary. A fully reproducible benchmark using the same short +English WAV is being prepared, pending a redistributable fixture. + +| Test | Default median | Custom median | Observation | +| --- | ---: | ---: | --- | +| 11-second JFK WAV | 181.41 ms | 178.26 ms | Similar performance | +| Short local speech | 195.10 ms | 73.96 ms | Large preliminary difference | + +See [Pascal performance observations](docs/pascal-performance-observations.md) for the complete +methodology, raw values, limitations, and reproduction instructions. ### Current status @@ -230,6 +237,7 @@ Windows, and container instructions are in | [Client integration](docs/clients.md) | OpenAI SDKs, curl, and Riva gRPC clients | | [Troubleshooting](docs/troubleshooting.md) | `doctor` output and common runtime failures | | [Build from source](docs/build.md) | Presets, optional components, dependencies, containers, and artifacts | +| [Pascal performance](docs/pascal-performance-observations.md) | GTX 1060 short/long latency observations and reproducible benchmark instructions | | [All documentation](docs/README.md) | ASR, TTS, NMT, configuration, and developer references | ## License diff --git a/docs/README.md b/docs/README.md index 8e1b423..b40ee62 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,8 @@ Start with: - [Native SDK integration](sdk.md) - [Troubleshooting](troubleshooting.md) - [Build from source](build.md) +- [Pascal performance observations](pascal-performance-observations.md) - GTX 1060 short/long + latency observations and the planned reproducible short-WAV benchmark. ## ASR diff --git a/docs/pascal-fork-overview.md b/docs/pascal-fork-overview.md index 25abbf4..6eb63a1 100644 --- a/docs/pascal-fork-overview.md +++ b/docs/pascal-fork-overview.md @@ -84,38 +84,19 @@ change kernels, affect memory use, or improve inference speed. | Model | Nemotron 3.5 ASR Streaming 0.6B Q8 GGUF | | Mode | Persistent HTTP server | -## Preliminary observed results +## Performance observations -The following runs were manually verified by **UNDER192103 / Under Nouzen**, using the microphone -client and a persistent HTTP server. +The source of truth for the current numerical results is +[Pascal performance observations](pascal-performance-observations.md). It records both the +reproducible 11-second JFK comparison and the preliminary two-to-three-second microphone comparison, +including raw values, methodology, and limitations. -| Observed runtime/test | Audio duration | HTTP + inference | RTF | Speed | -| --- | ---: | ---: | ---: | ---: | -| Previous configuration | approximately 2.2 s | approximately 114–116 ms | not recorded | approximately 19–20x | -| Custom — complete phrase | 2.054 s | 75.98 ms | 0.0370 | 27.035x | -| Custom — complete phrase | 1.976 s | 72.59 ms | 0.0367 | 27.220x | -| Custom — short speech | 1.040 s | 50.74 ms | 0.0488 | 20.497x | -| Custom — very short capture | 0.650 s | 67.70 ms | 0.1042 | 9.601x | - -Recognized text in the complete-phrase tests: - -```text -Olá! Como que você está hoje? -``` - -> [!IMPORTANT] -> These are real observations from development, but the previous and current tests did not -> necessarily use the same WAV input, audio duration, or identical formal test suite. The -> difference between approximately 114–116 ms and approximately 72–76 ms must therefore be treated -> as preliminary, not as a proven percentage improvement. -> -> A controlled benchmark using the same audio file, warm-up, multiple repetitions, median, and P95 -> will be published separately. Do not describe this fork as “40% faster” until that experiment -> exists. - -The main contributors to low observed latency are CUDA execution and keeping the model loaded in a -persistent server. The new flags are primarily improvements to compatibility, safety, diagnostics, -ease of execution, and log readability. +The custom runtime is effectively equivalent to default for the measured 11-second sample. The +manual short-request observations are promising but not yet a controlled same-WAV benchmark. The +main expected low-latency factors are persistent serving and CUDA execution; the fork flags are for +compatibility, safety, diagnostics, and log readability. `--suppress-cuda-graph-log` is not a speed +optimization, and `--skinny-q8 auto` selects the existing Pascal-compatible fallback rather than a +new Pascal kernel. ## Reproduction @@ -163,7 +144,7 @@ Not yet implemented: - DP4A-based optimization; - CUDA Graphs for Pascal; - tests on other GTX 10 GPUs; -- formal reproducible benchmark between upstream and this fork. +- formal reproducible same-WAV short-request benchmark between upstream and this fork. ## Community contributions @@ -183,7 +164,8 @@ controlled benchmark. ## Next steps -1. Create a reproducible upstream-versus-fork benchmark protocol. +1. Add a reviewed, redistributable short English WAV and run the reproducible upstream-versus-fork + benchmark protocol. 2. Gather community validation across the GTX 10 family. 3. Investigate Pascal-appropriate optimizations, including DP4A, only after measurement identifies a meaningful bottleneck. diff --git a/docs/pascal-performance-observations.md b/docs/pascal-performance-observations.md new file mode 100644 index 0000000..214b4c2 --- /dev/null +++ b/docs/pascal-performance-observations.md @@ -0,0 +1,133 @@ +# Pascal performance observations + +This document records observed default-versus-custom behavior on one local Pascal system. It is the source of truth for the fork's performance numbers; it does not claim an upstream-wide performance improvement. + +## Hardware used + +| Item | Value | +| --- | --- | +| Operating system | Windows 11 | +| GPU | NVIDIA GeForce GTX 1060 6 GB | +| Architecture | Pascal | +| Compute Capability | 6.1 | +| CPU | Intel Xeon E5-2660 v2 | +| RAM | 32 GB | +| CUDA Toolkit | 12.6 | +| Model | Nemotron 3.5 ASR Streaming 0.6B Q8 GGUF | +| Server | Persistent HTTP server | +| Endpoint | `/v1/audio/transcriptions` | + +All tests and observations below were run and manually verified by **UNDER192103 / Under Nouzen** ([GitHub](https://github.com/UNDER192103)). This attribution applies to the fork-specific tests and documentation, not to the original NeMo-Speech.cpp project. + +## Long-audio result + +The reproducible long-audio test used `test_files/asr/wav/test/jfk.wav`. + +| Property | Value | +| --- | --- | +| Duration | 11 seconds | +| Sample rate | 16 kHz | +| Channels | mono | +| Bit depth | 16 bits | +| Warm-up | 1 execution | +| Measured executions | 10 | +| Language | English | + +| Runtime | Minimum | Maximum | Mean | Median | P95 | +| --- | ---: | ---: | ---: | ---: | ---: | +| Default | 178.47 ms | 197.39 ms | 183.00 ms | 181.41 ms | 197.39 ms | +| Custom | 175.08 ms | 193.77 ms | 179.64 ms | 178.26 ms | 193.77 ms | + +The observed median difference is approximately **3.15 ms**, or an observed relative reduction of approximately **1.7%**. This is small and may be within normal system variation. For this 11-second test, default and custom performance were practically equivalent; this result does not support a claim of a significant long-audio speed improvement. + +## Short-speech result + +These preliminary observations came from the HTTP microphone client while repeatedly speaking: + +```text +Olá! Como que você está hoje? +``` + +Each capture was approximately 2.3–2.9 seconds including leading and trailing silence. + +### Default observed + +```text +670.57 ms +195.10 ms +151.69 ms +163.22 ms +620.39 ms +``` + +Minimum: 151.69 ms; maximum: 670.57 ms; median: 195.10 ms. + +### Custom observed + +```text +73.63 ms +77.19 ms +74.29 ms +72.31 ms +``` + +Minimum: 72.31 ms; maximum: 77.19 ms; median: approximately 73.96 ms. + +| Short scenario | Samples | Minimum | Maximum | Median | +| --- | ---: | ---: | ---: | ---: | +| Default | 5 | 151.69 ms | 670.57 ms | 195.10 ms | +| Custom | 4 | 72.31 ms | 77.19 ms | 73.96 ms | + +The preliminary median comparison is 195.10 ms versus 73.96 ms: an observed difference of approximately 121.14 ms and an observed relative reduction of approximately 62%. + +> [!IMPORTANT] +> The short-speech results are preliminary observations made with the local microphone. Although the same phrase, computer, microphone, HTTP client, and model were used, every recording has small differences in duration, silence, intensity, and pronunciation. +> +> Therefore, the approximately 62% reduction represents behavior observed in this environment. It is not yet a scientific benchmark or performance guarantee. + +The microphone does not execute inference. It creates an in-memory mono PCM16 WAV and sends it to the same HTTP endpoint; inference runs entirely in whichever default or custom server is open. The capture interval is not included in the reported `HTTP + inference` time. The pattern was consistent enough to warrant investigation, but it still needs repeated testing with exactly the same short WAV on both runtimes. + +## Current interpretation + +The current evidence suggests that the custom runtime does not significantly change throughput for longer recordings, such as the 11-second JFK sample. + +However, on the tested GTX 1060 6 GB, the custom runtime showed substantially lower and more consistent latency for short requests around two to three seconds. + +This may indicate that the fork changes affect fixed per-request overhead, backend initialization, buffer preparation, kernel selection, synchronization, or another short-request execution path. The exact cause has not yet been isolated. + +Do not attribute the observation to one flag. `--suppress-cuda-graph-log` only suppresses a log; it does not enable CUDA Graphs or improve inference speed. `--skinny-q8 auto` does not introduce a new Pascal kernel: on the GTX 1060 it selects the existing compatible fallback. + +## Reproducible short-WAV benchmark + +The planned fixture is `test_files/fork/asr/short-en.wav`, a two-to-three-second English PCM16, mono, 16 kHz WAV with a documented phrase. It has **not** been added: no appropriate local short English WAV with a verified redistribution license was found. Consequently there is no source, license, duration, transcript, or SHA-256 to publish yet, and the short comparison is not fully reproducible. + +Do not add a personal microphone capture, a Portuguese fixture, or any WAV without confirmed redistribution rights. When a suitable audio source is approved, document its exact transcript in `short-en.txt`, copy it byte-for-byte to `test_files/fork/asr/short-en.wav`, compare SHA-256, and record its source, license, format, duration, and attribution in the fixture README. + +### Benchmark commands + +Start either server yourself; the benchmark deliberately does not start or switch a server. + +```powershell +.\scripts\windows\benchmark-short-wav.ps1 ` + -Runtime default ` + -Model "C:\Models\nemotron.gguf" ` + -Runs 20 +``` + +```powershell +.\scripts\windows\benchmark-short-wav.ps1 ` + -Runtime custom ` + -Model "C:\Models\nemotron.gguf" ` + -Runs 20 +``` + +The script checks `/ready`, warms up once, sends the exact same WAV for every measured request, measures HTTP plus inference only, reports each run plus minimum, maximum, mean, median, P95, RTF, and realtime speed, and writes ignored JSON and Markdown results under `benchmark-results/`. + +The public microphone client can be used separately for exploratory testing: + +```powershell +python .\examples\python\microphone_http.py ` + --url "http://127.0.0.1:8081/v1/audio/transcriptions" ` + --language en ` + --show-words +``` diff --git a/docs/testing-with-audio-and-microphone.md b/docs/testing-with-audio-and-microphone.md index 5f1f93f..1d68b47 100644 --- a/docs/testing-with-audio-and-microphone.md +++ b/docs/testing-with-audio-and-microphone.md @@ -4,6 +4,10 @@ This guide covers the Pascal-oriented Windows workflow: build the runtime, test start a persistent HTTP server, and use the separate microphone client. The microphone client does not load a model; the server owns the loaded model. +For the long and short latency observations, raw measurements, and method limitations, see +[Pascal performance observations](pascal-performance-observations.md). That document is the source +of truth for performance numbers; this guide focuses on the workflow. + ## Test-fixture licensing status The expected fixture path is `test_files/fork/asr/teste-en.wav`, but the WAV is intentionally not @@ -114,6 +118,14 @@ It performs an excluded warm-up, records wall-clock time for each subsequent run and Markdown with minimum, maximum, mean, median, P95, hardware, model, commit, and command in `benchmark-results/`. That directory is ignored by Git. +## Short HTTP benchmark + +The comparison script is `scripts/windows/benchmark-short-wav.ps1`. It sends the same +`test_files/fork/asr/short-en.wav` to an already-running default or custom server and measures only +HTTP plus inference. The fixture is currently pending a verified redistribution license, so the +script intentionally stops until `short-en.wav` is supplied. See +[Pascal performance observations](pascal-performance-observations.md) for usage and methodology. + ## Publishing checklist Before publishing the WAV, confirm its actual source and redistribution license, preserve its bytes, diff --git a/scripts/windows/benchmark-short-wav.ps1 b/scripts/windows/benchmark-short-wav.ps1 new file mode 100644 index 0000000..c0b5138 --- /dev/null +++ b/scripts/windows/benchmark-short-wav.ps1 @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +<#[ +.SYNOPSIS + Benchmark HTTP plus inference for the same short English WAV. + +.DESCRIPTION + Start either the default or custom server first. This script never starts, stops, or swaps a + server; -Runtime records the selected runtime in the result metadata. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string]$Model, + [ValidateSet('default', 'custom')] [string]$Runtime, + [ValidateRange(1, 10000)] [int]$Runs = 20, + [string]$Url = 'http://127.0.0.1:8081/v1/audio/transcriptions', + [string]$Audio +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +if (-not $Audio) { $Audio = Join-Path $RepoRoot 'test_files\fork\asr\short-en.wav' } +if (-not (Test-Path -LiteralPath $Audio -PathType Leaf)) { + throw "Short WAV fixture not found: $Audio. It is intentionally pending a reviewed redistribution license; see test_files\fork\asr\README.md." +} +if (-not (Get-Command curl.exe -ErrorAction SilentlyContinue)) { throw 'curl.exe is required but was not found on PATH.' } + +function Get-WavDurationSeconds([string]$Path) { + $stream = [System.IO.File]::OpenRead($Path) + $reader = [System.IO.BinaryReader]::new($stream) + try { + if ([Text.Encoding]::ASCII.GetString($reader.ReadBytes(4)) -ne 'RIFF') { throw 'Expected a RIFF WAV file.' } + $null = $reader.ReadUInt32() + if ([Text.Encoding]::ASCII.GetString($reader.ReadBytes(4)) -ne 'WAVE') { throw 'Expected a WAVE file.' } + $byteRate = 0 + while ($stream.Position -lt $stream.Length) { + $chunk = [Text.Encoding]::ASCII.GetString($reader.ReadBytes(4)) + $size = [int64]$reader.ReadUInt32() + if ($chunk -eq 'fmt ') { + if ($size -lt 16) { throw 'Invalid fmt chunk.' } + $format = $reader.ReadUInt16(); $channels = $reader.ReadUInt16(); $sampleRate = $reader.ReadUInt32(); $byteRate = $reader.ReadUInt32() + $null = $reader.ReadUInt16(); $bits = $reader.ReadUInt16() + if ($format -ne 1 -or $channels -ne 1 -or $sampleRate -ne 16000 -or $bits -ne 16) { throw 'Expected PCM16 mono 16 kHz WAV.' } + $stream.Position += $size - 16 + ($size % 2) + } elseif ($chunk -eq 'data') { + if ($byteRate -le 0) { throw 'WAV fmt chunk was missing.' } + return $size / [double]$byteRate + } else { + $stream.Position += $size + ($size % 2) + } + } + throw 'WAV data chunk was not found.' + } finally { $reader.Dispose(); $stream.Dispose() } +} +function Get-Percentile([double[]]$Values, [double]$Percentile) { + $ordered = @($Values | Sort-Object); $index = ($ordered.Count - 1) * $Percentile + $lower = [Math]::Floor($index); $upper = [Math]::Ceiling($index) + if ($lower -eq $upper) { return $ordered[$lower] } + return $ordered[$lower] + (($ordered[$upper] - $ordered[$lower]) * ($index - $lower)) +} +function Invoke-HttpInference([string]$RequestUrl, [string]$WavPath) { + $response = New-TemporaryFile + try { + $metric = & curl.exe --silent --show-error --output $response --write-out '%{http_code}|%{time_total}' --form "file=@$WavPath;type=audio/wav" --form 'model=default' --form 'response_format=verbose_json' $RequestUrl + $exitCode = $LASTEXITCODE + $parts = (($metric | Out-String).Trim()) -split '\|', 2 + if ($exitCode -ne 0) { throw "curl.exe exited with $exitCode" } + if ($parts.Count -ne 2 -or [int]$parts[0] -lt 200 -or [int]$parts[0] -ge 300) { throw "HTTP request failed: $($parts -join '|')" } + return [double]$parts[1] * 1000 + } finally { Remove-Item -LiteralPath $response -Force -ErrorAction SilentlyContinue } +} + +$baseUrl = ([uri]$Url).GetLeftPart([System.UriPartial]::Authority) +Write-Host "Checking readiness: $baseUrl/ready" -ForegroundColor Cyan +& curl.exe --silent --show-error --fail "$baseUrl/ready" | Out-Host +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +$durationSeconds = Get-WavDurationSeconds $Audio +Write-Host "Warm-up ($Runtime): $Url" -ForegroundColor Cyan +$warmupMs = Invoke-HttpInference $Url $Audio +Write-Host ("Warm-up completed in {0:N2} ms (excluded)." -f $warmupMs) +$measurements = [System.Collections.Generic.List[double]]::new() +for ($i = 1; $i -le $Runs; $i++) { + $milliseconds = Invoke-HttpInference $Url $Audio + $measurements.Add($milliseconds) + $rtf = ($milliseconds / 1000) / $durationSeconds + Write-Host ("Run {0}/{1}: {2:N2} ms | RTF {3:N4} | {4:N3}x realtime" -f $i, $Runs, $milliseconds, $rtf, (1 / $rtf)) +} + +$values = [double[]]$measurements.ToArray() +$stats = [ordered]@{ minimum_ms = [Math]::Round(($values | Measure-Object -Minimum).Minimum, 3); maximum_ms = [Math]::Round(($values | Measure-Object -Maximum).Maximum, 3); mean_ms = [Math]::Round(($values | Measure-Object -Average).Average, 3); median_ms = [Math]::Round((Get-Percentile $values 0.5), 3); p95_ms = [Math]::Round((Get-Percentile $values 0.95), 3) } +$medianRtf = ($stats.median_ms / 1000) / $durationSeconds +$timestamp = Get-Date -Format 'yyyyMMdd-HHmmss'; $resultsDir = Join-Path $RepoRoot 'benchmark-results'; New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null +$jsonPath = Join-Path $resultsDir "short-wav-$Runtime-$timestamp.json"; $markdownPath = Join-Path $resultsDir "short-wav-$Runtime-$timestamp.md" +$gpu = @(Get-CimInstance Win32_VideoController -ErrorAction SilentlyContinue | ForEach-Object { $_.Name }) +$commit = (& git -C $RepoRoot rev-parse HEAD 2>$null).Trim() +$result = [ordered]@{ timestamp = (Get-Date).ToString('o'); runtime = $Runtime; model = $Model; audio = $Audio; url = $Url; audio_duration_seconds = $durationSeconds; warmup_ms = $warmupMs; runs_ms = $values; statistics = $stats; median_rtf = $medianRtf; median_realtime_speed = (1 / $medianRtf); repository_commit = $commit; gpu = $gpu } +$result | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $jsonPath -Encoding utf8 +$rows = $values | ForEach-Object -Begin { $number = 0 } -Process { $number++; "| $number | $([Math]::Round($_, 3)) |" } +@('# Short WAV HTTP benchmark', '', "- Runtime: $Runtime", "- Model: $Model", "- Audio: $Audio", "- Server: $Url", "- Commit: $commit", "- Audio duration: $([Math]::Round($durationSeconds, 6)) s", "- Warm-up excluded: $warmupMs ms", "- GPU: $($gpu -join '; ')", '', '| Minimum (ms) | Maximum (ms) | Mean (ms) | Median (ms) | P95 (ms) | Median RTF | Speed |', '| ---: | ---: | ---: | ---: | ---: | ---: | ---: |', "| $($stats.minimum_ms) | $($stats.maximum_ms) | $($stats.mean_ms) | $($stats.median_ms) | $($stats.p95_ms) | $([Math]::Round($medianRtf, 4)) | $([Math]::Round((1 / $medianRtf), 3))x |", '', '## Runs', '', '| Run | HTTP + inference (ms) |', '| ---: | ---: |') + $rows | Set-Content -LiteralPath $markdownPath -Encoding utf8 +Write-Host "Saved JSON: $jsonPath" -ForegroundColor Green +Write-Host "Saved Markdown: $markdownPath" -ForegroundColor Green diff --git a/test_files/fork/asr/README.md b/test_files/fork/asr/README.md index a186d42..c016a6e 100644 --- a/test_files/fork/asr/README.md +++ b/test_files/fork/asr/README.md @@ -36,3 +36,10 @@ not as a redistribution grant: It is intended only for functional testing and reproducible benchmarking once its legal status is verified. Do not invent a license for this recording. A reviewer should document the actual source, license, and any attribution requirements before it is added to the repository. + +## Short English benchmark fixture + +`short-en.wav` is reserved for the two-to-three-second HTTP latency benchmark. No suitable short +English WAV with a verified redistribution license was found in the local project material, so it is +not included and `short-en.txt` is intentionally not created. Before adding one, document its exact +phrase, duration, PCM16/16 kHz/mono format, SHA-256, source, license, and attribution here. From c7e20dd00c0ddb7008a0a7b7f591feec34c19d37 Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:51:21 -0300 Subject: [PATCH 05/10] docs: correct short benchmark sample attribution --- README.md | 8 ++++---- docs/pascal-performance-observations.md | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 6dc27c6..4a0ed7b 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,9 @@ math. On the tested GTX 1060 6 GB, the custom runtime showed performance similar to the default runtime for the included 11-second JFK sample. -For manually recorded short requests around two to three seconds, the custom runtime showed -substantially lower latency in the local test environment, typically around 72–80 ms, while the -default runtime was generally above 130 ms and showed larger latency spikes. +For manually recorded short requests around two to three seconds, four of the custom observations +were around 72–80 ms, while one first custom observation was 620.39 ms. The default observations +were generally above 130 ms and also included a large latency spike. These short-request results are preliminary. A fully reproducible benchmark using the same short English WAV is being prepared, pending a redistributable fixture. @@ -67,7 +67,7 @@ English WAV is being prepared, pending a redistributable fixture. | Test | Default median | Custom median | Observation | | --- | ---: | ---: | --- | | 11-second JFK WAV | 181.41 ms | 178.26 ms | Similar performance | -| Short local speech | 195.10 ms | 73.96 ms | Large preliminary difference | +| Short local speech | 179.16 ms | 74.29 ms | Large preliminary median difference | See [Pascal performance observations](docs/pascal-performance-observations.md) for the complete methodology, raw values, limitations, and reproduction instructions. diff --git a/docs/pascal-performance-observations.md b/docs/pascal-performance-observations.md index 214b4c2..ac7c765 100644 --- a/docs/pascal-performance-observations.md +++ b/docs/pascal-performance-observations.md @@ -57,33 +57,33 @@ Each capture was approximately 2.3–2.9 seconds including leading and trailing 195.10 ms 151.69 ms 163.22 ms -620.39 ms ``` -Minimum: 151.69 ms; maximum: 670.57 ms; median: 195.10 ms. +Minimum: 151.69 ms; maximum: 670.57 ms; median: 179.16 ms. ### Custom observed ```text +620.39 ms 73.63 ms 77.19 ms 74.29 ms 72.31 ms ``` -Minimum: 72.31 ms; maximum: 77.19 ms; median: approximately 73.96 ms. +Minimum: 72.31 ms; maximum: 620.39 ms; median: 74.29 ms. The 620.39 ms reading was the first custom observation. | Short scenario | Samples | Minimum | Maximum | Median | | --- | ---: | ---: | ---: | ---: | -| Default | 5 | 151.69 ms | 670.57 ms | 195.10 ms | -| Custom | 4 | 72.31 ms | 77.19 ms | 73.96 ms | +| Default | 4 | 151.69 ms | 670.57 ms | 179.16 ms | +| Custom | 5 | 72.31 ms | 620.39 ms | 74.29 ms | -The preliminary median comparison is 195.10 ms versus 73.96 ms: an observed difference of approximately 121.14 ms and an observed relative reduction of approximately 62%. +The preliminary median comparison is 179.16 ms versus 74.29 ms: an observed difference of approximately 104.87 ms and an observed relative reduction of approximately 58.5%. > [!IMPORTANT] > The short-speech results are preliminary observations made with the local microphone. Although the same phrase, computer, microphone, HTTP client, and model were used, every recording has small differences in duration, silence, intensity, and pronunciation. > -> Therefore, the approximately 62% reduction represents behavior observed in this environment. It is not yet a scientific benchmark or performance guarantee. +> Therefore, the approximately 58.5% median reduction represents behavior observed in this environment. It is not yet a scientific benchmark or performance guarantee. The microphone does not execute inference. It creates an in-memory mono PCM16 WAV and sends it to the same HTTP endpoint; inference runs entirely in whichever default or custom server is open. The capture interval is not included in the reported `HTTP + inference` time. The pattern was consistent enough to warrant investigation, but it still needs repeated testing with exactly the same short WAV on both runtimes. @@ -91,7 +91,7 @@ The microphone does not execute inference. It creates an in-memory mono PCM16 WA The current evidence suggests that the custom runtime does not significantly change throughput for longer recordings, such as the 11-second JFK sample. -However, on the tested GTX 1060 6 GB, the custom runtime showed substantially lower and more consistent latency for short requests around two to three seconds. +However, on the tested GTX 1060 6 GB, the custom runtime showed a substantially lower median latency for short requests around two to three seconds. Four custom observations were around 72–80 ms, but the first custom observation was a 620.39 ms outlier, so consistency is not established. This may indicate that the fork changes affect fixed per-request overhead, backend initialization, buffer preparation, kernel selection, synchronization, or another short-request execution path. The exact cause has not yet been isolated. From 7a9a9d4896d068fb105bae84c528367bbc6fd1f2 Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:55:07 -0300 Subject: [PATCH 06/10] feat(cli): add runtime controls for CUDA compatibility --- app/bench.cpp | 3 +++ app/main.cpp | 43 ++++++++++++++++++++++++++++++++++++++++++- app/serve.cpp | 3 +++ app/transcribe.cpp | 3 +++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/app/bench.cpp b/app/bench.cpp index 84a355d..e0e4ef0 100644 --- a/app/bench.cpp +++ b/app/bench.cpp @@ -343,6 +343,9 @@ print_bench_help(const char* program) { " --mode offline|stream Recognition mode (default: offline)\n" " --device, --backend DEVICE\n" " auto, cpu, cuda[:N], metal, or vulkan[:N]\n" + " --skinny-q8 auto|on|off CUDA control: auto uses the safe Pascal fallback; off\n" + " replaces GGML_SKINNY_Q8=0; on requires SM 8.0+\n" + " --suppress-cuda-graph-log Suppress only the repeated CUDA-graph architecture log\n" " -l, --language CODE Prompt language code\n" " -r, --recursive Recurse into input directories\n" " --json Emit machine-readable results\n" diff --git a/app/main.cpp b/app/main.cpp index 53204dc..a35c571 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include +#include #include #include @@ -84,10 +86,29 @@ print_help(const char* program) { " --version Show version\n" " --json Emit machine-readable results and errors\n" " --quiet Suppress non-result progress messages\n" - " --verbose Emit additional diagnostics on stderr\n", + " --verbose Emit additional diagnostics on stderr\n" + " --suppress-cuda-graph-log\n" + " Suppress only the repeated CUDA-graph architecture message\n" + " --skinny-q8 MODE Skinny Q8: auto, on, or off (CLI overrides GGML_SKINNY_Q8)\n", NEMO_SPEECH_VERSION_STR, program); } +void +set_process_environment(const char* name, const char* value) { +#if defined(_WIN32) + if (_putenv_s(name, value) != 0) + throw std::runtime_error(std::string("failed to set environment variable ") + name); +#else + if (setenv(name, value, 1) != 0) + throw std::runtime_error(std::string("failed to set environment variable ") + name); +#endif +} + +bool +parse_skinny_q8_mode(const std::string& value) { + return value == "auto" || value == "on" || value == "off"; +} + } // namespace int @@ -100,6 +121,8 @@ main(int argc, char** argv) { bool json = false; bool quiet = false; bool verbose = false; + bool suppress_cuda_graph_log = false; + std::string skinny_q8_mode; std::vector filtered; filtered.reserve(static_cast(argc)); filtered.push_back(argv[0]); @@ -111,6 +134,15 @@ main(int argc, char** argv) { quiet = true; else if (arg == "--verbose") verbose = true; + else if (arg == "--suppress-cuda-graph-log") + suppress_cuda_graph_log = true; + else if (arg.rfind("--skinny-q8=", 0) == 0) + skinny_q8_mode = arg.substr(std::strlen("--skinny-q8=")); + else if (arg == "--skinny-q8") { + if (++i >= argc) + return print_cli_error("", "--skinny-q8 requires auto, on, or off", 2, "invalid_argument"); + skinny_q8_mode = argv[i]; + } else filtered.push_back(argv[i]); } @@ -118,6 +150,15 @@ main(int argc, char** argv) { if (quiet && verbose) return print_cli_error( "", "--quiet and --verbose cannot be used together", 2, "invalid_argument"); + if (!skinny_q8_mode.empty() && !parse_skinny_q8_mode(skinny_q8_mode)) + return print_cli_error( + "", "--skinny-q8 must be auto, on, or off", 2, "invalid_argument"); + if (suppress_cuda_graph_log) + set_process_environment("NEMO_SPEECH_SUPPRESS_CUDA_GRAPH_LOG", "1"); + if (!skinny_q8_mode.empty()) { + set_process_environment("NEMO_SPEECH_SKINNY_Q8_MODE", skinny_q8_mode.c_str()); + set_process_environment("NEMO_SPEECH_SKINNY_Q8_SOURCE", "cli"); + } argc = static_cast(filtered.size()); argv = filtered.data(); diff --git a/app/serve.cpp b/app/serve.cpp index e6d92b2..9260e3b 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -641,6 +641,9 @@ print_serve_help(const char* program) { #endif " --device, --backend DEVICE\n" " auto, cpu, cuda[:N], metal, or vulkan[:N]\n" + " --skinny-q8 auto|on|off CUDA control: auto uses the safe Pascal fallback; off\n" + " replaces GGML_SKINNY_Q8=0; on requires SM 8.0+\n" + " --suppress-cuda-graph-log Suppress only the repeated CUDA-graph architecture log\n" " --config FILE Apply YAML configuration\n" #if defined(NEMO_SPEECH_CLI_ASR) " --asr.* VALUE Override ASR engine configuration\n" diff --git a/app/transcribe.cpp b/app/transcribe.cpp index 8dd1670..ee671d6 100644 --- a/app/transcribe.cpp +++ b/app/transcribe.cpp @@ -440,6 +440,9 @@ print_transcribe_help(const char* program) { " -l, --language CODE Language code or prompt\n" " --device, --backend DEVICE\n" " auto, cpu, cuda[:N], metal, or vulkan[:N]\n" + " --skinny-q8 auto|on|off CUDA control: auto uses the safe Pascal fallback; off\n" + " replaces GGML_SKINNY_Q8=0; on requires SM 8.0+\n" + " --suppress-cuda-graph-log Suppress only the repeated CUDA-graph architecture log\n" " -c, --concurrency N Concurrent utterances; one shared model\n" " -f, --format FORMAT text, json, srt, or vtt (default: text)\n" " -o, --output PATH Output path for one input\n" From 1855d476305d86e54a1f5d50be93b548e8c4d2a4 Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:55:07 -0300 Subject: [PATCH 07/10] fix(cuda): add safe Skinny Q8 fallback for pre-Ampere GPUs --- src/runtime/ggml/backend.cpp | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/runtime/ggml/backend.cpp b/src/runtime/ggml/backend.cpp index 886648a..eb68ddc 100644 --- a/src/runtime/ggml/backend.cpp +++ b/src/runtime/ggml/backend.cpp @@ -10,8 +10,73 @@ #include "runtime.h" +#if defined(GGML_USE_CUDA) +#include +#endif + namespace ggml_runtime { +namespace { + +void +set_process_environment(const char* name, const char* value) { +#if defined(_WIN32) + if (_putenv_s(name, value) != 0) + throw std::runtime_error(std::string("failed to set environment variable ") + name); +#else + if (setenv(name, value, 1) != 0) + throw std::runtime_error(std::string("failed to set environment variable ") + name); +#endif +} + +std::string +compute_capability_name(int cc) { + if (cc <= 0) + return "unknown"; + return std::to_string(cc / 100) + "." + std::to_string((cc % 100) / 10); +} + +void +configure_skinny_q8(ggml_backend_dev_t device, int gpu_index) { +#if defined(GGML_USE_CUDA) + const char* mode = std::getenv("NEMO_SPEECH_SKINNY_Q8_MODE"); + if (mode == nullptr) + return; // No CLI control: preserve the original GGML environment behavior. + + const int cc = ggml_backend_cuda_get_device_compute_capability(gpu_index); + const char* description = ggml_backend_dev_description(device); + const std::string gpu = description ? description : "unknown GPU"; + const std::string cc_name = compute_capability_name(cc); + if (std::strcmp(mode, "off") == 0) { + set_process_environment("GGML_SKINNY_Q8", "0"); + GGMLF_LOG_INFO("[cuda] skinny-q8=off source=cli\n"); + } else if (std::strcmp(mode, "on") == 0) { + if (cc <= 0 || cc < 800) { + throw std::runtime_error( + "Skinny Q8 is not compatible with the selected GPU.\nGPU: " + gpu + + "\nCompute Capability: " + cc_name + + "\nCurrent kernel requirement: SM 8.0 or higher\nUse: --skinny-q8 off"); + } + set_process_environment("GGML_SKINNY_Q8", "1"); + GGMLF_LOG_INFO("[cuda] skinny-q8=on source=cli compute-capability=%s\n", cc_name.c_str()); + } else { // auto + if (cc <= 0 || cc < 800) { + set_process_environment("GGML_SKINNY_Q8", "0"); + GGMLF_LOG_INFO( + "[cuda] skinny-q8=off source=auto reason=compute-capability-%s\n", cc_name.c_str()); + } else { + GGMLF_LOG_INFO( + "[cuda] skinny-q8=on source=auto compute-capability=%s\n", cc_name.c_str()); + } + } +#else + (void)device; + (void)gpu_index; +#endif +} + +} // namespace + BackendManager::BackendManager(Params params) { this->params = params; init_backends(); @@ -71,6 +136,9 @@ BackendManager::init_backends() { std::to_string(params.gpu_device_idx) + ")"); } GGMLF_LOG_INFO("Using GPU backend: %s\n", ggml_backend_dev_name(dev)); + configure_skinny_q8(dev, params.gpu_device_idx); + if (std::getenv("NEMO_SPEECH_SUPPRESS_CUDA_GRAPH_LOG") != nullptr) + GGMLF_LOG_INFO("[cuda] cuda-graph-architecture-log=suppressed\n"); ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); if (backend == nullptr) { throw std::runtime_error( From debe6ddccc1ca01136c2ed9bb45ac509c7f43207 Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Thu, 6 Aug 2026 16:55:08 -0300 Subject: [PATCH 08/10] build(ggml): add reproducible runtime controls patch --- ggml-patches/0014-runtime-cli-controls.patch | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 ggml-patches/0014-runtime-cli-controls.patch diff --git a/ggml-patches/0014-runtime-cli-controls.patch b/ggml-patches/0014-runtime-cli-controls.patch new file mode 100644 index 0000000..7163713 --- /dev/null +++ b/ggml-patches/0014-runtime-cli-controls.patch @@ -0,0 +1,38 @@ +diff --git a/include/ggml-cuda.h b/include/ggml-cuda.h +--- a/include/ggml-cuda.h ++++ b/include/ggml-cuda.h +@@ -37,6 +37,8 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type( + GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); + + GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void); ++// Returns the GGML CUDA architecture id (for NVIDIA: major * 100 + minor * 10), or 0. ++GGML_BACKEND_API int ggml_backend_cuda_get_device_compute_capability(int device); + GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size); + GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); + +diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu +--- a/src/ggml-cuda/ggml-cuda.cu ++++ b/src/ggml-cuda/ggml-cuda.cu +@@ -5013,6 +5013,6 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co + if (graph->graph == nullptr) { + if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_AMPERE) { +- if (!graph->disable_due_to_gpu_arch) { ++ if (!graph->disable_due_to_gpu_arch && getenv("NEMO_SPEECH_SUPPRESS_CUDA_GRAPH_LOG") == nullptr) { + GGML_LOG_DEBUG("%s: disabling CUDA graphs due to GPU architecture\n", __func__); + } + graph->disable_due_to_gpu_arch = true; +@@ -5383,6 +5383,14 @@ int ggml_backend_cuda_get_device_count() { + return ggml_cuda_info().device_count; + } + ++int ggml_backend_cuda_get_device_compute_capability(int device) { ++ const auto & info = ggml_cuda_info(); ++ if (device < 0 || device >= info.device_count) { ++ return 0; ++ } ++ return info.devices[device].cc; ++} ++ + void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { + cudaDeviceProp prop; + CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); From acb5028e418dd33bbb8b05b2f4969dafbe7f1444 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:39:56 +0000 Subject: [PATCH 09/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- app/main.cpp | 11 +++--- examples/python/microphone_http.py | 61 ++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/app/main.cpp b/app/main.cpp index 4abbc58..e9ff435 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 #include -#include #include +#include #include #include #include @@ -165,10 +165,10 @@ main(int argc, char** argv) { skinny_q8_mode = arg.substr(std::strlen("--skinny-q8=")); else if (arg == "--skinny-q8") { if (++i >= argc) - return print_cli_error("", "--skinny-q8 requires auto, on, or off", 2, "invalid_argument"); + return print_cli_error( + "", "--skinny-q8 requires auto, on, or off", 2, "invalid_argument"); skinny_q8_mode = argv[i]; - } - else + } else filtered.push_back(argv[i]); } configure_cli_output(json, quiet, verbose); @@ -176,8 +176,7 @@ main(int argc, char** argv) { return print_cli_error( "", "--quiet and --verbose cannot be used together", 2, "invalid_argument"); if (!skinny_q8_mode.empty() && !parse_skinny_q8_mode(skinny_q8_mode)) - return print_cli_error( - "", "--skinny-q8 must be auto, on, or off", 2, "invalid_argument"); + return print_cli_error("", "--skinny-q8 must be auto, on, or off", 2, "invalid_argument"); if (suppress_cuda_graph_log) set_process_environment("NEMO_SPEECH_SUPPRESS_CUDA_GRAPH_LOG", "1"); if (!skinny_q8_mode.empty()) { diff --git a/examples/python/microphone_http.py b/examples/python/microphone_http.py index 53071c8..a404db8 100644 --- a/examples/python/microphone_http.py +++ b/examples/python/microphone_http.py @@ -36,9 +36,13 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--url", default="http://127.0.0.1:8081/v1/audio/transcriptions") parser.add_argument("--language", default="en") - parser.add_argument("--device", help="Input-device index or a case-insensitive part of its name.") + parser.add_argument( + "--device", help="Input-device index or a case-insensitive part of its name." + ) parser.add_argument("--list-devices", action="store_true", help="List input devices and exit.") - parser.add_argument("--show-words", action="store_true", help="Print word timestamps when present.") + parser.add_argument( + "--show-words", action="store_true", help="Print word timestamps when present." + ) parser.add_argument("--timeout", type=float, default=120.0, help="HTTP timeout in seconds.") return parser @@ -68,7 +72,9 @@ def resolve_device(selector: str | None) -> tuple[int | None, str]: raise ValueError(f"No input device contains {selector!r}. Use --list-devices.") if len(matches) > 1: choices = ", ".join(f"{i}: {d['name']}" for i, d in matches) - raise ValueError(f"More than one input device matches {selector!r}: {choices}. Use its index.") + raise ValueError( + f"More than one input device matches {selector!r}: {choices}. Use its index." + ) index, device = matches[0] return index, str(device["name"]) for available_index, device in devices: @@ -84,7 +90,9 @@ def calculate_rms(samples: np.ndarray) -> float: def record_until_enter(device: int | None) -> RecordedAudio: blocks: queue.Queue[np.ndarray] = queue.Queue() - def callback(input_data: np.ndarray, frames: int, time_info: Any, status: sd.CallbackFlags) -> None: + def callback( + input_data: np.ndarray, frames: int, time_info: Any, status: sd.CallbackFlags + ) -> None: del frames, time_info if status: print(f"Microphone warning: {status}", file=sys.stderr) @@ -92,11 +100,17 @@ def callback(input_data: np.ndarray, frames: int, time_info: Any, status: sd.Cal input("Press Enter to start recording...") started = time.perf_counter() - with sd.InputStream(samplerate=SAMPLE_RATE, channels=CHANNELS, dtype="float32", device=device, callback=callback): + with sd.InputStream( + samplerate=SAMPLE_RATE, channels=CHANNELS, dtype="float32", device=device, callback=callback + ): input("Recording. Press Enter again to stop...") capture_ms = (time.perf_counter() - started) * 1000 captured = [blocks.get_nowait() for _ in range(blocks.qsize())] - samples = np.ascontiguousarray(np.concatenate(captured), dtype=np.float32) if captured else np.empty(0, dtype=np.float32) + samples = ( + np.ascontiguousarray(np.concatenate(captured), dtype=np.float32) + if captured + else np.empty(0, dtype=np.float32) + ) return RecordedAudio(samples, samples.size / SAMPLE_RATE, calculate_rms(samples), capture_ms) @@ -126,9 +140,16 @@ def check_server(url: str, timeout: float) -> None: response.raise_for_status() -def request_transcription(session: requests.Session, url: str, wav_data: bytes, language: str, timeout: float) -> tuple[dict[str, Any], float]: +def request_transcription( + session: requests.Session, url: str, wav_data: bytes, language: str, timeout: float +) -> tuple[dict[str, Any], float]: started = time.perf_counter() - response = session.post(url, files={"file": ("microphone.wav", wav_data, "audio/wav")}, data={"model": "default", "language": language, "response_format": "verbose_json"}, timeout=timeout) + response = session.post( + url, + files={"file": ("microphone.wav", wav_data, "audio/wav")}, + data={"model": "default", "language": language, "response_format": "verbose_json"}, + timeout=timeout, + ) elapsed_ms = (time.perf_counter() - started) * 1000 response.raise_for_status() try: @@ -148,7 +169,13 @@ def result_words(payload: dict[str, Any]) -> list[dict[str, Any]]: return collected -def print_result(audio: RecordedAudio, preparation_ms: float, request_ms: float, payload: dict[str, Any], show_words: bool) -> None: +def print_result( + audio: RecordedAudio, + preparation_ms: float, + request_ms: float, + payload: dict[str, Any], + show_words: bool, +) -> None: text = str(payload.get("text", "")).strip() print("\n" + "=" * 64) print(f'Text: "{text}"') @@ -172,8 +199,16 @@ def print_result(audio: RecordedAudio, preparation_ms: float, request_ms: float, for word in words: token = str(word.get("word", word.get("text", ""))).strip() start, end = word.get("start"), word.get("end") - timing = f"{float(start):.3f}–{float(end):.3f}" if start is not None and end is not None else "unknown time" - confidence = f" | confidence={word['confidence']}" if word.get("confidence") is not None else "" + timing = ( + f"{float(start):.3f}–{float(end):.3f}" + if start is not None and end is not None + else "unknown time" + ) + confidence = ( + f" | confidence={word['confidence']}" + if word.get("confidence") is not None + else "" + ) print(f" {timing} {token}{confidence}") else: print("\nWords: not returned by the server.") @@ -211,7 +246,9 @@ def main() -> int: continue wav_data, preparation_ms = encode_wav(audio.samples) try: - payload, request_ms = request_transcription(session, args.url, wav_data, args.language, args.timeout) + payload, request_ms = request_transcription( + session, args.url, wav_data, args.language, args.timeout + ) print_result(audio, preparation_ms, request_ms, payload, args.show_words) except requests.HTTPError as error: body = error.response.text if error.response is not None else str(error) From 583c91a17b138cc263e9ed122c278c403a194d49 Mon Sep 17 00:00:00 2001 From: UNDER192103 Date: Sat, 22 Aug 2026 08:07:31 -0300 Subject: [PATCH 10/10] chore: add required license headers Signed-off-by: UNDER192103 --- examples/python/microphone_http.py | 2 ++ scripts/windows/benchmark-pascal-wav.ps1 | 1 + scripts/windows/benchmark-short-wav.ps1 | 1 + scripts/windows/build-pascal.ps1 | 1 + scripts/windows/run-pascal-server.ps1 | 1 + scripts/windows/setup-microphone-client.ps1 | 1 + scripts/windows/test-http-wav.ps1 | 1 + scripts/windows/test-pascal-wav.ps1 | 1 + 8 files changed, 9 insertions(+) diff --git a/examples/python/microphone_http.py b/examples/python/microphone_http.py index a404db8..d36c3bb 100644 --- a/examples/python/microphone_http.py +++ b/examples/python/microphone_http.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 """Record microphone audio and send it to a persistent NeMo-Speech.cpp HTTP server.""" from __future__ import annotations diff --git a/scripts/windows/benchmark-pascal-wav.ps1 b/scripts/windows/benchmark-pascal-wav.ps1 index 394195e..f9a146c 100644 --- a/scripts/windows/benchmark-pascal-wav.ps1 +++ b/scripts/windows/benchmark-pascal-wav.ps1 @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [CmdletBinding()] param( diff --git a/scripts/windows/benchmark-short-wav.ps1 b/scripts/windows/benchmark-short-wav.ps1 index c0b5138..637e3cc 100644 --- a/scripts/windows/benchmark-short-wav.ps1 +++ b/scripts/windows/benchmark-short-wav.ps1 @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 <#[ .SYNOPSIS diff --git a/scripts/windows/build-pascal.ps1 b/scripts/windows/build-pascal.ps1 index 9c69236..37224e8 100644 --- a/scripts/windows/build-pascal.ps1 +++ b/scripts/windows/build-pascal.ps1 @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [CmdletBinding()] param( diff --git a/scripts/windows/run-pascal-server.ps1 b/scripts/windows/run-pascal-server.ps1 index e68c3d4..c5386c7 100644 --- a/scripts/windows/run-pascal-server.ps1 +++ b/scripts/windows/run-pascal-server.ps1 @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [CmdletBinding()] param( diff --git a/scripts/windows/setup-microphone-client.ps1 b/scripts/windows/setup-microphone-client.ps1 index b082273..f37ca07 100644 --- a/scripts/windows/setup-microphone-client.ps1 +++ b/scripts/windows/setup-microphone-client.ps1 @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [CmdletBinding()] param([string]$Python) diff --git a/scripts/windows/test-http-wav.ps1 b/scripts/windows/test-http-wav.ps1 index e795ab2..79b1d2a 100644 --- a/scripts/windows/test-http-wav.ps1 +++ b/scripts/windows/test-http-wav.ps1 @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [CmdletBinding()] param( diff --git a/scripts/windows/test-pascal-wav.ps1 b/scripts/windows/test-pascal-wav.ps1 index ebaf265..c696351 100644 --- a/scripts/windows/test-pascal-wav.ps1 +++ b/scripts/windows/test-pascal-wav.ps1 @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [CmdletBinding()] param(