From 2f5af4ee1e6534e0a22dee0cd846754f6abe7247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Sch=C3=B6nlaub?= Date: Tue, 8 Sep 2026 16:12:42 +0200 Subject: [PATCH 1/2] cli: let devices refine the product name via getMetadata() getMetadata() existed in the device API but was never called from the CLI, so a device that knows more than the USB strings (for example which headset is currently paired to a generic dongle) could not show it. Ask the device once, on the handle already opened for the first feature request; no extra device opens, and the default implementation returns the same HID strings, so other devices are unaffected. --- cli/main.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cli/main.cpp b/cli/main.cpp index 8f8db86..9bf3100 100644 --- a/cli/main.cpp +++ b/cli/main.cpp @@ -431,6 +431,7 @@ struct DiscoveredDevice { std::vector feature_requests; std::wstring vendor_name; std::wstring product_name; + bool metadata_queried = false; [[nodiscard]] uint16_t vendorId() const { @@ -513,6 +514,23 @@ std::vector discoverDevices(const Options& opts) // Feature handling // ============================================================================ +/** + * @brief Let the device refine the product name once it is open. + * + * A device may know more than its USB strings, e.g. which headset is currently + * paired to a generic dongle. The default getMetadata() returns the HID strings, + * so devices without such knowledge are unaffected. + */ +static void refineProductName(DiscoveredDevice& dev, hid_device* handle) +{ + if (dev.metadata_queried) + return; + dev.metadata_queried = true; + + if (auto meta = dev.device->getMetadata(handle); meta && !meta->product.empty()) + dev.product_name = std::wstring(meta->product.begin(), meta->product.end()); +} + hid_device* connectForCapability(HIDConnection& conn, const HIDDevice* device, uint16_t product_id, capabilities cap) { auto detail = device->getCapabilityDetail(cap); @@ -584,6 +602,7 @@ FeatureResult handleFeature(DiscoveredDevice& dev, capabilities cap, const Featu if (!handle) { return make_error(-1, "Could not open device"); } + refineProductName(dev, handle); } // Execute via handler registry (no more giant switch!) From 9858099973992443fa893d8254b5f57272e0b6b5 Mon Sep 17 00:00:00 2001 From: Denis Arnst Date: Fri, 11 Sep 2026 00:15:27 +0200 Subject: [PATCH 2/2] string_utils: convert HID and metadata strings as UTF-8, not via the locale refineProductName() widened the device-supplied name with std::wstring(s.begin(), s.end()). That is wrong in two ways: char is signed on most platforms, so a byte above 0x7F sign-extends into a negative wchar_t, and a multi-byte UTF-8 sequence becomes one wide character per byte rather than one per character. It happens to survive on macOS because wcstombs() masks the value back on the way out, but on Windows the same bytes come out as mojibake and on glibc they hit the '?' fallback. Decoding it properly is not enough on its own: the value is converted straight back with wstring_to_string(), which goes through wcstombs() and so can only represent what the current locale encodes. Nothing calls setlocale(), so in the default C locale every non-ASCII character became '?'. DeviceMetadata and the HID descriptor strings are defined to be UTF-8, so this adds an explicit, locale-independent pair for them - string_to_wstring() and wstring_to_utf8() - and uses it wherever those strings are converted. The pair is an exact inverse, verified in C, en_US.UTF-8 and de_DE.UTF-8. Surrogate pairs are handled for the 16-bit wchar_t on Windows, and malformed input becomes U+FFFD rather than discarding the rest of the name. hid_error() keeps using wstring_to_string(): those really are platform strings in the local encoding, not UTF-8. This also fixes non-ASCII manufacturer and product strings read from the HID descriptor, which were already being mangled before any of this. --- cli/dev.cpp | 6 +- cli/main.cpp | 3 +- cli/output/output.cpp | 6 +- cli/output/output_data.hpp | 4 +- lib/devices/hid_device.hpp | 6 +- lib/headsetcontrol.cpp | 2 +- lib/string_utils.hpp | 173 +++++++++++++++++++++++++++++++++++++ tests/test_utilities.cpp | 81 ++++++++++++++++- 8 files changed, 267 insertions(+), 14 deletions(-) diff --git a/cli/dev.cpp b/cli/dev.cpp index 1dbd1ed..b1714ec 100644 --- a/cli/dev.cpp +++ b/cli/dev.cpp @@ -83,13 +83,13 @@ void print_devices(uint16_t vendorid, uint16_t productid) cur->vendor_id, cur->product_id, cur->path); if (cur->serial_number) { - std::cout << " Serial: " << headsetcontrol::wstring_to_string(cur->serial_number) << '\n'; + std::cout << " Serial: " << headsetcontrol::wstring_to_utf8(cur->serial_number) << '\n'; } if (cur->manufacturer_string) { - std::cout << " Manufacturer: " << headsetcontrol::wstring_to_string(cur->manufacturer_string) << '\n'; + std::cout << " Manufacturer: " << headsetcontrol::wstring_to_utf8(cur->manufacturer_string) << '\n'; } if (cur->product_string) { - std::cout << " Product: " << headsetcontrol::wstring_to_string(cur->product_string) << '\n'; + std::cout << " Product: " << headsetcontrol::wstring_to_utf8(cur->product_string) << '\n'; } std::cout << std::format( diff --git a/cli/main.cpp b/cli/main.cpp index 9bf3100..81cf99e 100644 --- a/cli/main.cpp +++ b/cli/main.cpp @@ -29,6 +29,7 @@ #include "hid_utility.hpp" #include "output.hpp" #include "result_types.hpp" +#include "string_utils.hpp" #include "utility.hpp" #include "version.h" @@ -528,7 +529,7 @@ static void refineProductName(DiscoveredDevice& dev, hid_device* handle) dev.metadata_queried = true; if (auto meta = dev.device->getMetadata(handle); meta && !meta->product.empty()) - dev.product_name = std::wstring(meta->product.begin(), meta->product.end()); + dev.product_name = headsetcontrol::string_to_wstring(meta->product); } hid_device* connectForCapability(HIDConnection& conn, const HIDDevice* device, uint16_t product_id, capabilities cap) diff --git a/cli/output/output.cpp b/cli/output/output.cpp index 319892f..574186e 100644 --- a/cli/output/output.cpp +++ b/cli/output/output.cpp @@ -269,8 +269,8 @@ void outputYaml(const OutputData& data) s.writeListItem("status", statusToString(dev.status)); s.pushIndent(1); // Align subsequent keys with "status" after "- " s.write("device", dev.device_name); - s.write("vendor", dev.vendor_name.empty() ? "" : headsetcontrol::wstring_to_string(dev.vendor_name.c_str())); - s.write("product", dev.product_name.empty() ? "" : headsetcontrol::wstring_to_string(dev.product_name.c_str())); + s.write("vendor", dev.vendor_name.empty() ? "" : headsetcontrol::wstring_to_utf8(dev.vendor_name)); + s.write("product", dev.product_name.empty() ? "" : headsetcontrol::wstring_to_utf8(dev.product_name)); s.write("id_vendor", dev.vendor_id); s.write("id_product", dev.product_id); @@ -463,7 +463,7 @@ void outputStandard(const OutputData& data, bool print_capabilities) for (const auto& dev : data.devices) { if (!dev.product_name.empty()) { - s.println(" {} ({}) [{}:{}]", dev.device_name, headsetcontrol::wstring_to_string(dev.product_name.c_str()), dev.vendor_id, dev.product_id); + s.println(" {} ({}) [{}:{}]", dev.device_name, headsetcontrol::wstring_to_utf8(dev.product_name), dev.vendor_id, dev.product_id); } else { s.println(" {} [{}:{}]", dev.device_name, dev.vendor_id, dev.product_id); } diff --git a/cli/output/output_data.hpp b/cli/output/output_data.hpp index cfceb4e..d1e6dd4 100644 --- a/cli/output/output_data.hpp +++ b/cli/output/output_data.hpp @@ -209,8 +209,8 @@ struct DeviceData { s.beginObject(""); s.write("status", statusToString(status)); s.write("device", device_name); - s.write("vendor", vendor_name.empty() ? "" : headsetcontrol::wstring_to_string(vendor_name.c_str())); - s.write("product", product_name.empty() ? "" : headsetcontrol::wstring_to_string(product_name.c_str())); + s.write("vendor", vendor_name.empty() ? "" : headsetcontrol::wstring_to_utf8(vendor_name)); + s.write("product", product_name.empty() ? "" : headsetcontrol::wstring_to_utf8(product_name)); s.write("id_vendor", vendor_id); s.write("id_product", product_id); diff --git a/lib/devices/hid_device.hpp b/lib/devices/hid_device.hpp index 1baa0c7..326948c 100644 --- a/lib/devices/hid_device.hpp +++ b/lib/devices/hid_device.hpp @@ -185,15 +185,15 @@ class HIDDevice { wchar_t serial[128] = {}; if (hid_get_manufacturer_string(device_handle, manufacturer, 128) == 0) { - meta.manufacturer = wstring_to_string(manufacturer); + meta.manufacturer = wstring_to_utf8(manufacturer); } if (hid_get_product_string(device_handle, product, 128) == 0) { - meta.product = wstring_to_string(product); + meta.product = wstring_to_utf8(product); } if (hid_get_serial_number_string(device_handle, serial, 128) == 0 && serial[0] != 0) { - meta.serial_number = wstring_to_string(serial); + meta.serial_number = wstring_to_utf8(serial); } return meta; diff --git a/lib/headsetcontrol.cpp b/lib/headsetcontrol.cpp index 56da583..f1eab1b 100644 --- a/lib/headsetcontrol.cpp +++ b/lib/headsetcontrol.cpp @@ -28,7 +28,7 @@ namespace { std::string hidStringOrEmpty(const wchar_t* value) { - return value ? wstring_to_string(value) : std::string(); + return value ? wstring_to_utf8(value) : std::string(); } class LibraryState { diff --git a/lib/string_utils.hpp b/lib/string_utils.hpp index b62fa07..e522c3d 100644 --- a/lib/string_utils.hpp +++ b/lib/string_utils.hpp @@ -1,7 +1,10 @@ #pragma once +#include #include #include +#include +#include namespace headsetcontrol { @@ -60,4 +63,174 @@ inline std::string wstring_to_string(const wchar_t* wstr) #endif } +/** + * @brief Convert UTF-8 string to wide string + * + * Inverse of wstring_to_string() for the UTF-8 strings carried in DeviceMetadata. + * The decoding is written out rather than delegated to mbstowcs() so that it does + * not depend on the process locale: a device-supplied name arrives as raw bytes + * off the wire and never passes through the C library's conversion on the way in. + * + * Widening byte by byte instead would produce one wide character per byte, so a + * name like "Muller" spelled with an umlaut would come back out mangled. + * + * Malformed input is replaced with U+FFFD rather than rejected, so one bad byte + * in a name read from a device does not discard the rest of it. + * + * @param str UTF-8 string + * @return Wide string: UTF-16 where wchar_t is 16 bits, UTF-32 where it is wider + */ +inline std::wstring string_to_wstring(std::string_view str) +{ + constexpr char32_t REPLACEMENT = 0xFFFD; + constexpr char32_t MAX_CODEPOINT = 0x10FFFF; + + std::wstring result; + result.reserve(str.size()); + + auto append = [&result](char32_t codepoint) { + if constexpr (sizeof(wchar_t) >= 4) { + result += static_cast(codepoint); + } else if (codepoint <= 0xFFFF) { + result += static_cast(codepoint); + } else { + // Split into a UTF-16 surrogate pair, as on Windows where wchar_t is 16 bits + const char32_t offset = codepoint - 0x10000; + result += static_cast(0xD800 + (offset >> 10)); + result += static_cast(0xDC00 + (offset & 0x3FF)); + } + }; + + for (std::size_t i = 0; i < str.size();) { + const auto lead = static_cast(str[i]); + + // Length of the sequence, and the smallest codepoint it may legally encode - + // a larger sequence than a codepoint needs is an overlong encoding + std::size_t continuations = 0; + char32_t codepoint = 0; + char32_t minimum = 0; + if (lead < 0x80) { + codepoint = lead; + } else if ((lead & 0xE0) == 0xC0) { + continuations = 1; + codepoint = lead & 0x1FU; + minimum = 0x80; + } else if ((lead & 0xF0) == 0xE0) { + continuations = 2; + codepoint = lead & 0x0FU; + minimum = 0x800; + } else if ((lead & 0xF8) == 0xF0) { + continuations = 3; + codepoint = lead & 0x07U; + minimum = 0x10000; + } else { + append(REPLACEMENT); + ++i; + continue; + } + + if (i + continuations >= str.size()) { + append(REPLACEMENT); + ++i; + continue; + } + + bool valid = true; + for (std::size_t k = 1; k <= continuations; ++k) { + const auto continuation = static_cast(str[i + k]); + if ((continuation & 0xC0) != 0x80) { + valid = false; + break; + } + codepoint = (codepoint << 6) | (continuation & 0x3FU); + } + + // Surrogates are not valid on their own, and are not encodable in UTF-8 + const bool is_surrogate = codepoint >= 0xD800 && codepoint <= 0xDFFF; + if (!valid || codepoint < minimum || codepoint > MAX_CODEPOINT || is_surrogate) { + append(REPLACEMENT); + ++i; + continue; + } + + append(codepoint); + i += continuations + 1; + } + + return result; +} + +/** + * @brief Convert wide string to UTF-8 string + * + * Counterpart to string_to_wstring(), and an exact inverse of it. Unlike + * wstring_to_string() this does not consult the locale: it is for strings that are + * defined to be UTF-8, such as the device names in DeviceMetadata, where going + * through wcstombs() would replace anything outside the current locale's encoding + * with '?' - which in the default C locale means every non-ASCII character. + * + * Where wchar_t is 16 bits a surrogate pair is recombined into the codepoint it + * encodes; an unpaired surrogate is replaced with U+FFFD. + * + * @param str Wide string + * @return UTF-8 string + */ +inline std::string wstring_to_utf8(std::wstring_view str) +{ + constexpr char32_t REPLACEMENT = 0xFFFD; + constexpr char32_t MAX_CODEPOINT = 0x10FFFF; + + std::string result; + result.reserve(str.size()); + + auto append = [&result](char32_t codepoint) { + if (codepoint < 0x80) { + result += static_cast(codepoint); + } else if (codepoint < 0x800) { + result += static_cast(0xC0 | (codepoint >> 6)); + result += static_cast(0x80 | (codepoint & 0x3F)); + } else if (codepoint < 0x10000) { + result += static_cast(0xE0 | (codepoint >> 12)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); + result += static_cast(0x80 | (codepoint & 0x3F)); + } else { + result += static_cast(0xF0 | (codepoint >> 18)); + result += static_cast(0x80 | ((codepoint >> 12) & 0x3F)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); + result += static_cast(0x80 | (codepoint & 0x3F)); + } + }; + + for (std::size_t i = 0; i < str.size(); ++i) { + // Mask rather than cast: char32_t is unsigned, but wchar_t is signed on + // some platforms, and a plain conversion would sign-extend + auto codepoint = static_cast( + static_cast>(str[i])); + + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // High surrogate: needs the matching low surrogate to mean anything + const bool has_low = i + 1 < str.size(); + const auto low = has_low + ? static_cast(static_cast>(str[i + 1])) + : char32_t { 0 }; + if (has_low && low >= 0xDC00 && low <= 0xDFFF) { + codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low - 0xDC00); + ++i; + } else { + codepoint = REPLACEMENT; + } + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + // Low surrogate without a high one before it + codepoint = REPLACEMENT; + } + + if (codepoint > MAX_CODEPOINT) { + codepoint = REPLACEMENT; + } + append(codepoint); + } + + return result; +} + } // namespace headsetcontrol diff --git a/tests/test_utilities.cpp b/tests/test_utilities.cpp index b56d90b..c659c2e 100644 --- a/tests/test_utilities.cpp +++ b/tests/test_utilities.cpp @@ -6,7 +6,7 @@ * - device_utils.hpp: mapSidetoneToDiscrete, mapSidetoneWithToggle, map, voltageToPercent, etc. * - utility.hpp: round_to_multiples, spline_battery_level, parse_byte_data, etc. * - result_types.hpp: Result, DeviceError - * - string_utils.hpp: wstring_to_string + * - string_utils.hpp: wstring_to_string, string_to_wstring, wstring_to_utf8 * - feature_utils.hpp: make_success, make_info, make_error * - output/output_data.hpp: statusToString, batteryStatusToString */ @@ -579,6 +579,83 @@ void testWstringToString() std::cout << " ✓ wstring_to_string works correctly" << std::endl; } +void testStringToWstring() +{ + std::cout << " Testing string_to_wstring..." << std::endl; + + ASSERT_TRUE(string_to_wstring("Hello") == L"Hello", "ASCII should convert unchanged"); + ASSERT_TRUE(string_to_wstring("").empty(), "Empty should give empty"); + + // The point of the function: a multi-byte sequence is one character, not one + // per byte. "Muller" with an umlaut is 7 UTF-8 bytes but 6 characters. + const std::string umlaut = "M\xC3\xBCller"; + ASSERT_EQ(7u, umlaut.size(), "Input should be 7 UTF-8 bytes"); + const std::wstring wide = string_to_wstring(umlaut); + ASSERT_EQ(6u, wide.size(), "Should decode to 6 characters, not 7"); + ASSERT_TRUE(wide[1] == static_cast(0x00FC), "Second character should be U+00FC"); + + // Three-byte sequence (U+20AC EURO SIGN) + const std::wstring euro = string_to_wstring("\xE2\x82\xAC"); + ASSERT_EQ(1u, euro.size(), "Euro sign should decode to one character"); + ASSERT_TRUE(euro[0] == static_cast(0x20AC), "Should be U+20AC"); + + // Four-byte sequence (U+1F50A SPEAKER). One character where wchar_t is 32 bits, + // a surrogate pair where it is 16. + const std::wstring speaker = string_to_wstring("\xF0\x9F\x94\x8A"); + ASSERT_EQ(sizeof(wchar_t) >= 4 ? 1u : 2u, speaker.size(), "Astral codepoint width"); + + // Malformed input keeps the rest of the string rather than discarding it + const std::wstring truncated = string_to_wstring("A\xC3"); + ASSERT_EQ(2u, truncated.size(), "Truncated sequence should not eat the string"); + ASSERT_TRUE(truncated[0] == L'A', "Leading character should survive"); + ASSERT_TRUE(truncated[1] == static_cast(0xFFFD), "Truncated byte becomes U+FFFD"); + + const std::wstring lone = string_to_wstring("\x80z"); + ASSERT_EQ(2u, lone.size(), "Stray continuation byte should be replaced, not dropped"); + ASSERT_TRUE(lone[0] == static_cast(0xFFFD), "Stray continuation becomes U+FFFD"); + ASSERT_TRUE(lone[1] == L'z', "Following character should survive"); + + // Overlong encoding of '/' must not decode to '/' + const std::wstring overlong = string_to_wstring("\xC0\xAF"); + ASSERT_TRUE(overlong[0] == static_cast(0xFFFD), "Overlong encoding should be rejected"); + + std::cout << " ✓ string_to_wstring works correctly" << std::endl; +} + +void testUtf8RoundTrip() +{ + std::cout << " Testing string_to_wstring/wstring_to_utf8 round trip..." << std::endl; + + // The pair has to be an exact inverse regardless of locale: these are the + // conversions a device-supplied name goes through on its way to the output. + const char* cases[] = { + "", + "Jabra Evolve2 65 Flex", + "M\xC3\xBCller", // U+00FC, two UTF-8 bytes + "\xE2\x82\xAC", // U+20AC euro sign, three bytes + "\xF0\x9F\x94\x8A", // U+1F50A speaker, four bytes (surrogate pair on Windows) + "caf\xC3\xA9 \xE2\x82\xAC 5", + }; + + for (const char* text : cases) { + ASSERT_EQ(std::string(text), wstring_to_utf8(string_to_wstring(text)), + "UTF-8 should survive the round trip unchanged"); + } + + // Sign extension check: a byte above 0x7F must not become a negative wchar_t. + // Widening byte by byte instead of decoding is what this guards against. + const std::wstring wide = string_to_wstring("\xC3\xBC"); + ASSERT_EQ(1u, wide.size(), "Two UTF-8 bytes are one character"); + ASSERT_TRUE(wide[0] > 0, "Decoded character must not be negative"); + + // An unpaired surrogate cannot be encoded and must not produce invalid UTF-8 + const std::wstring lone_surrogate(1, static_cast(0xD800)); + const std::string encoded = wstring_to_utf8(lone_surrogate); + ASSERT_EQ("\xEF\xBF\xBD", encoded, "Unpaired surrogate should become U+FFFD"); + + std::cout << " ✓ UTF-8 round trip works correctly" << std::endl; +} + // ============================================================================ // feature_utils.hpp Tests // ============================================================================ @@ -715,6 +792,8 @@ void runAllUtilityTests() std::cout << "\n=== string_utils.hpp Tests ===" << std::endl; runTest("wstring_to_string", testWstringToString); + runTest("string_to_wstring", testStringToWstring); + runTest("UTF-8 round trip", testUtf8RoundTrip); std::cout << "\n=== feature_utils.hpp Tests ===" << std::endl; runTest("make_success", testMakeSuccess);