Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions cpp/src/gandiva/precompiled/string_ops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3125,9 +3125,14 @@ const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len,

int start_idx = 0;
for (int i = 0; i < in_len; ++i) {
if (isalpha(static_cast<unsigned char>(in[i])) > 0) {
unsigned char ch = static_cast<unsigned char>(in[i]);
if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z')) {
// Retain the first letter
ret[0] = toupper(static_cast<unsigned char>(in[i]));
if (ch >= 'a') {
ret[0] = static_cast<char>(ch - 'a' + 'A');
} else {
ret[0] = static_cast<char>(ch);
}
start_idx = i + 1;
break;
}
Expand All @@ -3143,12 +3148,17 @@ const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len,
soundex[0] = '\0';
// Replace consonants with digits and special letters with 0
for (int i = start_idx; i < in_len; i++) {
if (isalpha(static_cast<unsigned char>(in[i])) > 0) {
c = toupper(static_cast<unsigned char>(in[i])) - 65;
if (mappings[c] != soundex[si - 1]) {
soundex[si] = mappings[c];
si++;
}
unsigned char ch = static_cast<unsigned char>(in[i]);
if (ch >= 'A' && ch <= 'Z') {
c = ch - 'A';
} else if (ch >= 'a' && ch <= 'z') {
c = ch - 'a';
} else {
continue;
}
if (mappings[c] != soundex[si - 1]) {
soundex[si] = mappings[c];
si++;
}
}

Expand Down
34 changes: 34 additions & 0 deletions cpp/src/gandiva/precompiled/string_ops_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include <cctype>
#include <cstring>
#include <limits>
#include <memory>

#include "arrow/testing/gtest_util.h"
#include "gandiva/execution_context.h"
#include "gandiva/precompiled/types.h"

Expand Down Expand Up @@ -2929,6 +2931,38 @@ TEST(TestStringOps, TestFromHex) {
EXPECT_EQ(out_valid, false);
}

static void CheckSoundexNonAsciiBytes() {
for (int byte = 128; byte <= 255; ++byte) {
const std::string non_ascii(1, static_cast<char>(byte));
for (const auto& input : {non_ascii + "Robert", "R" + non_ascii + "obert"}) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = -1;
bool validity = false;
const auto* out =
soundex_utf8(ctx_ptr, input.data(), static_cast<int32_t>(input.size()), true,
&validity, &out_len);
ASSERT_TRUE(validity);
ASSERT_EQ(out_len, 4);
EXPECT_EQ(std::string(out, out_len), "R163");
EXPECT_FALSE(ctx.has_error());
}
}
}

TEST(TestStringOps, TestSoundexNonAsciiBytes) {
arrow::LocaleGuard locale_guard("C");
CheckSoundexNonAsciiBytes();
}

TEST(TestStringOps, TestSoundexNonAsciiLocale) {
arrow::LocaleGuard locale_guard("en_US.ISO-8859-1");
if (std::isalpha(static_cast<unsigned char>(0xe9)) == 0) {
GTEST_SKIP() << "Requires an ISO-8859-1 locale classifying high bytes as letters";
}
CheckSoundexNonAsciiBytes();
}

TEST(TestStringOps, TestSoundex) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
Expand Down
Loading