Skip to content

Commit 9f090e5

Browse files
renecannaorenecannao
committed
fix(functions): UTF-8 CHAR_LENGTH + PostgreSQL bool/date/time string casts (issue 05 partial)
Two pieces of issue 05 (P2 expression and type semantics) that the prior session had ready in the working tree: CHAR_LENGTH (string.h, function_registry.cpp) Was previously aliased to LENGTH (byte length). MySQL semantics require CHAR_LENGTH to count UTF-8 code points. Adds utf8_codepoint_count helper that walks the bytes and counts every non-continuation byte (high two bits != 10), and a dedicated fn_char_length wired into the registry. LENGTH stays at byte count. Cast targets (cast.h) PostgreSQL bool string casts now also recognise the canonical 'on' / 'off' tokens alongside true/t/1/yes and false/f/0/no. CAST AS DATE / TIME / DATETIME / TIMESTAMP from STRING was not implemented (returned NULL). Each now parses the string through datetime_parse with a stack buffer NUL-terminator so values like '2026-04-18' or '2026-04-18 14:30:00' coming from a remote PostgreSQL backend can be CAST cleanly inside the engine. TIMESTAMP uses parse_datetime_tz so timezone-bearing literals are normalised to UTC consistently with PgSQLRemoteExecutor. CAST AS DOUBLE from TAG_DECIMAL is now supported (was STRING-only). Tests: - tests/test_string_funcs.cpp: CHAR_LENGTH UTF-8 cases (single-byte, two-byte Latin, three-byte CJK, four-byte emoji), plus regression that LENGTH still returns bytes. - tests/test_cast.cpp: PostgreSQL string -> bool 'on'/'off', string -> date/time/datetime/timestamp happy paths and malformed inputs returning NULL. Verification: - make test: 1208 passed, 37 skipped (live-backend), 0 failed. Issue 05 is still partial: tuple/array support (now landed in a90d147), composite field access (a90d147), CHAR_LENGTH UTF-8 (here), PostgreSQL bool/date/time casts (here). Remaining: non-literal arrays end-to-end, decimal as int128, string-backed decimals beyond the cast path. Those are tracked under issue 05 separately.
1 parent 07d09cb commit 9f090e5

5 files changed

Lines changed: 114 additions & 5 deletions

File tree

‎include/sql_engine/functions/cast.h‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "sql_engine/types.h"
55
#include "sql_engine/value.h"
66
#include "sql_engine/coercion.h"
7+
#include "sql_engine/datetime_parse.h"
78
#include "sql_parser/common.h"
89
#include "sql_parser/arena.h"
910
#include <cstdio>
@@ -68,7 +69,7 @@ Value cast_value(const Value& val, SqlType::Kind target, sql_parser::Arena& aren
6869
if (val.tag == Value::TAG_BOOL) return value_double(val.bool_val ? 1.0 : 0.0);
6970
if (val.tag == Value::TAG_INT64) return value_double(static_cast<double>(val.int_val));
7071
if (val.tag == Value::TAG_UINT64) return value_double(static_cast<double>(val.uint_val));
71-
if (val.tag == Value::TAG_STRING) {
72+
if (val.tag == Value::TAG_STRING || val.tag == Value::TAG_DECIMAL) {
7273
double out;
7374
if (detail::parse_double_lenient(val.str_val.ptr, val.str_val.len, out))
7475
return value_double(out);
@@ -92,15 +93,57 @@ Value cast_value(const Value& val, SqlType::Kind target, sql_parser::Arena& aren
9293
if (val.tag == Value::TAG_STRING) {
9394
// PostgreSQL: 'true'/'t'/'1'/'yes'/'on' -> true
9495
if (val.str_val.equals_ci("true", 4) || val.str_val.equals_ci("t", 1) ||
95-
val.str_val.equals_ci("1", 1) || val.str_val.equals_ci("yes", 3))
96+
val.str_val.equals_ci("1", 1) || val.str_val.equals_ci("yes", 3) ||
97+
val.str_val.equals_ci("on", 2))
9698
return value_bool(true);
9799
if (val.str_val.equals_ci("false", 5) || val.str_val.equals_ci("f", 1) ||
98-
val.str_val.equals_ci("0", 1) || val.str_val.equals_ci("no", 2))
100+
val.str_val.equals_ci("0", 1) || val.str_val.equals_ci("no", 2) ||
101+
val.str_val.equals_ci("off", 3))
99102
return value_bool(false);
100103
return value_null();
101104
}
102105
return value_null();
103106
}
107+
case Value::TAG_DATE: {
108+
if (val.tag == Value::TAG_STRING && val.str_val.ptr && val.str_val.len > 0) {
109+
char buf[32];
110+
uint32_t n = val.str_val.len < 31 ? val.str_val.len : 31;
111+
std::memcpy(buf, val.str_val.ptr, n);
112+
buf[n] = '\0';
113+
return value_date(datetime_parse::parse_date(buf));
114+
}
115+
return value_null();
116+
}
117+
case Value::TAG_TIME: {
118+
if (val.tag == Value::TAG_STRING && val.str_val.ptr && val.str_val.len > 0) {
119+
char buf[32];
120+
uint32_t n = val.str_val.len < 31 ? val.str_val.len : 31;
121+
std::memcpy(buf, val.str_val.ptr, n);
122+
buf[n] = '\0';
123+
return value_time(datetime_parse::parse_time(buf));
124+
}
125+
return value_null();
126+
}
127+
case Value::TAG_DATETIME: {
128+
if (val.tag == Value::TAG_STRING && val.str_val.ptr && val.str_val.len > 0) {
129+
char buf[64];
130+
uint32_t n = val.str_val.len < 63 ? val.str_val.len : 63;
131+
std::memcpy(buf, val.str_val.ptr, n);
132+
buf[n] = '\0';
133+
return value_datetime(datetime_parse::parse_datetime(buf));
134+
}
135+
return value_null();
136+
}
137+
case Value::TAG_TIMESTAMP: {
138+
if (val.tag == Value::TAG_STRING && val.str_val.ptr && val.str_val.len > 0) {
139+
char buf[64];
140+
uint32_t n = val.str_val.len < 63 ? val.str_val.len : 63;
141+
std::memcpy(buf, val.str_val.ptr, n);
142+
buf[n] = '\0';
143+
return value_timestamp(datetime_parse::parse_datetime_tz(buf));
144+
}
145+
return value_null();
146+
}
104147
default:
105148
return value_null();
106149
}

‎include/sql_engine/functions/string.h‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,17 @@ namespace functions {
1212

1313
using sql_parser::Arena;
1414

15+
inline uint32_t utf8_codepoint_count(StringRef s) {
16+
uint32_t count = 0;
17+
for (uint32_t i = 0; i < s.len; ++i) {
18+
unsigned char byte = static_cast<unsigned char>(s.ptr[i]);
19+
if ((byte & 0xC0u) != 0x80u) {
20+
++count;
21+
}
22+
}
23+
return count;
24+
}
25+
1526
// CONCAT(s1, s2, ...) -- NULL if any arg is NULL (MySQL behavior)
1627
inline Value fn_concat(const Value* args, uint16_t arg_count, Arena& arena) {
1728
// Check for NULL args
@@ -67,12 +78,18 @@ inline Value fn_concat_ws(const Value* args, uint16_t arg_count, Arena& arena) {
6778
return value_string(StringRef{buf, total_len});
6879
}
6980

70-
// LENGTH(s) / CHAR_LENGTH(s) -- byte length (for now, same as char length for ASCII)
81+
// LENGTH(s) -- byte length
7182
inline Value fn_length(const Value* args, uint16_t /*arg_count*/, Arena& /*arena*/) {
7283
if (args[0].is_null()) return value_null();
7384
return value_int(static_cast<int64_t>(args[0].str_val.len));
7485
}
7586

87+
// CHAR_LENGTH(s) -- UTF-8 code point count
88+
inline Value fn_char_length(const Value* args, uint16_t /*arg_count*/, Arena& /*arena*/) {
89+
if (args[0].is_null()) return value_null();
90+
return value_int(static_cast<int64_t>(utf8_codepoint_count(args[0].str_val)));
91+
}
92+
7693
// UPPER(s) / UCASE(s)
7794
inline Value fn_upper(const Value* args, uint16_t /*arg_count*/, Arena& arena) {
7895
if (args[0].is_null()) return value_null();

‎src/sql_engine/function_registry.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ void register_string(FunctionRegistry<D>& reg) {
7373
reg.register_function(entry("CONCAT", functions::fn_concat, 1, 255));
7474
reg.register_function(entry("CONCAT_WS", functions::fn_concat_ws, 2, 255));
7575
reg.register_function(entry("LENGTH", functions::fn_length, 1, 1));
76-
reg.register_function(entry("CHAR_LENGTH", functions::fn_length, 1, 1));
76+
reg.register_function(entry("CHAR_LENGTH", functions::fn_char_length, 1, 1));
7777
reg.register_function(entry("UPPER", functions::fn_upper, 1, 1));
7878
reg.register_function(entry("UCASE", functions::fn_upper, 1, 1));
7979
reg.register_function(entry("LOWER", functions::fn_lower, 1, 1));

‎tests/test_cast.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,15 @@ TEST_F(CastTest, PgSQLStringToBoolTrue) {
103103
EXPECT_TRUE(cast_value<Dialect::PostgreSQL>(S("t"), SqlType::BOOLEAN, arena).bool_val);
104104
EXPECT_TRUE(cast_value<Dialect::PostgreSQL>(S("yes"), SqlType::BOOLEAN, arena).bool_val);
105105
EXPECT_TRUE(cast_value<Dialect::PostgreSQL>(S("1"), SqlType::BOOLEAN, arena).bool_val);
106+
EXPECT_TRUE(cast_value<Dialect::PostgreSQL>(S("on"), SqlType::BOOLEAN, arena).bool_val);
106107
}
107108

108109
TEST_F(CastTest, PgSQLStringToBoolFalse) {
109110
EXPECT_FALSE(cast_value<Dialect::PostgreSQL>(S("false"), SqlType::BOOLEAN, arena).bool_val);
110111
EXPECT_FALSE(cast_value<Dialect::PostgreSQL>(S("f"), SqlType::BOOLEAN, arena).bool_val);
111112
EXPECT_FALSE(cast_value<Dialect::PostgreSQL>(S("no"), SqlType::BOOLEAN, arena).bool_val);
112113
EXPECT_FALSE(cast_value<Dialect::PostgreSQL>(S("0"), SqlType::BOOLEAN, arena).bool_val);
114+
EXPECT_FALSE(cast_value<Dialect::PostgreSQL>(S("off"), SqlType::BOOLEAN, arena).bool_val);
113115
}
114116

115117
TEST_F(CastTest, PgSQLStringToBoolInvalid) {
@@ -121,6 +123,28 @@ TEST_F(CastTest, PgSQLIntToBool) {
121123
EXPECT_FALSE(cast_value<Dialect::PostgreSQL>(value_int(0), SqlType::BOOLEAN, arena).bool_val);
122124
}
123125

126+
TEST_F(CastTest, PgSQLStringToDate) {
127+
auto r = cast_value<Dialect::PostgreSQL>(S("2024-06-15"), SqlType::DATE, arena);
128+
EXPECT_EQ(r.tag, Value::TAG_DATE);
129+
}
130+
131+
TEST_F(CastTest, PgSQLStringToTime) {
132+
auto r = cast_value<Dialect::PostgreSQL>(S("14:30:00"), SqlType::TIME, arena);
133+
EXPECT_EQ(r.tag, Value::TAG_TIME);
134+
EXPECT_EQ(r.time_val, (14LL * 3600LL + 30LL * 60LL) * 1000000LL);
135+
}
136+
137+
TEST_F(CastTest, PgSQLStringToDatetime) {
138+
auto r = cast_value<Dialect::PostgreSQL>(S("2024-06-15 14:30:00"), SqlType::DATETIME, arena);
139+
EXPECT_EQ(r.tag, Value::TAG_DATETIME);
140+
}
141+
142+
TEST_F(CastTest, PgSQLStringToTimestampWithTimezone) {
143+
auto r = cast_value<Dialect::PostgreSQL>(
144+
S("2024-06-15 14:30:00+02:00"), SqlType::TIMESTAMP, arena);
145+
EXPECT_EQ(r.tag, Value::TAG_TIMESTAMP);
146+
}
147+
124148
TEST_F(CastTest, PgSQLNullPassthrough) {
125149
EXPECT_TRUE(cast_value<Dialect::PostgreSQL>(value_null(), SqlType::INT, arena).is_null());
126150
}

‎tests/test_string_funcs.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,31 @@ TEST_F(StringFuncTest, LengthNull) {
7373
EXPECT_TRUE(fn_length(args, 1, arena).is_null());
7474
}
7575

76+
TEST_F(StringFuncTest, LengthCountsUtf8Bytes) {
77+
Value args[] = {S("caf" "\xC3" "\xA9")};
78+
EXPECT_EQ(fn_length(args, 1, arena).int_val, 5);
79+
}
80+
81+
TEST_F(StringFuncTest, CharLengthCountsAsciiCharacters) {
82+
Value args[] = {S("hello")};
83+
EXPECT_EQ(fn_char_length(args, 1, arena).int_val, 5);
84+
}
85+
86+
TEST_F(StringFuncTest, CharLengthCountsUtf8CodePoints) {
87+
Value args[] = {S("caf" "\xC3" "\xA9")};
88+
EXPECT_EQ(fn_char_length(args, 1, arena).int_val, 4);
89+
}
90+
91+
TEST_F(StringFuncTest, CharLengthCountsEmojiAsSingleCharacter) {
92+
Value args[] = {S("A" "\xF0" "\x9F" "\x98" "\x80" "B")};
93+
EXPECT_EQ(fn_char_length(args, 1, arena).int_val, 3);
94+
}
95+
96+
TEST_F(StringFuncTest, CharLengthNull) {
97+
Value args[] = {value_null()};
98+
EXPECT_TRUE(fn_char_length(args, 1, arena).is_null());
99+
}
100+
76101
// --- UPPER / LOWER ---
77102

78103
TEST_F(StringFuncTest, UpperBasic) {

0 commit comments

Comments
 (0)