feat: add real MySQL + PostgreSQL backend connections via libmysqlclient/libpq

Implements MySQLRemoteExecutor, PgSQLRemoteExecutor, and MultiRemoteExecutor
for executing queries against real database backends. Includes datetime parsing
utilities, Docker-based test infrastructure, and 37 integration tests that
skip gracefully when backends are not available.

Author: renecannao

Makefile.new

@@ -2,6 +2,12 @@ CXX = g++
 CXXFLAGS = -std=c++17 -Wall -Wextra -g -O2
 CPPFLAGS = -I./include -I./third_party/googletest/googletest/include
 
+# MySQL and PostgreSQL client libraries
+MYSQL_CFLAGS = $(shell mysql_config --cflags 2>/dev/null)
+MYSQL_LIBS = $(shell mysql_config --libs 2>/dev/null)
+PG_CFLAGS = -I$(shell pg_config --includedir 2>/dev/null || echo /usr/include/postgresql)
+PG_LIBS = -L$(shell pg_config --libdir 2>/dev/null || echo /usr/lib/x86_64-linux-gnu) -lpq
+
 PROJECT_ROOT = .
 SRC_DIR = $(PROJECT_ROOT)/src/sql_parser
 ENGINE_SRC_DIR = $(PROJECT_ROOT)/src/sql_engine
@@ -15,7 +21,11 @@ LIB_TARGET = $(PROJECT_ROOT)/libsqlparser.a
 
 # SQL Engine sources
 ENGINE_SRCS = $(ENGINE_SRC_DIR)/function_registry.cpp \
-              $(ENGINE_SRC_DIR)/in_memory_catalog.cpp
+              $(ENGINE_SRC_DIR)/in_memory_catalog.cpp \
+              $(ENGINE_SRC_DIR)/datetime_parse.cpp \
+              $(ENGINE_SRC_DIR)/mysql_remote_executor.cpp \
+              $(ENGINE_SRC_DIR)/pgsql_remote_executor.cpp \
+              $(ENGINE_SRC_DIR)/multi_remote_executor.cpp
 ENGINE_OBJS = $(ENGINE_SRCS:.cpp=.o)
 
 # Google Test library
@@ -59,7 +69,10 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
             $(TEST_DIR)/test_optimizer.cpp \
             $(TEST_DIR)/test_distributed_planner.cpp \
             $(TEST_DIR)/test_dml.cpp \
-            $(TEST_DIR)/test_distributed_dml.cpp
+            $(TEST_DIR)/test_distributed_dml.cpp \
+            $(TEST_DIR)/test_mysql_executor.cpp \
+            $(TEST_DIR)/test_pgsql_executor.cpp \
+            $(TEST_DIR)/test_distributed_real.cpp
 TEST_OBJS = $(TEST_SRCS:.cpp=.o)
 TEST_TARGET = $(PROJECT_ROOT)/run_tests
 
@@ -95,21 +108,21 @@ $(SRC_DIR)/%.o: $(SRC_DIR)/%.cpp
 
 # SQL Engine objects
 $(ENGINE_SRC_DIR)/%.o: $(ENGINE_SRC_DIR)/%.cpp
-	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $< -o $@
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(MYSQL_CFLAGS) $(PG_CFLAGS) -c $< -o $@
 
 # Google Test object
 $(GTEST_OBJ): $(GTEST_SRC)
 	$(CXX) $(CXXFLAGS) $(GTEST_CPPFLAGS) -c $< -o $@
 
 # Test objects
 $(TEST_DIR)/%.o: $(TEST_DIR)/%.cpp
-	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(GTEST_CPPFLAGS) -c $< -o $@
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(GTEST_CPPFLAGS) $(MYSQL_CFLAGS) $(PG_CFLAGS) -c $< -o $@
 
 test: $(TEST_TARGET)
 	./$(TEST_TARGET)
 
 $(TEST_TARGET): $(TEST_OBJS) $(GTEST_OBJ) $(LIB_TARGET) $(ENGINE_OBJS)
-	$(CXX) $(CXXFLAGS) -o $@ $(TEST_OBJS) $(GTEST_OBJ) $(ENGINE_OBJS) -L$(PROJECT_ROOT) -lsqlparser -lpthread
+	$(CXX) $(CXXFLAGS) -o $@ $(TEST_OBJS) $(GTEST_OBJ) $(ENGINE_OBJS) -L$(PROJECT_ROOT) -lsqlparser -lpthread $(MYSQL_LIBS) $(PG_LIBS)
 
 # Benchmark objects
 $(GBENCH_DIR)/src/%.o: $(GBENCH_DIR)/src/%.cc

include/sql_engine/backend_config.h

@@ -0,0 +1,22 @@
+#ifndef SQL_ENGINE_BACKEND_CONFIG_H
+#define SQL_ENGINE_BACKEND_CONFIG_H
+
+#include "sql_parser/common.h"
+#include <cstdint>
+#include <string>
+
+namespace sql_engine {
+
+struct BackendConfig {
+    std::string name;       // logical name: "shard_1", "analytics_db"
+    std::string host;
+    uint16_t port = 0;
+    std::string user;
+    std::string password;
+    std::string database;
+    sql_parser::Dialect dialect = sql_parser::Dialect::MySQL;
+};
+
+} // namespace sql_engine
+
+#endif // SQL_ENGINE_BACKEND_CONFIG_H

include/sql_engine/datetime_parse.h

@@ -0,0 +1,25 @@
+#ifndef SQL_ENGINE_DATETIME_PARSE_H
+#define SQL_ENGINE_DATETIME_PARSE_H
+
+#include <cstdint>
+
+namespace sql_engine {
+namespace datetime_parse {
+
+// Parse "YYYY-MM-DD" to days since 1970-01-01.
+int32_t parse_date(const char* s);
+
+// Parse "YYYY-MM-DD HH:MM:SS[.uuuuuu]" to microseconds since epoch.
+int64_t parse_datetime(const char* s);
+
+// Parse "HH:MM:SS[.uuuuuu]" to microseconds since midnight.
+int64_t parse_time(const char* s);
+
+// Calendar math: compute days since 1970-01-01 for the given date.
+// Handles leap years correctly.
+int32_t days_since_epoch(int year, int month, int day);
+
+} // namespace datetime_parse
+} // namespace sql_engine
+
+#endif // SQL_ENGINE_DATETIME_PARSE_H

include/sql_engine/multi_remote_executor.h

@@ -0,0 +1,34 @@
+#ifndef SQL_ENGINE_MULTI_REMOTE_EXECUTOR_H
+#define SQL_ENGINE_MULTI_REMOTE_EXECUTOR_H
+
+#include "sql_engine/remote_executor.h"
+#include "sql_engine/mysql_remote_executor.h"
+#include "sql_engine/pgsql_remote_executor.h"
+#include "sql_engine/backend_config.h"
+#include "sql_parser/arena.h"
+#include "sql_parser/common.h"
+
+#include <unordered_map>
+#include <string>
+
+namespace sql_engine {
+
+class MultiRemoteExecutor : public RemoteExecutor {
+public:
+    explicit MultiRemoteExecutor(sql_parser::Arena& arena);
+    ~MultiRemoteExecutor() override;
+
+    void add_backend(const BackendConfig& config);
+    ResultSet execute(const char* backend_name, sql_parser::StringRef sql) override;
+    DmlResult execute_dml(const char* backend_name, sql_parser::StringRef sql) override;
+    void disconnect_all();
+
+private:
+    MySQLRemoteExecutor mysql_exec_;
+    PgSQLRemoteExecutor pgsql_exec_;
+    std::unordered_map<std::string, sql_parser::Dialect> backend_dialects_;
+};
+
+} // namespace sql_engine
+
+#endif // SQL_ENGINE_MULTI_REMOTE_EXECUTOR_H

include/sql_engine/mysql_remote_executor.h

@@ -0,0 +1,44 @@
+#ifndef SQL_ENGINE_MYSQL_REMOTE_EXECUTOR_H
+#define SQL_ENGINE_MYSQL_REMOTE_EXECUTOR_H
+
+#include "sql_engine/remote_executor.h"
+#include "sql_engine/backend_config.h"
+#include "sql_engine/value.h"
+#include "sql_engine/row.h"
+#include "sql_parser/arena.h"
+
+#include <mysql/mysql.h>
+#include <unordered_map>
+#include <string>
+
+namespace sql_engine {
+
+class MySQLRemoteExecutor : public RemoteExecutor {
+public:
+    explicit MySQLRemoteExecutor(sql_parser::Arena& arena);
+    ~MySQLRemoteExecutor() override;
+
+    void add_backend(const BackendConfig& config);
+    ResultSet execute(const char* backend_name, sql_parser::StringRef sql) override;
+    DmlResult execute_dml(const char* backend_name, sql_parser::StringRef sql) override;
+    void disconnect_all();
+
+private:
+    struct Connection {
+        BackendConfig config;
+        MYSQL* conn = nullptr;
+        bool connected = false;
+    };
+
+    std::unordered_map<std::string, Connection> backends_;
+    sql_parser::Arena& arena_;
+
+    Connection& get_or_connect(const std::string& name);
+    ResultSet mysql_result_to_resultset(MYSQL_RES* res);
+    Value mysql_field_to_value(const char* data, unsigned long length,
+                               enum_field_types type, bool is_null);
+};
+
+} // namespace sql_engine
+
+#endif // SQL_ENGINE_MYSQL_REMOTE_EXECUTOR_H

include/sql_engine/pgsql_remote_executor.h

@@ -0,0 +1,43 @@
+#ifndef SQL_ENGINE_PGSQL_REMOTE_EXECUTOR_H
+#define SQL_ENGINE_PGSQL_REMOTE_EXECUTOR_H
+
+#include "sql_engine/remote_executor.h"
+#include "sql_engine/backend_config.h"
+#include "sql_engine/value.h"
+#include "sql_engine/row.h"
+#include "sql_parser/arena.h"
+
+#include <libpq-fe.h>
+#include <unordered_map>
+#include <string>
+
+namespace sql_engine {
+
+class PgSQLRemoteExecutor : public RemoteExecutor {
+public:
+    explicit PgSQLRemoteExecutor(sql_parser::Arena& arena);
+    ~PgSQLRemoteExecutor() override;
+
+    void add_backend(const BackendConfig& config);
+    ResultSet execute(const char* backend_name, sql_parser::StringRef sql) override;
+    DmlResult execute_dml(const char* backend_name, sql_parser::StringRef sql) override;
+    void disconnect_all();
+
+private:
+    struct Connection {
+        BackendConfig config;
+        PGconn* conn = nullptr;
+        bool connected = false;
+    };
+
+    std::unordered_map<std::string, Connection> backends_;
+    sql_parser::Arena& arena_;
+
+    Connection& get_or_connect(const std::string& name);
+    ResultSet pg_result_to_resultset(PGresult* res);
+    Value pg_field_to_value(const char* data, int length, Oid type, bool is_null);
+};
+
+} // namespace sql_engine
+
+#endif // SQL_ENGINE_PGSQL_REMOTE_EXECUTOR_H

scripts/start_test_backends.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+
+echo "Starting test backends..."
+
+# Remove any existing containers
+docker rm -f parsersql-test-mysql parsersql-test-pgsql 2>/dev/null || true
+
+# Start MySQL 8
+docker run -d --name parsersql-test-mysql \
+    -p 13306:3306 \
+    -e MYSQL_ROOT_PASSWORD=test \
+    -e MYSQL_DATABASE=testdb \
+    mysql:8.0
+
+# Start PostgreSQL 16
+docker run -d --name parsersql-test-pgsql \
+    -p 15432:5432 \
+    -e POSTGRES_PASSWORD=test \
+    -e POSTGRES_DB=testdb \
+    postgres:16
+
+# Wait for MySQL to be ready
+echo "Waiting for MySQL..."
+for i in $(seq 1 60); do
+    if docker exec parsersql-test-mysql mysql -uroot -ptest -e "SELECT 1" &>/dev/null 2>&1; then
+        echo "MySQL ready after ${i}s"
+        break
+    fi
+    if [ "$i" -eq 60 ]; then
+        echo "MySQL failed to start"
+        exit 1
+    fi
+    sleep 1
+done
+
+# Wait for PostgreSQL to be ready
+echo "Waiting for PostgreSQL..."
+for i in $(seq 1 60); do
+    if docker exec parsersql-test-pgsql pg_isready -Upostgres &>/dev/null 2>&1; then
+        echo "PostgreSQL ready after ${i}s"
+        break
+    fi
+    if [ "$i" -eq 60 ]; then
+        echo "PostgreSQL failed to start"
+        exit 1
+    fi
+    sleep 1
+done
+
+# Load test data
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+echo "Loading MySQL test data..."
+docker exec -i parsersql-test-mysql mysql -uroot -ptest testdb < "${SCRIPT_DIR}/test_data_mysql.sql"
+echo "Loading PostgreSQL test data..."
+docker exec -i parsersql-test-pgsql psql -Upostgres testdb < "${SCRIPT_DIR}/test_data_pgsql.sql"
+
+echo "Backends ready!"

scripts/stop_test_backends.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker rm -f parsersql-test-mysql parsersql-test-pgsql 2>/dev/null || true
+echo "Test backends stopped."

scripts/test_data_mysql.sql

@@ -0,0 +1,29 @@
+DROP TABLE IF EXISTS orders;
+DROP TABLE IF EXISTS users;
+
+CREATE TABLE users (
+    id INT PRIMARY KEY,
+    name VARCHAR(255),
+    age INT,
+    dept VARCHAR(100)
+);
+
+INSERT INTO users VALUES
+    (1, 'Alice', 30, 'Engineering'),
+    (2, 'Bob', 25, 'Sales'),
+    (3, 'Carol', 35, 'Engineering'),
+    (4, 'Dave', 28, 'Sales'),
+    (5, 'Eve', 32, 'Engineering');
+
+CREATE TABLE orders (
+    id INT PRIMARY KEY,
+    user_id INT,
+    total DECIMAL(10,2),
+    status VARCHAR(50)
+);
+
+INSERT INTO orders VALUES
+    (101, 1, 150.00, 'completed'),
+    (102, 2, 75.50, 'pending'),
+    (103, 1, 200.00, 'completed'),
+    (104, 3, 50.00, 'cancelled');

scripts/test_data_pgsql.sql

@@ -0,0 +1,29 @@
+DROP TABLE IF EXISTS orders;
+DROP TABLE IF EXISTS users;
+
+CREATE TABLE users (
+    id INT PRIMARY KEY,
+    name VARCHAR(255),
+    age INT,
+    dept VARCHAR(100)
+);
+
+INSERT INTO users VALUES
+    (1, 'Alice', 30, 'Engineering'),
+    (2, 'Bob', 25, 'Sales'),
+    (3, 'Carol', 35, 'Engineering'),
+    (4, 'Dave', 28, 'Sales'),
+    (5, 'Eve', 32, 'Engineering');
+
+CREATE TABLE orders (
+    id INT PRIMARY KEY,
+    user_id INT,
+    total DECIMAL(10,2),
+    status VARCHAR(50)
+);
+
+INSERT INTO orders VALUES
+    (101, 1, 150.00, 'completed'),
+    (102, 2, 75.50, 'pending'),
+    (103, 1, 200.00, 'completed'),
+    (104, 3, 50.00, 'cancelled');