From a0206271d1c9aa81568826e3674d2723141d81e5 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Sat, 11 Jul 2026 11:48:44 -0700 Subject: [PATCH 1/2] pandas: handle non-contiguous memory Use the numpy backend to ensure all numpy arrays extracted from pandas columns are C-contiguous before scanning. Non-contiguous arrays can arise from DataFrames backed by views into larger 2D arrays (e.g. pd.DataFrame(arr2d), transpose, concat, slicing). Previously the scan used memcpy with sizeof(T) stride which reads wrong data on non-contiguous layouts. --- src_cpp/pandas/pandas_bind.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src_cpp/pandas/pandas_bind.cpp b/src_cpp/pandas/pandas_bind.cpp index 62672af..de36918 100644 --- a/src_cpp/pandas/pandas_bind.cpp +++ b/src_cpp/pandas/pandas_bind.cpp @@ -7,6 +7,18 @@ namespace lbug { using namespace lbug::common; +// Ensure a numpy array is C-contiguous so that the scan code can safely use +// memcpy on the raw data pointer. Non-contiguous arrays can arise from +// DataFrame columns backed by views into larger 2D arrays (e.g. after +// transpose, concat, or slicing). +static py::array ensureContiguous(py::object obj) { + auto arr = py::array(obj); + if (!arr.attr("flags").attr("c_contiguous").cast()) { + arr = py::module_::import("numpy").attr("ascontiguousarray")(arr); + } + return arr; +} + struct PandasBindColumn { public: PandasBindColumn(py::handle name, py::handle type, py::object column) @@ -54,9 +66,8 @@ static common::LogicalType bindColumn(PandasBindColumn& bindColumn, } if (bindData->npType.type == NumpyNullableType::FLOAT_16) { - auto pandasArray = column.attr("array"); bindData->pandasCol = - std::make_unique(py::array(column.attr("to_numpy")("float32"))); + std::make_unique(ensureContiguous(column.attr("to_numpy")("float32"))); bindData->npType.type = NumpyNullableType::FLOAT_32; columnType = NumpyTypeUtils::numpyToLogicalType(bindData->npType); } else { @@ -64,15 +75,15 @@ static common::LogicalType bindColumn(PandasBindColumn& bindColumn, if (py::hasattr(pandasArray, "_data")) { // This means we can access the numpy array directly. bindData->pandasCol = - std::make_unique(column.attr("array").attr("_data")); + std::make_unique(ensureContiguous(column.attr("array").attr("_data"))); } else if (py::hasattr(pandasArray, "asi8")) { // This is a datetime object, has the option to get the array as int64_t's. bindData->pandasCol = - std::make_unique(py::array(pandasArray.attr("asi8"))); + std::make_unique(ensureContiguous(pandasArray.attr("asi8"))); } else { // Otherwise we have to get it through 'to_numpy()'. bindData->pandasCol = - std::make_unique(py::array(column.attr("to_numpy")())); + std::make_unique(ensureContiguous(column.attr("to_numpy")())); } columnType = NumpyTypeUtils::numpyToLogicalType(bindData->npType); } From 2dde5012212490071978aaf888e789e3cc535802 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Sat, 11 Jul 2026 12:34:34 -0700 Subject: [PATCH 2/2] build: improve cmake generator handling, ccache, and build parallelism - Honour GEN env var (default to Ninja) - Wipe stale build directory on generator mismatch - Only clean build when BUILD_CLEAN=1 - Use ccache when available - Honour CMAKE_BUILD_PARALLEL_LEVEL / PARALLEL for parallelism - Handle artifact location flexibility --- scripts/build_pybind_from_subdir.sh | 95 +++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/scripts/build_pybind_from_subdir.sh b/scripts/build_pybind_from_subdir.sh index b26788f..711d7ff 100755 --- a/scripts/build_pybind_from_subdir.sh +++ b/scripts/build_pybind_from_subdir.sh @@ -31,24 +31,93 @@ export CCACHE_TEMPDIR mkdir -p "${CCACHE_TEMPDIR}" -rm -rf "${BUILD_DIR}" +# --------------------------------------------------------------------------- +# Generator selection +# --------------------------------------------------------------------------- +# Honour GEN env var (e.g. GEN=Unix\ Makefiles), otherwise default to Ninja. +if [[ -n "${GEN:-}" ]]; then + GENERATOR="${GEN}" +else + GENERATOR="Ninja" +fi +echo "[pybind] Generator: ${GENERATOR}" + +# If the build directory exists with a different generator, wipe it so +# CMake doesn't error out with a generator mismatch. +if [[ -f "${BUILD_DIR}/CMakeCache.txt" ]]; then + cached_gen="$(grep -m1 'CMAKE_GENERATOR:INTERNAL=' "${BUILD_DIR}/CMakeCache.txt" 2>/dev/null | cut -d= -f2)" + if [[ -n "${cached_gen}" && "${cached_gen}" != "${GENERATOR}" ]]; then + echo "[pybind] Generator changed from '${cached_gen}' to '${GENERATOR}' — wiping stale build directory." + rm -rf "${BUILD_DIR}" + fi +fi + +# --------------------------------------------------------------------------- +# CMake configuration (always re-runs if CMakeLists.txt changed) +# --------------------------------------------------------------------------- +# Only wipe the build tree when BUILD_CLEAN=1 is set explicitly. +if [[ "${BUILD_CLEAN:-0}" == "1" ]]; then + echo "[pybind] BUILD_CLEAN=1 — removing ${BUILD_DIR}" + rm -rf "${BUILD_DIR}" +fi + +cmake_args=( + -S "${ROOT_DIR}" + -B "${BUILD_DIR}" + -G "${GENERATOR}" + -DCMAKE_BUILD_TYPE=Release + -DLBUG_SOURCE_DIR="${LBUG_DIR}" + -DPYTHON_EXECUTABLE="${PYTHON_BIN}" + -DPython_EXECUTABLE="${PYTHON_BIN}" + -DPython3_EXECUTABLE="${PYTHON_BIN}" + -DPYBIND11_PYTHON_VERSION="${PYTHON_VERSION}" +) + +# Wire in ccache if it is available (vastly speeds up rebuilds). +if command -v ccache &>/dev/null; then + cmake_args+=( + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + ) +fi + +cmake "${cmake_args[@]}" + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- +# Parallelism: honour CMAKE_BUILD_PARALLEL_LEVEL or PARALLEL first, +# then fall back to the number of logical CPUs. +if [[ -n "${CMAKE_BUILD_PARALLEL_LEVEL:-}" ]]; then + NPROC="${CMAKE_BUILD_PARALLEL_LEVEL}" +elif [[ -n "${PARALLEL:-}" ]]; then + NPROC="${PARALLEL}" +else + NPROC="$(nproc 2>/dev/null || echo 4)" +fi -cmake \ - -S "${ROOT_DIR}" \ - -B "${BUILD_DIR}" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLBUG_SOURCE_DIR="${LBUG_DIR}" \ - -DPYTHON_EXECUTABLE="${PYTHON_BIN}" \ - -DPython_EXECUTABLE="${PYTHON_BIN}" \ - -DPython3_EXECUTABLE="${PYTHON_BIN}" \ - -DPYBIND11_PYTHON_VERSION="${PYTHON_VERSION}" +build_args=( + --build "${BUILD_DIR}" + --config Release + --target _lbug + --parallel "${NPROC}" +) -cmake --build "${BUILD_DIR}" --config Release --target _lbug +echo "[pybind] Starting build with ${NPROC} parallel job(s) ..." +cmake "${build_args[@]}" -if compgen -G "${ROOT_DIR}/build/ladybug/_lbug*" > /dev/null; then +# --------------------------------------------------------------------------- +# Post-build check +# --------------------------------------------------------------------------- +if compgen -G "${ROOT_DIR}/build/ladybug/_lbug*" &>/dev/null; then echo "[pybind] Built extension into ${ROOT_DIR}/build/ladybug" +elif compgen -G "${BUILD_DIR}/ladybug/_lbug*" &>/dev/null; then + echo "[pybind] Built extension into ${BUILD_DIR}/ladybug" + # Copy to expected location so Makefile's test target finds it. + mkdir -p "${ROOT_DIR}/build/ladybug" + cp "${BUILD_DIR}"/ladybug/_lbug* "${ROOT_DIR}/build/ladybug/" else echo "[pybind] Build finished, but no _lbug extension artifact was found." >&2 - echo "Checked: ${ROOT_DIR}/build/ladybug" >&2 + echo "Checked: ${ROOT_DIR}/build/ladybug and ${BUILD_DIR}/ladybug" >&2 exit 1 fi