diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcb38f4..ad20d03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,13 +7,6 @@ on: branches: [main] workflow_dispatch: {} -# Explicit least-privilege GITHUB_TOKEN scope. None of the jobs below write -# to the repo, comment on PRs, or publish anything - they only check out -# code, build/test it, and upload log artifacts - so read-only access to -# contents is all that's needed. Without this block, jobs run with the -# default token scope, which can be broader than necessary depending on -# repo/org settings (this is what CodeQL's "workflow does not contain -# permissions" alert flags). permissions: contents: read @@ -24,9 +17,6 @@ jobs: strategy: fail-fast: false matrix: - # Both compilers are exercised in CI since this project's behavior - # (SIMD dispatch, precision) has measurably differed between them - - # see CONTRIBUTING.md and the README's GCC vs. Clang benchmarks. include: - compiler: gcc cc: gcc @@ -49,9 +39,6 @@ jobs: python-version: "3.x" - name: Install system dependencies - # NOTE: per CMakeLists.txt, SLEEF and Google Benchmark are fetched - # from source via FetchContent at configure time - they are NOT - # system packages, so they are deliberately absent here. run: | sudo apt-get update sudo apt-get install -y \ @@ -82,15 +69,6 @@ jobs: build-and-test-windows: name: build-and-test (windows, clang) runs-on: windows-latest - # NOTE: unlike the Linux job above, this one does NOT go through - # build.py. build.py's configure step is hardcoded to - # `-DCMAKE_BUILD_TYPE=... -DDEEPITY_BUILD_TESTS=ON` with no way to pass - # extra defines, but Windows+Clang needs several more (OpenMP_libomp_ - # LIBRARY, CMAKE_TOOLCHAIN_FILE, BLA_VENDOR). Those live in the - # "windows-clang" preset in CMakePresets.json, which this job invokes - # directly via `cmake --preset` - see the Configure step below. If - # build.py grows support for passthrough CMake args or preset selection, - # this job should switch to using it, the same as the Linux job. steps: - name: Check out repository uses: actions/checkout@v4 @@ -194,3 +172,69 @@ jobs: - name: Run pyright run: pyright + + build-wheels: + name: wheels (${{ matrix.os }}) + needs: [build-and-test, build-and-test-windows] + strategy: + fail-fast: false + matrix: + # macOS intentionally excluded for now: macos-latest runners are + # arm64, and CMakeLists.txt's SIMD flags (DEEPITY_ARCH_FLAGS's + # x86-64-v2, SLEEF_ENABLE_AVX2/AVX512F) are x86-only with no + # arch-conditional branching yet. + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + # --- Linux and macOS need no host-side setup: OpenBLAS/libgomp + # install inside the manylinux container (Linux) or via Homebrew + # (macOS) through the before-all hooks in pyproject.toml's + # [tool.cibuildwheel.linux] / [tool.cibuildwheel.macos] tables. + - name: Build wheels + uses: pypa/cibuildwheel@v2.21 + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse/*.whl + if-no-files-found: error + + build-sdist: + name: sdist + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install build + run: pip install build + + - name: Build sdist + # NOT `python -m build` - `-m` makes Python search sys.path with the + # current working directory prepended, and this repo has a + # `build.py` at the root, which shadows the installed `build` + # package entirely (the error you'd see is literally build.py's own + # --help text). `pyproject-build` is the console-script entry point + # the `build` package ships specifically to avoid this collision. + run: pyproject-build --sdist + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error + + # Not wired up yet: publishing needs the PyPI project configured for + # Trusted Publishing (pypi.org project settings -> Publishing) with this + # repo/workflow registered, then a job here gated on a tag push using + # `pypa/gh-action-pypi-publish` and `permissions: id-token: write` - + # no API token needs to live in repo secrets with that approach. diff --git a/.gitignore b/.gitignore index cb7e3f1..a6d472d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .vscode/* -build/* -bin/* +build/ +bin/ __pycache__/* MNIST/* pyrightconfig.json @@ -10,6 +10,7 @@ repomix-output.xml experiments/outputs/ experiments/data/ experiments/checkpoints/ + *.lib *.dll *.so @@ -17,3 +18,11 @@ experiments/checkpoints/ *.exe *.obj *.o +CMakeCache.txt +CMakeFiles/ +.ninja_log + +wheelhouse/ +dist/ +*.egg-info/ +_skbuild/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index a2e0187..a79ec99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,33 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) option(DEEPITY_ENABLE_CUDA "Build with CUDA support" ON) option(DEEPITY_BUILD_TESTS "Build the DeepityTests target" OFF) option(DEEPITY_BUILD_PYTHON_BINDINGS "Build the Python bindings target" ON) -option(DEEPITY_NATIVE_OPT "Enable -march=native for max performance (disable for portable builds)" ON) + +# --- Global Architecture & SIMD Flags ------------------------------- +# NOTE: build.py always sets these explicitly based on --native/--fast/ +# --distributed. These CACHE defaults only matter if someone invokes cmake +# directly. Default to the portable "fast" profile (AVX2+FMA, x86-64-v3) +# rather than -march=native, since native binaries crash on other CPUs. +set(DEEPITY_ARCH_FLAGS "-march=x86-64-v3 -mtune=generic" CACHE STRING "Architecture/SIMD flags (e.g., -march=native, -march=x86-64-v2)") +set(DEEPITY_MSVC_ARCH_FLAGS "/arch:AVX2" CACHE STRING "Architecture flags for MSVC (empty = compiler default / most portable)") + +if(MSVC) + if(DEEPITY_MSVC_ARCH_FLAGS) + add_compile_options(${DEEPITY_MSVC_ARCH_FLAGS}) + endif() + + if(DEEPITY_MSVC_ARCH_FLAGS MATCHES "AVX512") + add_compile_definitions(__SSE2__ __AVX__ __AVX2__ __AVX512F__) + elseif(DEEPITY_MSVC_ARCH_FLAGS MATCHES "AVX2") + add_compile_definitions(__SSE2__ __AVX__ __AVX2__) + elseif(DEEPITY_MSVC_ARCH_FLAGS MATCHES "AVX") + add_compile_definitions(__SSE2__ __AVX__) + else() + add_compile_definitions(__SSE2__) + endif() +else() + separate_arguments(ARCH_FLAGS_LIST UNIX_COMMAND ${DEEPITY_ARCH_FLAGS}) + add_compile_options(${ARCH_FLAGS_LIST}) +endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -18,9 +44,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) message(STATUS "CXX compiler: ${CMAKE_CXX_COMPILER}") -# -------------------------------------------------------------------- -# Dependencies -# -------------------------------------------------------------------- +# --- DEPENDENCIES --------------------------------------------------- find_package(OpenMP REQUIRED) @@ -40,9 +64,7 @@ endif() include(FetchContent) -# -------------------------------------------------------------------- -# OpenBLAS -# -------------------------------------------------------------------- +# --- OpenBLAS ------------------------------------------------------- if(WIN32) set(OPENBLAS_WIN_URL "https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.34/OpenBLAS-0.3.34-x64.zip" @@ -67,6 +89,7 @@ if(WIN32) PATHS "${openblas_prebuilt_SOURCE_DIR}/bin" REQUIRED ) + message(STATUS "OPENBLAS_DLL resolved to: ${OPENBLAS_DLL}") add_library(openblas_dll SHARED IMPORTED) set_target_properties(openblas_dll PROPERTIES @@ -84,10 +107,8 @@ else() ) endif() -# -------------------------------------------------------------------- -# SLEEF -# -------------------------------------------------------------------- - +# --- SLEEF ---------------------------------------------------------- +# These are necessary, but don't ask me why set(SLEEF_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(SLEEF_BUILD_STATIC_TEST_BINS OFF CACHE BOOL "" FORCE) set(SLEEF_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) @@ -115,9 +136,7 @@ set(CMAKE_REQUIRED_QUIET ON) FetchContent_MakeAvailable(sleef) set(CMAKE_REQUIRED_QUIET OFF) -# -------------------------------------------------------------------- -# Google Benchmark -# -------------------------------------------------------------------- +# --- Google Benchmark ----------------------------------------------- if(DEEPITY_BUILD_TESTS) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -140,11 +159,9 @@ if(DEEPITY_BUILD_TESTS) set(CMAKE_REQUIRED_QUIET OFF) endif() -# -------------------------------------------------------------------- -# Deepity library -# -------------------------------------------------------------------- +# --- Deepity library ------------------------------------------------ -add_library(Deepity STATIC +add_library(Deepity src/DiscriminativePCLayer.cpp src/RBLayer.cpp src/ConvPCLayer.cpp @@ -158,10 +175,6 @@ add_library(Deepity STATIC src/StreamAlignedBatcher.cpp ) -if(NOT MSVC AND DEEPITY_NATIVE_OPT) - target_compile_options(Deepity PUBLIC -march=native) -endif() - target_include_directories(Deepity PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${BLAS_INCLUDE_DIR} @@ -185,9 +198,7 @@ endif() target_compile_definitions(Deepity PUBLIC SLEEF_STATIC_LIBS) -# -------------------------------------------------------------------- -# Compiler flags -# -------------------------------------------------------------------- +# --- Compiler flags ------------------------------------------------- if(MSVC) target_compile_options(Deepity PUBLIC @@ -195,14 +206,6 @@ if(MSVC) $<$:/O2 /Zi /fp:fast /openmp:llvm> $<$:/Od /Zi /W4 /openmp:llvm> ) - target_compile_definitions(Deepity PUBLIC __SSE2__) - - # Note: MSVC handles vectorization under /O2 automatically. - # /arch:AVX2 can be added if you want to force native-like behavior. - if(DEEPITY_NATIVE_OPT) - target_compile_definitions(Deepity PUBLIC __AVX__ __AVX2__) - target_compile_options(Deepity PUBLIC /arch:AVX2) - endif() endif() target_compile_definitions(Deepity PUBLIC $<$:_DEBUG>) @@ -216,15 +219,12 @@ set_target_properties(Deepity PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin ) -# -------------------------------------------------------------------- -# Python bindings -# -------------------------------------------------------------------- +# --- pydeepity ------------------------------------------------------ if(DEEPITY_BUILD_PYTHON_BINDINGS AND NOT CMAKE_BUILD_TYPE STREQUAL "Debug") pybind11_add_module(pydeepity bindings/pybinding.cpp) target_link_libraries(pydeepity PRIVATE Deepity) - target_compile_options(pydeepity PRIVATE -march=native) if(WIN32) target_compile_definitions(pydeepity PRIVATE NOMINMAX) @@ -234,33 +234,33 @@ if(DEEPITY_BUILD_PYTHON_BINDINGS AND NOT CMAKE_BUILD_TYPE STREQUAL "Debug") PREFIX "" ) - # Modern cross-platform way to copy required DLLs (like OpenMP/BLAS) to the Python package on Windows - if(WIN32) - # Copy .pyd - add_custom_command(TARGET pydeepity POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - $ - ${CMAKE_SOURCE_DIR}/pydeepity/ - ) - # Copy required DLLs - add_custom_command(TARGET pydeepity POST_BUILD - COMMAND ${CMAKE_COMMAND} -E $>,copy_if_different,true> - $ - ${CMAKE_SOURCE_DIR}/pydeepity/ - COMMAND_EXPAND_LISTS - ) - endif() + # Modern cross-platform way to copy the compiled extension and required runtime libs + add_custom_command(TARGET pydeepity POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + $ + ${CMAKE_SOURCE_DIR}/pydeepity/ + ) + add_custom_command(TARGET pydeepity POST_BUILD + COMMAND ${CMAKE_COMMAND} -E $>,copy_if_different,true> + $ + ${CMAKE_SOURCE_DIR}/pydeepity/ + COMMAND_EXPAND_LISTS + ) + + install(TARGETS pydeepity + LIBRARY DESTINATION pydeepity + RUNTIME DESTINATION pydeepity + ) endif() -# -------------------------------------------------------------------- -# Tests -# -------------------------------------------------------------------- +if(WIN32) + install(FILES ${OPENBLAS_DLL} DESTINATION pydeepity) +endif() +# --- Tests ---------------------------------------------------------- if(DEEPITY_BUILD_TESTS) add_executable(DeepityTests tests/tSimpleConvVerify.cpp) - target_compile_options(DeepityTests PRIVATE -march=native) - target_link_libraries(DeepityTests PRIVATE Deepity benchmark::benchmark @@ -270,7 +270,6 @@ if(DEEPITY_BUILD_TESTS) RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin ) - # Copy DLLs for the test executable on Windows if(WIN32) add_custom_command(TARGET DeepityTests POST_BUILD COMMAND ${CMAKE_COMMAND} -E $>,copy_if_different,true> @@ -281,9 +280,20 @@ if(DEEPITY_BUILD_TESTS) endif() endif() -# -------------------------------------------------------------------- -# Profiling -# -------------------------------------------------------------------- +# --- Profiling ------------------------------------------------------ +if(DEEPITY_BUILD_TESTS) + add_executable(Layer1Isolate513 tests/t513.cpp) + target_link_libraries(Layer1Isolate513 PRIVATE Deepity) + set_target_properties(Layer1Isolate513 PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + if(WIN32) + add_custom_command(TARGET Layer1Isolate513 POST_BUILD + COMMAND ${CMAKE_COMMAND} -E $>,copy_if_different,true> + $ + $ + COMMAND_EXPAND_LISTS + ) + endif() +endif() if(DEEPITY_BUILD_TESTS) add_library(DeepityProfiled STATIC @@ -299,8 +309,6 @@ if(DEEPITY_BUILD_TESTS) src/StreamAlignedBatcher.cpp ) - target_compile_options(DeepityProfiled PRIVATE -march=native) - target_include_directories(DeepityProfiled PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${BLAS_INCLUDE_DIR} @@ -322,16 +330,11 @@ if(DEEPITY_BUILD_TESTS) endif() if(MSVC) - target_compile_definitions(DeepityProfiled PUBLIC __SSE2__) - if(DEEPITY_NATIVE_OPT) - target_compile_definitions(DeepityProfiled PUBLIC __AVX__ __AVX2__) - target_compile_options(DeepityProfiled PUBLIC /arch:AVX2) - endif() target_compile_options(DeepityProfiled PRIVATE /O2 /fp:fast /openmp:llvm) elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(DeepityProfiled PRIVATE -O3 ${ARCH_FLAGS} -ffast-math -fvectorize -fslp-vectorize) + target_compile_options(DeepityProfiled PRIVATE -O3 -ffast-math -fvectorize -fslp-vectorize) else() - target_compile_options(DeepityProfiled PRIVATE -O3 ${ARCH_FLAGS} -ffast-math -ftree-vectorize -ftree-slp-vectorize) + target_compile_options(DeepityProfiled PRIVATE -O3 -ffast-math -ftree-vectorize -ftree-slp-vectorize) endif() add_executable(DeepityProfile tests/tProfile.cpp) @@ -351,17 +354,3 @@ if(DEEPITY_BUILD_TESTS) ) endif() endif() - -if(DEEPITY_BUILD_TESTS) - add_executable(MuCacheVerify tests/tMuCacheVerify.cpp) - target_link_libraries(MuCacheVerify PRIVATE Deepity) - set_target_properties(MuCacheVerify PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - if(WIN32) - add_custom_command(TARGET MuCacheVerify POST_BUILD - COMMAND ${CMAKE_COMMAND} -E $>,copy_if_different,true> - $ - $ - COMMAND_EXPAND_LISTS - ) - endif() -endif() diff --git a/bindings/pybinding.cpp b/bindings/pybinding.cpp index e8b41fe..2c41922 100644 --- a/bindings/pybinding.cpp +++ b/bindings/pybinding.cpp @@ -150,7 +150,7 @@ namespace .def_property_readonly("layers", [](NetT &self) { py::list result; - for (auto *layer : self.GetLayers()) result.append(py::cast(layer, py::return_value_policy::reference)); + for (const auto &layer : self.GetLayers()) result.append(py::cast(layer.get(), py::return_value_policy::reference)); return result; }, "List of layer objects owned by the network.") .def("__len__", [](const NetT &self) { return self.GetLayers().size(); }) @@ -159,7 +159,7 @@ namespace auto &layers = self.GetLayers(); if (index < 0) index += static_cast(layers.size()); if (index < 0 || index >= static_cast(layers.size())) throw py::index_error(); - return layers[index]; }, py::return_value_policy::reference_internal) + return layers[index].get(); }, py::return_value_policy::reference_internal) .def("__repr__", [className](const NetT &self) { return "<" + std::string(className) + " layers=" + std::to_string(self.GetLayers().size()) + " batch_size=" + std::to_string(self.GetBatchSize()) + ">"; }); } @@ -263,7 +263,7 @@ void bind_layers(py::module_ &m) .def_property_readonly("weights", [](Deep::ConvPCLayer &self) { return py::array_t({(py::ssize_t)self.GetOutChannels(), (py::ssize_t)self.GetInChannels(), (py::ssize_t)self.GetKernelH(), (py::ssize_t)self.GetKernelW()}, self.GetWeights(), py::cast(&self)); }) .def_property_readonly("biases", [](Deep::ConvPCLayer &self) - { return py::array_t({(py::ssize_t)self.GetOutChannels()}, self.GetBiases(), py::cast(&self)); }) + { return py::array_t((py::ssize_t)self.GetOutChannels(), self.GetBiases(), py::cast(&self)); }) .def_property_readonly("batch_size", &Deep::ConvPCLayer::GetBatchSize) .def_property_readonly("in_channels", &Deep::ConvPCLayer::GetInChannels) .def_property_readonly("out_channels", &Deep::ConvPCLayer::GetOutChannels) @@ -282,11 +282,11 @@ void bind_layers(py::module_ &m) .def_property_readonly("beliefs", [](Deep::SimpleConvPCLayer &self) { size_t n = self.GetBatchSize() * self.GetInputSize(); - return py::array_t({(py::ssize_t)n}, self.GetBeliefs()); }) + return py::array_t((py::ssize_t)n, self.GetBeliefs()); }) .def_property_readonly("errors", [](Deep::SimpleConvPCLayer &self) { size_t n = self.GetBatchSize() * self.GetInputSize(); - return py::array_t({(py::ssize_t)n}, self.GetErrors()); }) + return py::array_t((py::ssize_t)n, self.GetErrors()); }) .def_property_readonly("weights", [](Deep::SimpleConvPCLayer &self) { if (self.GetOutChannels() == 0) return py::array_t(); @@ -295,7 +295,7 @@ void bind_layers(py::module_ &m) .def_property_readonly("biases", [](Deep::SimpleConvPCLayer &self) { if (self.GetOutChannels() == 0) return py::array_t(); - return py::array_t({(py::ssize_t)self.GetOutChannels()}, self.GetBiases()); }) + return py::array_t((py::ssize_t)self.GetOutChannels(), self.GetBiases()); }) .def_property_readonly("in_channels", &Deep::SimpleConvPCLayer::GetInChannels) .def_property_readonly("out_channels", &Deep::SimpleConvPCLayer::GetOutChannels) .def_property_readonly("in_height", &Deep::SimpleConvPCLayer::GetInHeight) @@ -349,12 +349,18 @@ void bind_networks(py::module_ &m) { if (opt == "ADAM") self.SetOptimizer(Deep::OptimizerType::ADAM); else if (opt == "ADAMW") self.SetOptimizer(Deep::OptimizerType::ADAMW); - else self.SetOptimizer(Deep::OptimizerType::SGD); }, py::arg("optimizer"), "Sets the optimizer: ADAM, ADAMW, or SGD."); + else self.SetOptimizer(Deep::OptimizerType::SGD); }, py::arg("optimizer"), "Sets the optimizer: ADAM, ADAMW, or SGD.") - simpleNetCls.def("project_forward", &Deep::SimplePCNetwork::ProjectForward, - "Seeds hidden layers from a genuine forward pass through current " - "weights, instead of zero-init. Call AFTER clamp_input(), BEFORE " - "the settling loop."); + .def("project_forward", &Deep::SimplePCNetwork::ProjectForward, "Seeds hidden layers from a genuine forward pass through current " + "weights, instead of zero-init. Call AFTER clamp_input(), BEFORE " + "the settling loop.") + + .def("train_step_with_projection", [](Deep::SimplePCNetwork &self, py::array_t x, py::array_t y, int steps) + { + auto xbuf = x.request(); auto ybuf = y.request(); + std::vector xvec(static_cast(xbuf.ptr), static_cast(xbuf.ptr) + xbuf.size); + std::vector yvec(static_cast(ybuf.ptr), static_cast(ybuf.ptr) + ybuf.size); + return self.TrainStepWithProjection(xvec, yvec, steps); }, py::arg("x"), py::arg("y"), py::arg("steps")); py::class_(m, "ConvPCNetwork", "Convolutional Predictive Coding Network.") .def(py::init(), py::arg("batch_size"), "Construct a network with a fixed batch size.") @@ -394,7 +400,7 @@ void bind_networks(py::module_ &m) .def_property_readonly("layers", [](Deep::ConvPCNetwork &self) { py::list result; - for (auto *layer : self.GetLayers()) result.append(py::cast(layer, py::return_value_policy::reference)); + for (auto &layer : self.GetLayers()) result.append(py::cast(layer.get(), py::return_value_policy::reference)); return result; }) .def("__len__", [](const Deep::ConvPCNetwork &self) { return self.GetLayers().size(); }) @@ -403,7 +409,7 @@ void bind_networks(py::module_ &m) auto &layers = self.GetLayers(); if (index < 0) index += static_cast(layers.size()); if (index < 0 || index >= static_cast(layers.size())) throw py::index_error(); - return layers[index]; }, py::return_value_policy::reference_internal) + return layers[index].get(); }, py::return_value_policy::reference_internal) .def("__repr__", [](const Deep::ConvPCNetwork &self) { return ""; }); @@ -449,7 +455,7 @@ void bind_networks(py::module_ &m) .def_property_readonly("layers", [](Deep::SimpleConvPCNetwork &self) { py::list result; - for (auto *layer : self.GetLayers()) result.append(py::cast(layer, py::return_value_policy::reference)); + for (auto &layer : self.GetLayers()) result.append(py::cast(layer.get(), py::return_value_policy::reference)); return result; }) .def("__len__", [](const Deep::SimpleConvPCNetwork &self) { return self.GetLayers().size(); }) @@ -458,7 +464,7 @@ void bind_networks(py::module_ &m) auto &layers = self.GetLayers(); if (index < 0) index += static_cast(layers.size()); if (index < 0 || index >= static_cast(layers.size())) throw py::index_error(); - return layers[index]; }, py::return_value_policy::reference_internal) + return layers[index].get(); }, py::return_value_policy::reference_internal) .def("__repr__", [](const Deep::SimpleConvPCNetwork &self) { return ""; }); } @@ -521,4 +527,4 @@ PYBIND11_MODULE(pydeepity, m) bind_layers(m); bind_networks(m); bind_utilities(m); -} +} \ No newline at end of file diff --git a/build.py b/build.py index 083bf32..40286c4 100644 --- a/build.py +++ b/build.py @@ -1,902 +1,14 @@ -import argparse -import multiprocessing -import os -import re -import shutil -import subprocess -import sys -import time -from abc import ABC, abstractmethod -from collections import deque -from collections.abc import Callable -from pathlib import Path -import importlib.util +#!/usr/bin/env python3 +""" +Deepity build entrypoint. +All real logic lives in the deepity_build package (config resolution, the +CMake command builder, and the rich/plain reporters). This file just wires +argv to that package so `python build.py ...` keeps working exactly as +before. +""" -def is_library_installed(library_name: str) -> bool: - """ - Check if a python library is installed without importing it. - - Args: - library_name (str): The name of the library to check - Returns: - bool: True if installed, False otherwise - """ - if not isinstance(library_name, str) or not library_name.strip(): - raise ValueError("Library name must be a non-empty string.") - return importlib.util.find_spec(library_name) is not None - - -try: - RICH_AVAILABLE = is_library_installed("rich") -except ValueError as e: - print(f"Error: {e}") - RICH_AVAILABLE = False - -if RICH_AVAILABLE: - print("✅ 'rich' is installed. Using the interactive dashboard.") -else: - print("❌ 'rich' is NOT installed. Falling back to plain-text output.") - - -BUILD_PROGRESS_RE = re.compile(r"\[(\d+)/(\d+)\]\s+(.*)") - - -def format_duration(seconds: float | None) -> str: - if seconds is None: - return "—" - - if seconds < 60: - return f"{seconds:.1f}s" - - minutes, seconds = divmod(seconds, 60) - - if minutes < 60: - return f"{int(minutes)}m {seconds:.0f}s" - - hours, minutes = divmod(int(minutes), 60) - return f"{hours}h {minutes}m" - - -def command_string(cmd: list[str]) -> str: - return " ".join(cmd) - -def get_git_info() -> tuple[str, str, bool]: - try: - branch = subprocess.check_output( - ["git", "branch", "--show-current"], - text=True, - stderr=subprocess.DEVNULL, - ).strip() - - commit = subprocess.check_output( - ["git", "rev-parse", "--short", "HEAD"], - text=True, - stderr=subprocess.DEVNULL, - ).strip() - - status = subprocess.call( - ["git", "diff", "--quiet"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - dirty = status != 0 - return branch or "detached", commit, dirty - - except (subprocess.CalledProcessError, FileNotFoundError): - return "unknown", "unknown", False - -# ══════════════════════════════════════════════════════════════════════════ -# Reporter interface -# -# main() only ever talks to this interface, never to rich or plain-text -# internals directly. That keeps main() free of "if RICH_AVAILABLE" checks -# (aside from picking which implementation to construct) and means neither -# implementation needs module-level conditional imports of rich symbols - -# each one imports what it needs locally, right where it's used, which is -# also what keeps static type checkers (Pylance/pyright) happy: nothing is -# "possibly unbound" because every rich import lives inside the exact -# function that uses it, in a class that is only ever instantiated when -# rich is actually installed. -# ══════════════════════════════════════════════════════════════════════════ -class Reporter(ABC): - @abstractmethod - def __enter__(self) -> "Reporter": ... - - @abstractmethod - def __exit__(self, exc_type, exc, tb) -> None: ... - - @abstractmethod - def build_summary(self, targets: int) -> None: ... - - @abstractmethod - def clean_reconfigure(self) -> None: ... - - @abstractmethod - def configure_started(self) -> None: ... - - @abstractmethod - def configure_cached(self) -> None: ... - - @abstractmethod - def configure_complete(self, duration: float) -> None: ... - - @abstractmethod - def configure_failed(self, output: str) -> None: ... - - @abstractmethod - def build_started(self) -> None: ... - - @abstractmethod - def build_line(self, line: str) -> None: ... - - @abstractmethod - def build_complete(self, duration: float) -> None: ... - - @abstractmethod - def build_failed(self, output: str) -> None: ... - - @abstractmethod - def tests_missing(self, exe_name: str, paths: list[Path]) -> None: ... - - @abstractmethod - def tests_started(self) -> None: ... - - @abstractmethod - def test_line(self, line: str) -> None: ... - - @abstractmethod - def tests_complete(self, duration: float) -> None: ... - - @abstractmethod - def tests_failed(self, output: str) -> None: ... - - @abstractmethod - def success( - self, - configure_time: float | None, - build_time: float, - test_time: float, - ) -> None: ... - - -# ══════════════════════════════════════════════════════════════════════════ -# Rich-backed reporter (used only when 'rich' is available) -# ══════════════════════════════════════════════════════════════════════════ -class RichReporter(Reporter): - """Single mutable Rich Live dashboard for the entire build.""" - - def __init__( - self, - build_type: str, - generator: str, - jobs: int, - git_branch: str, - git_commit: str, - git_dirty: bool, - ) -> None: - from rich.console import Console - from rich.progress import ( - BarColumn, - Progress, - SpinnerColumn, - TextColumn, - TimeElapsedColumn, - ) - - self.build_type = build_type - self.generator = generator - self.jobs = jobs - - self.git_branch = git_branch - self.git_commit = git_commit - self.git_dirty = git_dirty - - self.targets_built = 0 - self.num_passed = 0 - self.num_failed = 0 - - self.phase = "Starting..." - self.configure_status = "[dim]waiting[/dim]" - self.build_status = "[dim]waiting[/dim]" - self.test_status = "[dim]waiting[/dim]" - - self.build_message = "" - self.test_lines: deque[str] = deque(maxlen=8) - - self._configure_time_display = "—" - self._build_time_display = "—" - self._test_time_display = "—" - self._total_time_display = "—" - - self.console = Console() - self.progress = Progress( - SpinnerColumn(), - TextColumn("[bold blue]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - ) - self.build_task = self.progress.add_task("Waiting", total=1, completed=0) - - self._live = None - - # -- context manager: owns the Live session -------------------------- - def __enter__(self) -> "RichReporter": - from rich.live import Live - - self.console.clear() - self.console.print() - - self._live = Live( - self._render(), - refresh_per_second=12, - console=self.console, - transient=False, - ) - self._live.__enter__() - return self - - def __exit__(self, exc_type, exc, tb) -> None: - if self._live is not None: - self._live.__exit__(exc_type, exc, tb) - - # -- rendering --------------------------------------------------------- - def _header(self): - from rich.panel import Panel - from rich.table import Table - - table = Table.grid(padding=(0, 2)) - table.add_column() - table.add_column() - table.add_column() - table.add_column() - - table.add_row( - "[bold cyan]Deepity Engine[/bold cyan]", - f"[dim]Build[/dim] [bold]{self.build_type}[/bold]", - f"[dim]Generator[/dim] [bold]{self.generator}[/bold]", - f"[dim]Jobs[/dim] [bold]{self.jobs}[/bold]", - ) - - table.add_row( - "[dim]Git[/dim]", - f"[bold]{self.git_branch}[/bold] @ {self.git_commit}" - + (" [yellow]● modified[/yellow]" if self.git_dirty else " [green]✓ clean[/green]"), - ) - - return Panel(table, border_style="cyan", padding=(0, 1)) - - def _status(self): - from rich.panel import Panel - from rich.table import Table - - table = Table.grid(padding=(0, 2)) - table.add_column(style="bold", width=12) - table.add_column() - - table.add_row("Configure", self.configure_status) - table.add_row("Build", self.build_status) - table.add_row("Tests", self.test_status) - - return Panel(table, title="[bold]Build Status[/bold]", border_style="blue") - - def _activity(self): - from rich.console import Group - from rich.panel import Panel - from rich.table import Table - from rich.text import Text - - if self.phase == "Configuring": - body = Group( - Text("Configuring CMake...", style="bold yellow"), - Text(""), - Text.from_markup( - "[dim]CMake output is hidden while the build is running " - "and shown in full if configuration fails.[/dim]" - ), - ) - return Panel( - body, - title="[bold yellow]⚙ Configuration[/bold yellow]", - border_style="yellow", - ) - - if self.phase == "Building": - current = self.build_message or "Working..." - return Panel( - Group(self.progress, Text.from_markup(f"[dim]{current}[/dim]")), - title="[bold blue]⚙ Compilation[/bold blue]", - border_style="blue", - ) - - if self.phase == "Testing": - body = ( - "\n".join(self.test_lines) - if self.test_lines - else "[dim]Waiting for test output...[/dim]" - ) - return Panel( - body, - title="[bold magenta]▶ Tests[/bold magenta]", - border_style="magenta", - ) - - if self.phase == "Success": - body = Table.grid(padding=(0, 2)) - body.add_column(style="dim") - body.add_column(style="bold") - - body.add_row("Build type", self.build_type) - body.add_row("Generator", self.generator) - body.add_row("Parallel jobs", str(self.jobs)) - body.add_row("Targets", str(self.targets_built)) - body.add_row("Configuration", self._configure_time_display) - body.add_row("Compilation", self._build_time_display) - body.add_row("Tests", self._test_time_display) - body.add_row("Total", self._total_time_display) - - return Panel( - Group( - Text("✓ BUILD SUCCESSFUL", style="bold green", justify="center"), - Text(""), - body, - ), - title="[bold green]Deepity[/bold green]", - border_style="green", - padding=(1, 2), - ) - - return Panel("[dim]Waiting...[/dim]", title="[bold]Activity[/bold]", border_style="dim") - - def _render(self): - from rich.console import Group - - return Group(self._header(), self._status(), self._activity()) - - def _refresh(self) -> None: - if self._live is not None: - self._live.update(self._render(), refresh=True) - - def _print_failure(self, title: str, body: str) -> None: - from rich.panel import Panel - - body = body.rstrip() or "No output" - self.console.print() - self.console.print(Panel(body, title=f"[bold red]{title}[/bold red]", border_style="red")) - - # -- Reporter interface ------------------------------------------------- - - def build_summary(self, targets: int) -> None: - self.targets_built = targets - self._refresh() - - def clean_reconfigure(self) -> None: - self.phase = "Configuring" - self.configure_status = "[yellow]● clean reconfigure[/yellow]" - self._refresh() - - def configure_started(self) -> None: - self.phase = "Configuring" - self.configure_status = "[yellow]● configuring[/yellow]" - self._refresh() - - def configure_cached(self) -> None: - self.configure_status = "[green]✓ cached[/green]" - self._refresh() - - def configure_complete(self, duration: float) -> None: - self.configure_status = ( - f"[bold green]✓ complete[/bold green] [dim]({format_duration(duration)})[/dim]" - ) - self._refresh() - - def configure_failed(self, output: str) -> None: - self._print_failure("✗ CMake Configuration Failed", output) - - def build_started(self) -> None: - self.phase = "Building" - self.build_status = "[yellow]● compiling[/yellow]" - self.progress.update(self.build_task, total=1, completed=0, description="Compiling") - self._refresh() - - def build_line(self, line: str) -> None: - match = BUILD_PROGRESS_RE.search(line) - - if not match: - stripped = line.strip() - if stripped: - self.build_message = stripped[-160:] - self._refresh() - return - - current = int(match.group(1)) - total = int(match.group(2)) - message= match.group(3).strip() - self.targets_built = max(self.targets_built, total) - self.build_message = message - self.progress.update(self.build_task, total=total, completed=current) - - def build_complete(self, duration: float) -> None: - self.build_status = ( - f"[bold green]✓ complete[/bold green] [dim]({format_duration(duration)})[/dim]" - ) - self._refresh() - - def build_failed(self, output: str) -> None: - self.build_status = "[bold red]✗ failed[/bold red]" - self.phase = "Build failed" - self._refresh() - self._print_failure("✗ Build Failed", output) - - def tests_missing(self, exe_name: str, paths: list[Path]) -> None: - self.test_status = "[bold red]✗ binary missing[/bold red]" - self.phase = "Tests unavailable" - self._refresh() - self._print_failure(f"✗ Could Not Find {exe_name}", "\n".join(str(p) for p in paths)) - - def tests_started(self) -> None: - self.phase = "Testing" - self.test_status = "[yellow]● running[/yellow]" - self.test_lines.clear() - self._refresh() - - def test_line(self, line: str) -> None: - line = line.rstrip() - - if line: - self.test_lines.append(line) - self._refresh() - - def tests_complete(self, duration: float) -> None: - self.test_status = ( - f"[bold green]✓ complete[/bold green] [dim]({format_duration(duration)})[/dim]" - ) - self._refresh() - - def tests_failed(self, output: str) -> None: - self.test_status = "[bold red]✗ failed[/bold red]" - self.phase = "Tests failed" - self._refresh() - self._print_failure("✗ Test Suite Failed", output) - - def success( - self, - configure_time: float | None, - build_time: float, - test_time: float, - ) -> None: - self._configure_time_display = ( - format_duration(configure_time) if configure_time else "cached" - ) - self._build_time_display = format_duration(build_time) - self._test_time_display = format_duration(test_time) - - total = sum(value or 0 for value in (configure_time, build_time, test_time)) - self._total_time_display = format_duration(total) - - self.phase = "Success" - self._refresh() - - -# ══════════════════════════════════════════════════════════════════════════ -# Plain-text reporter (used when 'rich' is NOT available) -# ══════════════════════════════════════════════════════════════════════════ -class PlainReporter(Reporter): - """ - stdout-only mirror of RichReporter. Same public interface, no rich - dependency at all: statuses are printed as they change, build progress - is shown as an in-place percentage line, and test output streams - straight through. - """ - - def __init__(self, build_type: str, generator: str, jobs: int) -> None: - self.build_type = build_type - self.generator = generator - self.jobs = jobs - - self.phase = "Starting..." - self.configure_status = "waiting" - self.build_status = "waiting" - self.test_status = "waiting" - - self.targets_built = 0 - self._last_build_pct = -1 - self._build_line_open = False - - def __enter__(self) -> "PlainReporter": - print( - f"Deepity Engine — build={self.build_type} " - f"generator={self.generator} jobs={self.jobs}" - ) - return self - - def __exit__(self, exc_type, exc, tb) -> None: - return None - - def _status_line(self) -> None: - print( - f"[{self.phase}] configure={self.configure_status} " - f"build={self.build_status} tests={self.test_status}" - ) - - def _print_failure(self, title: str, body: str) -> None: - body = body.rstrip() or "No output" - print() - print(f"---- {title} " + "-" * max(0, 60 - len(title))) - print(body) - print("-" * 70) - - def _close_build_line(self) -> None: - if self._build_line_open: - print() - self._build_line_open = False - - # -- Reporter interface ------------------------------------------------- - - def build_summary(self, targets: int) -> None: - print(f"Targets built: {targets}") - - def clean_reconfigure(self) -> None: - self.phase = "Configuring" - self.configure_status = "clean reconfigure" - self._status_line() - - def configure_started(self) -> None: - self.phase = "Configuring" - self.configure_status = "configuring..." - self._status_line() - - def configure_cached(self) -> None: - self.configure_status = "cached" - self._status_line() - - def configure_complete(self, duration: float) -> None: - self.configure_status = f"complete ({format_duration(duration)})" - self._status_line() - - def configure_failed(self, output: str) -> None: - self._print_failure("✗ CMake Configuration Failed", output) - - def build_started(self) -> None: - self.phase = "Building" - self.build_status = "compiling..." - self._status_line() - - def build_line(self, line: str) -> None: - match = BUILD_PROGRESS_RE.search(line) - - if not match: - return - - current = int(match.group(1)) - total = int(match.group(2)) - message = match.group(3).strip() - - pct = int((current / total) * 100) if total else 0 - - # Avoid flooding the terminal: only print when percentage moves. - if pct != self._last_build_pct: - self._last_build_pct = pct - print(f"\r [{current}/{total}] {pct:3d}% {message[:80]}", end="", flush=True) - self._build_line_open = True - - def build_complete(self, duration: float) -> None: - self._close_build_line() - self.build_status = f"complete ({format_duration(duration)})" - self._status_line() - - def build_failed(self, output: str) -> None: - self._close_build_line() - self.build_status = "failed" - self.phase = "Build failed" - self._status_line() - self._print_failure("✗ Build Failed", output) - - def tests_missing(self, exe_name: str, paths: list[Path]) -> None: - self.test_status = "binary missing" - self.phase = "Tests unavailable" - self._status_line() - self._print_failure(f"✗ Could Not Find {exe_name}", "\n".join(str(p) for p in paths)) - - def tests_started(self) -> None: - self.phase = "Testing" - self.test_status = "running..." - self._status_line() - - def test_line(self, line: str) -> None: - line = line.rstrip() - if line: - print(f" {line}") - - def tests_complete(self, duration: float) -> None: - self.test_status = f"complete ({format_duration(duration)})" - self._status_line() - - def tests_failed(self, output: str) -> None: - self.test_status = "failed" - self.phase = "Tests failed" - self._status_line() - self._print_failure("✗ Test Suite Failed", output) - - def success( - self, - configure_time: float | None, - build_time: float, - test_time: float, - ) -> None: - total = sum(value or 0 for value in (configure_time, build_time, test_time)) - - print() - print("=" * 60) - print("BUILD SUCCESSFUL") - print("=" * 60) - print(f"Build type : {self.build_type}") - print(f"Generator : {self.generator}") - print(f"Parallel jobs : {self.jobs}") - print(f"Targets build : {self.targets_built}") - print( - "Configuration : " - + (format_duration(configure_time) if configure_time else "cached") - ) - print(f"Compilation : {format_duration(build_time)}") - print(f"Tests : {format_duration(test_time)}") - print(f"Total : {format_duration(total)}") - - -def run_streaming_command( - cmd: list[str], - *, - on_line: Callable[[str], None], -) -> tuple[int, str]: - """ - Run a command while streaming its combined stdout/stderr to on_line. - - The complete output is also retained so that failures can show the - original diagnostic output in full. - """ - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - - assert process.stdout is not None - - output: list[str] = [] - - for line in process.stdout: - output.append(line) - on_line(line) - - return process.wait(), "".join(output) - - -def run_captured_command(cmd: list[str]) -> tuple[int, str]: - result = subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - - return result.returncode, result.stdout - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Deepity Cross-Platform Build & Test Runner" - ) - parser.add_argument( - "build_type", - nargs="?", - default="Release", - choices=["Release", "Debug"], - ) - parser.add_argument( - "--jobs", - "-j", - type=int, - default=multiprocessing.cpu_count(), - help="Number of parallel build jobs", - ) - parser.add_argument( - "--no-tests", - action="store_true", - help="Skip running tests", - ) - parser.add_argument( - "--clean", - action="store_true", - help="Remove the build directory before building", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Show full build output", - ) - args = parser.parse_args() - - build_type = args.build_type - jobs = args.jobs - - build_dir = Path("build") / build_type - deps_dir = build_dir / "_deps" - cache_file = build_dir / "CMakeCache.txt" - if args.clean and build_dir.exists(): - shutil.rmtree(build_dir) - - ninja = shutil.which("ninja") - generator = "Ninja" if ninja else "CMake" - - # ──────────────────────────────────────────────────────────────────── - # Setup Logging - # ──────────────────────────────────────────────────────────────────── - log_dir = Path("logs") - log_dir.mkdir(exist_ok=True) - log_file = log_dir / "build.log" - - # Start a fresh log file for this run - with log_file.open("w", encoding="utf-8") as f: - f.write(f"--- Deepity Build Log ({build_type}) ---\n\n") - - def log_output(phase: str, output: str) -> None: - with log_file.open("a", encoding="utf-8") as f: - f.write(f"=== {phase} ===\n") - f.write(output) - f.write("\n\n") - - git_branch, git_commit, git_dirty = get_git_info() - - reporter: Reporter = ( - RichReporter(build_type, generator, jobs, git_branch, git_commit, git_dirty) - if RICH_AVAILABLE - else PlainReporter(build_type, generator, jobs) - ) - - with reporter: - # ──────────────────────────────────────────────────────────────── - # Dependencies - # ──────────────────────────────────────────────────────────────── - if not deps_dir.is_dir(): - reporter.clean_reconfigure() - - if build_dir.exists(): - shutil.rmtree(build_dir) - - # ──────────────────────────────────────────────────────────────── - # Configure - # ──────────────────────────────────────────────────────────────── - configure_time: float | None = None - - if not cache_file.is_file(): - config_cmd = [ - "cmake", - "-B", - str(build_dir), - f"-DCMAKE_BUILD_TYPE={build_type}", - "-DDEEPITY_BUILD_TESTS=ON", - "-DDEEPITY_ENABLE_CUDA=ON", - ] - - if sys.platform == "win32": - config_cmd.extend(["-A", "x64"]) - vcpkg_root = os.environ.get("VCPKG_ROOT") - if vcpkg_root: - toolchain = Path(vcpkg_root) / "scripts/buildsystems/vcpkg.cmake" - if toolchain.is_file(): - config_cmd.append(f"-DCMAKE_TOOLCHAIN_FILE={toolchain.as_posix()}") - else: - print("Warning: VCPKG_ROOT environment variable is not set. Dependencies may fail to resolve.") - else: - if ninja: - config_cmd.extend(["-G", "Ninja"]) - - vcpkg_root = os.environ.get("VCPKG_ROOT") - if vcpkg_root: - config_cmd.append(f"-DCMAKE_TOOLCHAIN_FILE={vcpkg_root}/scripts/buildsystems/vcpkg.cmake") - - reporter.configure_started() - - start = time.perf_counter() - - return_code, config_output = run_captured_command(config_cmd) - log_output("CMake Configuration", config_output) - - configure_time = time.perf_counter() - start - - if return_code != 0: - reporter.configure_failed(config_output) - sys.exit(return_code) - - reporter.configure_complete(configure_time) - - else: - reporter.configure_cached() - - # ──────────────────────────────────────────────────────────────── - # Build - # ──────────────────────────────────────────────────────────────── - build_cmd = [ - "cmake", - "--build", - str(build_dir), - "-j", - str(jobs), - "--config", - build_type, - ] - - reporter.build_started() - - start = time.perf_counter() - - return_code, build_output = run_streaming_command( - build_cmd, - on_line=reporter.build_line, - ) - log_output("Compilation", build_output) - - build_time = time.perf_counter() - start - - if return_code != 0: - reporter.build_failed(build_output) - sys.exit(return_code) - - reporter.build_complete(build_time) - if args.no_tests: - reporter.success(configure_time, build_time, 0.) - return - - # ──────────────────────────────────────────────────────────────── - # Find test executable - # ──────────────────────────────────────────────────────────────── - exe_name = "DeepityTests.exe" if os.name == "nt" else "DeepityTests" - - test_paths = [ - build_dir / "bin" / exe_name, - build_dir / "bin" / build_type / exe_name, - build_dir / exe_name, - build_dir / build_type / exe_name, - ] - - test_exe = next((path for path in test_paths if path.is_file()), None) - - if test_exe is None: - reporter.tests_missing(exe_name, test_paths) - sys.exit(1) - - # ──────────────────────────────────────────────────────────────── - # Tests - # ──────────────────────────────────────────────────────────────── - reporter.tests_started() - - test_cmd = [str(test_exe)] - - start = time.perf_counter() - - return_code, test_output = run_streaming_command( - test_cmd, - on_line=reporter.test_line, - ) - log_output("Tests", test_output) - - test_time = time.perf_counter() - start - - if return_code != 0: - reporter.tests_failed(test_output) - sys.exit(return_code) - - reporter.tests_complete(test_time) - - # ──────────────────────────────────────────────────────────────── - # Success - # ──────────────────────────────────────────────────────────────── - reporter.success(configure_time, build_time, test_time) - +from deepity_build.cli import main if __name__ == "__main__": main() \ No newline at end of file diff --git a/deepity_build/__pycache__/cli.cpython-314.pyc b/deepity_build/__pycache__/cli.cpython-314.pyc new file mode 100644 index 0000000..c18f20a Binary files /dev/null and b/deepity_build/__pycache__/cli.cpython-314.pyc differ diff --git a/deepity_build/__pycache__/cmake_runner.cpython-314.pyc b/deepity_build/__pycache__/cmake_runner.cpython-314.pyc new file mode 100644 index 0000000..4ab2044 Binary files /dev/null and b/deepity_build/__pycache__/cmake_runner.cpython-314.pyc differ diff --git a/deepity_build/__pycache__/config.cpython-314.pyc b/deepity_build/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000..fc0c1ae Binary files /dev/null and b/deepity_build/__pycache__/config.cpython-314.pyc differ diff --git a/deepity_build/__pycache__/git_info.cpython-314.pyc b/deepity_build/__pycache__/git_info.cpython-314.pyc new file mode 100644 index 0000000..585d650 Binary files /dev/null and b/deepity_build/__pycache__/git_info.cpython-314.pyc differ diff --git a/deepity_build/__pycache__/process.cpython-314.pyc b/deepity_build/__pycache__/process.cpython-314.pyc new file mode 100644 index 0000000..ecf1216 Binary files /dev/null and b/deepity_build/__pycache__/process.cpython-314.pyc differ diff --git a/deepity_build/cli.py b/deepity_build/cli.py new file mode 100644 index 0000000..d8b9239 --- /dev/null +++ b/deepity_build/cli.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import argparse +import multiprocessing +import shutil +import sys +import time +from pathlib import Path + +from .cmake_runner import ( + build_command, + configure_command, + find_generator, + find_test_executable, + test_executable_candidates, + test_executable_name, +) +from .config import ARCH_PROFILES, DEFAULT_ARCH_PROFILE, BuildConfig +from .git_info import get_git_info +from .process import run_captured_command, run_streaming_command +from .reporting import make_reporter + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Deepity Cross-Platform Build & Test Runner" + ) + parser.add_argument( + "build_type", + nargs="?", + default="Release", + choices=["Release", "Debug"], + ) + parser.add_argument( + "--jobs", + "-j", + type=int, + default=multiprocessing.cpu_count(), + help="Number of parallel build jobs", + ) + + arch_group = parser.add_mutually_exclusive_group() + arch_group.add_argument( + "--native", + action="store_const", + dest="arch_profile", + const="native", + help=( + "Compile with -march=native. Fastest, but the binary is only safe " + "to run on THIS machine. Never use for anything you plan to hand " + "to someone else." + ), + ) + arch_group.add_argument( + "--fast", + action="store_const", + dest="arch_profile", + const="fast", + help=( + "Compile for an AVX2/FMA baseline (x86-64-v3). Portable across " + "most machines from the last ~10 years. Default for local builds." + ), + ) + arch_group.add_argument( + "--distributed", + action="store_const", + dest="arch_profile", + const="distributed", + help=( + "Compile for a maximally portable SSE4.2 baseline (x86-64-v2). " + "Use this for anything you're going to package and ship, e.g. " + "wheels built for PyPI." + ), + ) + parser.set_defaults(arch_profile=None) + + parser.add_argument( + "--cuda", + dest="cuda", + action="store_true", + default=None, + help="Force CUDA support ON (requires CUDAToolkit to be found).", + ) + parser.add_argument( + "--no-cuda", + dest="cuda", + action="store_false", + help="Force CUDA support OFF. Recommended together with --distributed.", + ) + + parser.add_argument( + "--no-tests", + action="store_true", + help="Skip building and running the DeepityTests target", + ) + parser.add_argument( + "--no-python-bindings", + action="store_true", + help="Skip building the pydeepity extension module", + ) + parser.add_argument( + "--clean", + action="store_true", + help="Remove the build directory before building", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Show full build output", + ) + parser.add_argument( + "--list-profiles", + action="store_true", + help="Print available --native/--fast/--distributed profiles and exit", + ) + + args = parser.parse_args(argv) + + if args.list_profiles: + for profile in ARCH_PROFILES.values(): + marker = " (default)" if profile.name == DEFAULT_ARCH_PROFILE else "" + print(f"{profile.name}{marker}") + print(f" {profile.description}") + print(f" unix: {profile.unix_flags}") + print(f" msvc: {profile.msvc_flags or '(compiler default)'}") + print() + sys.exit(0) + + if args.arch_profile is None: + args.arch_profile = DEFAULT_ARCH_PROFILE + + if args.cuda is None: + # Distributed builds default to CPU-only, since a CUDA-linked binary + # isn't portable either. Everything else keeps the old default (ON). + args.cuda = args.arch_profile != "distributed" + + return args + + +def build_config_from_args(args: argparse.Namespace) -> BuildConfig: + return BuildConfig( + build_type=args.build_type, + jobs=args.jobs, + arch_profile=args.arch_profile, + cuda=args.cuda, + build_tests=not args.no_tests, + run_tests=not args.no_tests, + python_bindings=not args.no_python_bindings, + clean=args.clean, + verbose=args.verbose, + ) + + +def main(argv: list[str] | None = None) -> None: + args = parse_args(argv) + config = build_config_from_args(args) + + if config.clean and config.build_dir.exists(): + shutil.rmtree(config.build_dir) + + ninja, generator = find_generator() + + # ──────────────────────────────────────────────────────────────────── + # Setup logging + # ──────────────────────────────────────────────────────────────────── + log_dir = Path("logs") + log_dir.mkdir(exist_ok=True) + log_file = log_dir / "build.log" + + with log_file.open("w", encoding="utf-8") as f: + f.write(f"--- Deepity Build Log ({config.build_type}, arch={config.arch_profile}) ---\n\n") + + def log_output(phase: str, output: str) -> None: + with log_file.open("a", encoding="utf-8") as f: + f.write(f"=== {phase} ===\n") + f.write(output) + f.write("\n\n") + + git = get_git_info() + reporter = make_reporter(config, git, generator) + + with reporter: + # ──────────────────────────────────────────────────────────────── + # Dependencies: force a clean reconfigure if _deps is missing + # ──────────────────────────────────────────────────────────────── + if not config.deps_dir.is_dir(): + reporter.clean_reconfigure() + if config.build_dir.exists(): + shutil.rmtree(config.build_dir) + + # ──────────────────────────────────────────────────────────────── + # Configure + # ──────────────────────────────────────────────────────────────── + configure_time: float | None = None + + if not config.cache_file.is_file(): + cmd = configure_command(config, ninja) + + reporter.configure_started() + start = time.perf_counter() + + return_code, config_output = run_captured_command(cmd) + log_output("CMake Configuration", config_output) + + configure_time = time.perf_counter() - start + + if return_code != 0: + reporter.configure_failed(config_output) + sys.exit(return_code) + + reporter.configure_complete(configure_time) + else: + reporter.configure_cached() + + # ──────────────────────────────────────────────────────────────── + # Build + # ──────────────────────────────────────────────────────────────── + reporter.build_started() + start = time.perf_counter() + + return_code, build_output = run_streaming_command( + build_command(config), + on_line=reporter.build_line, + ) + log_output("Compilation", build_output) + + build_time = time.perf_counter() - start + + if return_code != 0: + reporter.build_failed(build_output) + sys.exit(return_code) + + reporter.build_complete(build_time) + + if not config.run_tests: + reporter.success(configure_time, build_time, 0.0) + return + + # ──────────────────────────────────────────────────────────────── + # Find + run tests + # ──────────────────────────────────────────────────────────────── + test_exe = find_test_executable(config) + + if test_exe is None: + reporter.tests_missing(test_executable_name(), test_executable_candidates(config)) + sys.exit(1) + + reporter.tests_started() + start = time.perf_counter() + + return_code, test_output = run_streaming_command( + [str(test_exe)], + on_line=reporter.test_line, + ) + log_output("Tests", test_output) + + test_time = time.perf_counter() - start + + if return_code != 0: + reporter.tests_failed(test_output) + sys.exit(return_code) + + reporter.tests_complete(test_time) + reporter.success(configure_time, build_time, test_time) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/deepity_build/cmake_runner.py b/deepity_build/cmake_runner.py new file mode 100644 index 0000000..71c3f7b --- /dev/null +++ b/deepity_build/cmake_runner.py @@ -0,0 +1,82 @@ +"""Turns a BuildConfig into the actual `cmake` command lines.""" + +from __future__ import annotations + +import os +import shutil +import sys +from pathlib import Path + +from .config import BuildConfig + + +def find_generator() -> tuple[str | None, str]: + """Returns (ninja_path_or_None, display_name).""" + ninja = shutil.which("ninja") + return ninja, "Ninja" if ninja else "CMake" + + +def configure_command(config: BuildConfig, ninja: str | None) -> list[str]: + profile = config.profile + + cmd = [ + "cmake", + "-B", + str(config.build_dir), + f"-DCMAKE_BUILD_TYPE={config.build_type}", + f"-DDEEPITY_BUILD_TESTS={'ON' if config.build_tests else 'OFF'}", + f"-DDEEPITY_ENABLE_CUDA={'ON' if config.cuda else 'OFF'}", + f"-DDEEPITY_BUILD_PYTHON_BINDINGS={'ON' if config.python_bindings else 'OFF'}", + f"-DDEEPITY_ARCH_FLAGS={profile.unix_flags}", + ] + + if profile.msvc_flags: + cmd.append(f"-DDEEPITY_MSVC_ARCH_FLAGS={profile.msvc_flags}") + + if sys.platform == "win32": + cmd.extend(["-A", "x64"]) + vcpkg_root = os.environ.get("VCPKG_ROOT") + if vcpkg_root: + toolchain = Path(vcpkg_root) / "scripts/buildsystems/vcpkg.cmake" + if toolchain.is_file(): + cmd.append(f"-DCMAKE_TOOLCHAIN_FILE={toolchain.as_posix()}") + else: + if ninja: + cmd.extend(["-G", "Ninja"]) + + vcpkg_root = os.environ.get("VCPKG_ROOT") + if vcpkg_root: + cmd.append(f"-DCMAKE_TOOLCHAIN_FILE={vcpkg_root}/scripts/buildsystems/vcpkg.cmake") + + return cmd + + +def build_command(config: BuildConfig) -> list[str]: + return [ + "cmake", + "--build", + str(config.build_dir), + "-j", + str(config.jobs), + "--config", + config.build_type, + ] + + +def test_executable_name() -> str: + return "DeepityTests.exe" if os.name == "nt" else "DeepityTests" + + +def test_executable_candidates(config: BuildConfig) -> list[Path]: + exe_name = test_executable_name() + + return [ + config.build_dir / "bin" / exe_name, + config.build_dir / "bin" / config.build_type / exe_name, + config.build_dir / exe_name, + config.build_dir / config.build_type / exe_name, + ] + + +def find_test_executable(config: BuildConfig) -> Path | None: + return next((path for path in test_executable_candidates(config) if path.is_file()), None) \ No newline at end of file diff --git a/deepity_build/config.py b/deepity_build/config.py new file mode 100644 index 0000000..04b21ab --- /dev/null +++ b/deepity_build/config.py @@ -0,0 +1,92 @@ +""" +Build configuration: everything that determines *what* CMake is asked to +build. No process handling and no reporting logic lives here — this module +should be safe to import and unit-test on its own. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +# ══════════════════════════════════════════════════════════════════════════ +# Architecture profiles +# +# "native" is fast but only safe on the exact machine that built it — the +# binary can hit an illegal-instruction crash on any other CPU. "fast" and +# "distributed" are portable baselines you can hand to other people; the +# difference is how old a CPU they still cover. +# ══════════════════════════════════════════════════════════════════════════ +@dataclass(frozen=True) +class ArchProfile: + name: str + unix_flags: str + msvc_flags: str + description: str + + +ARCH_PROFILES: dict[str, ArchProfile] = { + "native": ArchProfile( + name="native", + unix_flags="-march=native", + msvc_flags="/arch:AVX2", # MSVC has no true "native"; this is its closest knob + description=( + "Tuned for THIS machine's CPU. Fastest option, but the resulting " + "binary can crash with an illegal instruction on any other machine. " + "Local development only — never distribute a native build." + ), + ), + "fast": ArchProfile( + name="fast", + unix_flags="-march=x86-64-v3 -mtune=generic", + msvc_flags="/arch:AVX2", + description=( + "AVX2 + FMA baseline (x86-64-v3). Covers the large majority of " + "desktops/servers from the last decade. Good default for local " + "builds when you don't need maximum portability." + ), + ), + "distributed": ArchProfile( + name="distributed", + unix_flags="-march=x86-64-v2 -mtune=generic", + msvc_flags="", # MSVC's default codegen is already an SSE2-era baseline + description=( + "SSE4.2 baseline (x86-64-v2), maximally portable. Use this for any " + "build you intend to ship to other people (wheels, releases, CI " + "artifacts)." + ), + ), +} + +DEFAULT_ARCH_PROFILE = "fast" + + +@dataclass(frozen=True) +class BuildConfig: + build_type: str + jobs: int + arch_profile: str + cuda: bool + build_tests: bool + run_tests: bool + python_bindings: bool + clean: bool + verbose: bool + build_root: Path = Path("build") + + @property + def profile(self) -> ArchProfile: + return ARCH_PROFILES[self.arch_profile] + + @property + def build_dir(self) -> Path: + return self.build_root / self.build_type + + @property + def deps_dir(self) -> Path: + return self.build_dir / "_deps" + + @property + def cache_file(self) -> Path: + return self.build_dir / "CMakeCache.txt" \ No newline at end of file diff --git a/deepity_build/git_info.py b/deepity_build/git_info.py new file mode 100644 index 0000000..b2410b1 --- /dev/null +++ b/deepity_build/git_info.py @@ -0,0 +1,40 @@ +"""Best-effort git metadata for the reporter header. Never raises.""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GitInfo: + branch: str + commit: str + dirty: bool + + +def get_git_info() -> GitInfo: + try: + branch = subprocess.check_output( + ["git", "branch", "--show-current"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + + commit = subprocess.check_output( + ["git", "rev-parse", "--short", "HEAD"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + + status = subprocess.call( + ["git", "diff", "--quiet"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + dirty = status != 0 + return GitInfo(branch or "detached", commit, dirty) + + except (subprocess.CalledProcessError, FileNotFoundError): + return GitInfo("unknown", "unknown", False) \ No newline at end of file diff --git a/deepity_build/process.py b/deepity_build/process.py new file mode 100644 index 0000000..7896ab9 --- /dev/null +++ b/deepity_build/process.py @@ -0,0 +1,70 @@ +"""Small process/formatting utilities shared by the CMake runner and reporters.""" + +from __future__ import annotations + +import re +import subprocess +from collections.abc import Callable + +BUILD_PROGRESS_RE = re.compile(r"\[(\d+)/(\d+)\]\s+(.*)") + + +def format_duration(seconds: float | None) -> str: + if seconds is None: + return "—" + + if seconds < 60: + return f"{seconds:.1f}s" + + minutes, seconds = divmod(seconds, 60) + + if minutes < 60: + return f"{int(minutes)}m {seconds:.0f}s" + + hours, minutes = divmod(int(minutes), 60) + return f"{hours}h {minutes}m" + + +def command_string(cmd: list[str]) -> str: + return " ".join(cmd) + + +def run_streaming_command( + cmd: list[str], + *, + on_line: Callable[[str], None], +) -> tuple[int, str]: + """ + Run a command while streaming its combined stdout/stderr to on_line. + + The complete output is also retained so that failures can show the + original diagnostic output in full. + """ + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + assert process.stdout is not None + + output: list[str] = [] + + for line in process.stdout: + output.append(line) + on_line(line) + + return process.wait(), "".join(output) + + +def run_captured_command(cmd: list[str]) -> tuple[int, str]: + result = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + return result.returncode, result.stdout \ No newline at end of file diff --git a/deepity_build/reporting/__init__.py b/deepity_build/reporting/__init__.py new file mode 100644 index 0000000..f1e8a19 --- /dev/null +++ b/deepity_build/reporting/__init__.py @@ -0,0 +1,53 @@ +""" +Reporting package. + +main() only ever talks to the Reporter interface, never to rich or +plain-text internals directly. Each concrete reporter imports rich symbols +locally, right where they're used, so importing this package never requires +rich to be installed, and static type checkers never see a "possibly +unbound" import. +""" + +from __future__ import annotations + +import importlib.util + +from ..config import BuildConfig +from ..git_info import GitInfo +from .base import Reporter + + +def is_library_installed(library_name: str) -> bool: + """Check if a python library is installed without importing it.""" + if not isinstance(library_name, str) or not library_name.strip(): + raise ValueError("Library name must be a non-empty string.") + return importlib.util.find_spec(library_name) is not None + + +def rich_available() -> bool: + try: + return is_library_installed("rich") + except ValueError: + return False + + +def make_reporter(config: BuildConfig, git: GitInfo, generator: str) -> Reporter: + """ + Construct the best available reporter and announce the choice. + + Kept here (rather than in cli.py) so the "which reporter, and why" story + lives next to the reporters themselves. + """ + if rich_available(): + from .rich_reporter import RichReporter + + print("✅ 'rich' is installed. Using the interactive dashboard.") + return RichReporter(config, git, generator) + + from .plain_reporter import PlainReporter + + print("❌ 'rich' is NOT installed. Falling back to plain-text output.") + return PlainReporter(config, git, generator) + + +__all__ = ["Reporter", "make_reporter", "is_library_installed", "rich_available"] \ No newline at end of file diff --git a/deepity_build/reporting/__pycache__/__init__.cpython-314.pyc b/deepity_build/reporting/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..a4d9bdc Binary files /dev/null and b/deepity_build/reporting/__pycache__/__init__.cpython-314.pyc differ diff --git a/deepity_build/reporting/__pycache__/base.cpython-314.pyc b/deepity_build/reporting/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..4b83c27 Binary files /dev/null and b/deepity_build/reporting/__pycache__/base.cpython-314.pyc differ diff --git a/deepity_build/reporting/__pycache__/rich_reporter.cpython-314.pyc b/deepity_build/reporting/__pycache__/rich_reporter.cpython-314.pyc new file mode 100644 index 0000000..2d9ba00 Binary files /dev/null and b/deepity_build/reporting/__pycache__/rich_reporter.cpython-314.pyc differ diff --git a/deepity_build/reporting/base.py b/deepity_build/reporting/base.py new file mode 100644 index 0000000..1f77aac --- /dev/null +++ b/deepity_build/reporting/base.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path + + +class Reporter(ABC): + @abstractmethod + def __enter__(self) -> "Reporter": ... + + @abstractmethod + def __exit__(self, exc_type, exc, tb) -> None: ... + + @abstractmethod + def build_summary(self, targets: int) -> None: ... + + @abstractmethod + def clean_reconfigure(self) -> None: ... + + @abstractmethod + def configure_started(self) -> None: ... + + @abstractmethod + def configure_cached(self) -> None: ... + + @abstractmethod + def configure_complete(self, duration: float) -> None: ... + + @abstractmethod + def configure_failed(self, output: str) -> None: ... + + @abstractmethod + def build_started(self) -> None: ... + + @abstractmethod + def build_line(self, line: str) -> None: ... + + @abstractmethod + def build_complete(self, duration: float) -> None: ... + + @abstractmethod + def build_failed(self, output: str) -> None: ... + + @abstractmethod + def tests_missing(self, exe_name: str, paths: list[Path]) -> None: ... + + @abstractmethod + def tests_started(self) -> None: ... + + @abstractmethod + def test_line(self, line: str) -> None: ... + + @abstractmethod + def tests_complete(self, duration: float) -> None: ... + + @abstractmethod + def tests_failed(self, output: str) -> None: ... + + @abstractmethod + def success( + self, + configure_time: float | None, + build_time: float, + test_time: float, + ) -> None: ... \ No newline at end of file diff --git a/deepity_build/reporting/plain_reporter.py b/deepity_build/reporting/plain_reporter.py new file mode 100644 index 0000000..626de83 --- /dev/null +++ b/deepity_build/reporting/plain_reporter.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from pathlib import Path + +from ..config import BuildConfig +from ..git_info import GitInfo +from ..process import BUILD_PROGRESS_RE, format_duration +from .base import Reporter + + +class PlainReporter(Reporter): + """ + stdout-only mirror of RichReporter. Same public interface, no rich + dependency at all: statuses are printed as they change, build progress + is shown as an in-place percentage line, and test output streams + straight through. + """ + + def __init__(self, config: BuildConfig, git: GitInfo, generator: str) -> None: + self.config = config + self.git = git + self.generator = generator + + self.phase = "Starting..." + self.configure_status = "waiting" + self.build_status = "waiting" + self.test_status = "waiting" + + self.targets_built = 0 + self._last_build_pct = -1 + self._build_line_open = False + + def __enter__(self) -> "PlainReporter": + profile = self.config.profile + print( + f"Deepity Engine — build={self.config.build_type} " + f"generator={self.generator} jobs={self.config.jobs} " + f"arch={profile.name} cuda={'on' if self.config.cuda else 'off'}" + ) + if profile.name == "native": + print(" ⚠ arch=native is not portable — do not distribute this binary.") + return self + + def __exit__(self, exc_type, exc, tb) -> None: + return None + + def _status_line(self) -> None: + print( + f"[{self.phase}] configure={self.configure_status} " + f"build={self.build_status} tests={self.test_status}" + ) + + def _print_failure(self, title: str, body: str) -> None: + body = body.rstrip() or "No output" + print() + print(f"---- {title} " + "-" * max(0, 60 - len(title))) + print(body) + print("-" * 70) + + def _close_build_line(self) -> None: + if self._build_line_open: + print() + self._build_line_open = False + + # -- Reporter interface ------------------------------------------------- + + def build_summary(self, targets: int) -> None: + print(f"Targets built: {targets}") + + def clean_reconfigure(self) -> None: + self.phase = "Configuring" + self.configure_status = "clean reconfigure" + self._status_line() + + def configure_started(self) -> None: + self.phase = "Configuring" + self.configure_status = "configuring..." + self._status_line() + + def configure_cached(self) -> None: + self.configure_status = "cached" + self._status_line() + + def configure_complete(self, duration: float) -> None: + self.configure_status = f"complete ({format_duration(duration)})" + self._status_line() + + def configure_failed(self, output: str) -> None: + self._print_failure("✗ CMake Configuration Failed", output) + + def build_started(self) -> None: + self.phase = "Building" + self.build_status = "compiling..." + self._status_line() + + def build_line(self, line: str) -> None: + match = BUILD_PROGRESS_RE.search(line) + + if not match: + return + + current = int(match.group(1)) + total = int(match.group(2)) + message = match.group(3).strip() + + pct = int((current / total) * 100) if total else 0 + + # Avoid flooding the terminal: only print when percentage moves. + if pct != self._last_build_pct: + self._last_build_pct = pct + print(f"\r [{current}/{total}] {pct:3d}% {message[:80]}", end="", flush=True) + self._build_line_open = True + + def build_complete(self, duration: float) -> None: + self._close_build_line() + self.build_status = f"complete ({format_duration(duration)})" + self._status_line() + + def build_failed(self, output: str) -> None: + self._close_build_line() + self.build_status = "failed" + self.phase = "Build failed" + self._status_line() + self._print_failure("✗ Build Failed", output) + + def tests_missing(self, exe_name: str, paths: list[Path]) -> None: + self.test_status = "binary missing" + self.phase = "Tests unavailable" + self._status_line() + self._print_failure(f"✗ Could Not Find {exe_name}", "\n".join(str(p) for p in paths)) + + def tests_started(self) -> None: + self.phase = "Testing" + self.test_status = "running..." + self._status_line() + + def test_line(self, line: str) -> None: + line = line.rstrip() + if line: + print(f" {line}") + + def tests_complete(self, duration: float) -> None: + self.test_status = f"complete ({format_duration(duration)})" + self._status_line() + + def tests_failed(self, output: str) -> None: + self.test_status = "failed" + self.phase = "Tests failed" + self._status_line() + self._print_failure("✗ Test Suite Failed", output) + + def success( + self, + configure_time: float | None, + build_time: float, + test_time: float, + ) -> None: + total = sum(value or 0 for value in (configure_time, build_time, test_time)) + + print() + print("=" * 60) + print("BUILD SUCCESSFUL") + print("=" * 60) + print(f"Build type : {self.config.build_type}") + print(f"Generator : {self.generator}") + print(f"Arch profile : {self.config.profile.name}") + print(f"CUDA : {'ON' if self.config.cuda else 'off'}") + print(f"Parallel jobs : {self.config.jobs}") + print(f"Targets build : {self.targets_built}") + print( + "Configuration : " + + (format_duration(configure_time) if configure_time else "cached") + ) + print(f"Compilation : {format_duration(build_time)}") + print(f"Tests : {format_duration(test_time)}") + print(f"Total : {format_duration(total)}") \ No newline at end of file diff --git a/deepity_build/reporting/rich_reporter.py b/deepity_build/reporting/rich_reporter.py new file mode 100644 index 0000000..9d851f0 --- /dev/null +++ b/deepity_build/reporting/rich_reporter.py @@ -0,0 +1,331 @@ +from __future__ import annotations + +from collections import deque +from pathlib import Path + +from ..config import BuildConfig +from ..git_info import GitInfo +from ..process import BUILD_PROGRESS_RE, format_duration +from .base import Reporter + + +class RichReporter(Reporter): + """Single mutable Rich Live dashboard for the entire build.""" + + def __init__(self, config: BuildConfig, git: GitInfo, generator: str) -> None: + from rich.console import Console + from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, + ) + + self.config = config + self.git = git + self.generator = generator + + self.targets_built = 0 + self.num_passed = 0 + self.num_failed = 0 + + self.phase = "Starting..." + self.configure_status = "[dim]waiting[/dim]" + self.build_status = "[dim]waiting[/dim]" + self.test_status = "[dim]waiting[/dim]" + + self.build_message = "" + self.test_lines: deque[str] = deque(maxlen=8) + + self._configure_time_display = "—" + self._build_time_display = "—" + self._test_time_display = "—" + self._total_time_display = "—" + + self.console = Console() + self.progress = Progress( + SpinnerColumn(), + TextColumn("[bold blue]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeElapsedColumn(), + ) + self.build_task = self.progress.add_task("Waiting", total=1, completed=0) + + self._live = None + + # -- context manager: owns the Live session -------------------------- + def __enter__(self) -> "RichReporter": + from rich.live import Live + + self.console.clear() + self.console.print() + + self._live = Live( + self._render(), + refresh_per_second=12, + console=self.console, + transient=False, + ) + self._live.__enter__() + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self._live is not None: + self._live.__exit__(exc_type, exc, tb) + + # -- rendering --------------------------------------------------------- + def _header(self): + from rich.panel import Panel + from rich.table import Table + + profile = self.config.profile + arch_style = "green" if profile.name == "distributed" else ( + "yellow" if profile.name == "fast" else "red" + ) + + table = Table.grid(padding=(0, 2)) + table.add_column() + table.add_column() + table.add_column() + table.add_column() + + table.add_row( + "[bold cyan]Deepity Engine[/bold cyan]", + f"[dim]Build[/dim] [bold]{self.config.build_type}[/bold]", + f"[dim]Generator[/dim] [bold]{self.generator}[/bold]", + f"[dim]Jobs[/dim] [bold]{self.config.jobs}[/bold]", + ) + + table.add_row( + "[dim]Arch profile[/dim]", + f"[bold {arch_style}]{profile.name}[/bold {arch_style}]" + + ( + " [dim](not portable — local dev only)[/dim]" + if profile.name == "native" + else "" + ), + "[dim]CUDA[/dim]", + "[bold green]ON[/bold green]" if self.config.cuda else "[dim]off[/dim]", + ) + + table.add_row( + "[dim]Git[/dim]", + f"[bold]{self.git.branch}[/bold] @ {self.git.commit}" + + (" [yellow]● modified[/yellow]" if self.git.dirty else " [green]✓ clean[/green]"), + ) + + return Panel(table, border_style="cyan", padding=(0, 1)) + + def _status(self): + from rich.panel import Panel + from rich.table import Table + + table = Table.grid(padding=(0, 2)) + table.add_column(style="bold", width=12) + table.add_column() + + table.add_row("Configure", self.configure_status) + table.add_row("Build", self.build_status) + table.add_row("Tests", self.test_status) + + return Panel(table, title="[bold]Build Status[/bold]", border_style="blue") + + def _activity(self): + from rich.console import Group + from rich.panel import Panel + from rich.table import Table + from rich.text import Text + + if self.phase == "Configuring": + body = Group( + Text("Configuring CMake...", style="bold yellow"), + Text(""), + Text.from_markup( + "[dim]CMake output is hidden while the build is running " + "and shown in full if configuration fails.[/dim]" + ), + ) + return Panel( + body, + title="[bold yellow]⚙ Configuration[/bold yellow]", + border_style="yellow", + ) + + if self.phase == "Building": + current = self.build_message or "Working..." + return Panel( + Group(self.progress, Text.from_markup(f"[dim]{current}[/dim]")), + title="[bold blue]⚙ Compilation[/bold blue]", + border_style="blue", + ) + + if self.phase == "Testing": + body = ( + "\n".join(self.test_lines) + if self.test_lines + else "[dim]Waiting for test output...[/dim]" + ) + return Panel( + body, + title="[bold magenta]▶ Tests[/bold magenta]", + border_style="magenta", + ) + + if self.phase == "Success": + body = Table.grid(padding=(0, 2)) + body.add_column(style="dim") + body.add_column(style="bold") + + body.add_row("Build type", self.config.build_type) + body.add_row("Arch profile", self.config.profile.name) + body.add_row("CUDA", "ON" if self.config.cuda else "off") + body.add_row("Parallel jobs", str(self.config.jobs)) + body.add_row("Targets", str(self.targets_built)) + body.add_row("Configuration", self._configure_time_display) + body.add_row("Compilation", self._build_time_display) + body.add_row("Tests", self._test_time_display) + body.add_row("Total", self._total_time_display) + + return Panel( + Group( + Text("✓ BUILD SUCCESSFUL", style="bold green", justify="center"), + Text(""), + body, + ), + title="[bold green]Deepity[/bold green]", + border_style="green", + padding=(1, 2), + ) + + return Panel("[dim]Waiting...[/dim]", title="[bold]Activity[/bold]", border_style="dim") + + def _render(self): + from rich.console import Group + + return Group(self._header(), self._status(), self._activity()) + + def _refresh(self) -> None: + if self._live is not None: + self._live.update(self._render(), refresh=True) + + def _print_failure(self, title: str, body: str) -> None: + from rich.panel import Panel + + body = body.rstrip() or "No output" + self.console.print() + self.console.print(Panel(body, title=f"[bold red]{title}[/bold red]", border_style="red")) + + # -- Reporter interface ------------------------------------------------- + + def build_summary(self, targets: int) -> None: + self.targets_built = targets + self._refresh() + + def clean_reconfigure(self) -> None: + self.phase = "Configuring" + self.configure_status = "[yellow]● clean reconfigure[/yellow]" + self._refresh() + + def configure_started(self) -> None: + self.phase = "Configuring" + self.configure_status = "[yellow]● configuring[/yellow]" + self._refresh() + + def configure_cached(self) -> None: + self.configure_status = "[green]✓ cached[/green]" + self._refresh() + + def configure_complete(self, duration: float) -> None: + self.configure_status = ( + f"[bold green]✓ complete[/bold green] [dim]({format_duration(duration)})[/dim]" + ) + self._refresh() + + def configure_failed(self, output: str) -> None: + self._print_failure("✗ CMake Configuration Failed", output) + + def build_started(self) -> None: + self.phase = "Building" + self.build_status = "[yellow]● compiling[/yellow]" + self.progress.update(self.build_task, total=1, completed=0, description="Compiling") + self._refresh() + + def build_line(self, line: str) -> None: + match = BUILD_PROGRESS_RE.search(line) + + if not match: + stripped = line.strip() + if stripped: + self.build_message = stripped[-160:] + self._refresh() + return + + current = int(match.group(1)) + total = int(match.group(2)) + message = match.group(3).strip() + self.targets_built = max(self.targets_built, total) + self.build_message = message + self.progress.update(self.build_task, total=total, completed=current) + + def build_complete(self, duration: float) -> None: + self.build_status = ( + f"[bold green]✓ complete[/bold green] [dim]({format_duration(duration)})[/dim]" + ) + self._refresh() + + def build_failed(self, output: str) -> None: + self.build_status = "[bold red]✗ failed[/bold red]" + self.phase = "Build failed" + self._refresh() + self._print_failure("✗ Build Failed", output) + + def tests_missing(self, exe_name: str, paths: list[Path]) -> None: + self.test_status = "[bold red]✗ binary missing[/bold red]" + self.phase = "Tests unavailable" + self._refresh() + self._print_failure(f"✗ Could Not Find {exe_name}", "\n".join(str(p) for p in paths)) + + def tests_started(self) -> None: + self.phase = "Testing" + self.test_status = "[yellow]● running[/yellow]" + self.test_lines.clear() + self._refresh() + + def test_line(self, line: str) -> None: + line = line.rstrip() + + if line: + self.test_lines.append(line) + self._refresh() + + def tests_complete(self, duration: float) -> None: + self.test_status = ( + f"[bold green]✓ complete[/bold green] [dim]({format_duration(duration)})[/dim]" + ) + self._refresh() + + def tests_failed(self, output: str) -> None: + self.test_status = "[bold red]✗ failed[/bold red]" + self.phase = "Tests failed" + self._refresh() + self._print_failure("✗ Test Suite Failed", output) + + def success( + self, + configure_time: float | None, + build_time: float, + test_time: float, + ) -> None: + self._configure_time_display = ( + format_duration(configure_time) if configure_time else "cached" + ) + self._build_time_display = format_duration(build_time) + self._test_time_display = format_duration(test_time) + + total = sum(value or 0 for value in (configure_time, build_time, test_time)) + self._total_time_display = format_duration(total) + + self.phase = "Success" + self._refresh() \ No newline at end of file diff --git a/examples/train_mnist_deep.py b/examples/train_mnist_deep.py index e031307..4e4be64 100644 --- a/examples/train_mnist_deep.py +++ b/examples/train_mnist_deep.py @@ -4,95 +4,194 @@ from pydeepity import SimplePCN from time import perf_counter -from rich.console import Console -from rich.progress import track +# FINAL VALIDATION -- the complete stack found today: +# - Forward-projection initialization (seeds hidden layers from a real +# forward pass through current weights, not zero-init) +# - +-0.3 uniform weight init (matching ngc-learn's actual convention) +# - AdamW, lr=0.001 (matching ngc-learn's hard-coded Adam + eta=0.001 -- +# the init range and optimizer needed to be changed TOGETHER, not +# independently; tested and confirmed: 73.31% alone with SGD vs +# 93.96% paired with AdamW) +# - mu_cache_threshold=0.05 -- confirmed +30.6% faster than ngc-learn's +# real, measured per-batch time in an isolated speed test. THIS RUN +# checks whether that speed holds without costing the 93.96% accuracy +# already confirmed at threshold=disabled. +# - train_step_with_projection() -- single C++ call per batch, not a +# manual Python loop (confirmed real, if smaller, speedup from +# eliminating ~40 Python/pybind boundary crossings per batch) +# +# [0,1] normalization, labels clipped to [0.001, 0.999], tanh activation, +# 784->512->512->10, T=20 -- all matching ngc-learn's actual, real source +# as closely as possible. -console = Console() def load_full_mnist(): - console.print("[bold cyan]Fetching full MNIST dataset...[/bold cyan]") + print("Fetching full MNIST dataset (70,000 images)...") X, y = fetch_openml('mnist_784', version=1, return_X_y=True, as_frame=False, parser='auto') - X = (X.astype(np.float32) / 127.5) - 1.0 - y = y.astype(int) + X = X.astype(np.float32) / 255.0 # [0,1], matching ngc-learn + y = y.astype(int) # type: ignore - Y_bipolar = np.full((y.shape[0], 10), -0.9, dtype=np.float32) - Y_bipolar[np.arange(y.shape[0]), y] = 0.9 + # Clipped one-hot, matching ngc-learn's `jnp.clip(lab, eps, 1-eps)` + eps = 0.001 + Y = np.full((y.shape[0], 10), eps, dtype=np.float32) + Y[np.arange(y.shape[0]), y] = 1.0 - eps X_train, X_test, Y_train, Y_test, y_train_labels, y_test_labels = train_test_split( - X, Y_bipolar, y, train_size=60000, test_size=10000, stratify=y, random_state=42 + X, Y, y, train_size=60000, test_size=10000, stratify=y, random_state=42 ) - return X_train, Y_train, X_test, y_test_labels, y_train_labels - - -def train_and_evaluate(X_train, Y_train, y_train_labels, X_test, y_test_labels, - train_steps, test_steps, lr, epochs, batch_size): - - net = SimplePCN(batch_size=batch_size) - - # 3 Hidden Layers (784 -> 512 -> 256 -> 128 -> 10) with cooled-down ir=0.02 - net.add_layer(784, 512, lr=lr, ir=0.02, act="tanh", lmbda=0.0001) - net.add_layer(512, 256, lr=lr, ir=0.02, act="tanh", lmbda=0.0001) - net.add_layer(256, 128, lr=lr, ir=0.02, act="tanh", lmbda=0.0001) - net.add_layer(128, 10, lr=lr, ir=0.02, act="tanh", lmbda=0.0001) - net.add_layer( 10, 0, lr=lr, ir=0.02, act="linear", lmbda=0.0001) - - O_: str = "ADAMW" - net.set_optimizer(O_) - console.print(f"[bold blue]Set optimizer to {O_}[/bold blue]") - + return X_train, Y_train, X_test, y_test_labels + + +def main() -> None: + X_train, Y_train, X_test, y_test_labels = load_full_mnist() + + BATCH_SIZE = 256 + STEPS = 20 # matches ngc-learn's T=20 + EPOCHS = 15 + LR = 0.001 # Adam-appropriate scale, NOT the SGD-tuned 0.06 -- + # matches ngc-learn's own stated eta=0.001 directly, + # and today's earlier finding that Adam needs a + # much smaller rate than SGD on this codebase + DECAY_RATE = 0.98 + + print(f"\nBuilding network (784->512->512->10)...") + net = SimplePCN(batch_size=BATCH_SIZE) + net.add_layer(784, 512, lr=LR, ir=0.08, act="tanh", lmbda=0.0001) + net.add_layer(512, 512, lr=LR, ir=0.08, act="tanh", lmbda=0.0001) + net.add_layer(512, 10, lr=LR, ir=0.08, act="tanh", lmbda=0.0001) + # Explicit TERMINAL layer -- outChannels/nextSize=0. Without this, + # net[-1] is the 512->10 layer itself, whose OWN beliefs are its + # 512-dim INPUT, not the 10-dim prediction it produces -- exactly + # the bug that just crashed this script (and was already silently + # corrupting every batch before that: clamp_state() truncates rather + # than erroring on a size mismatch). + net.add_layer(10, 0, lr=LR, ir=0.08, act="linear", lmbda=0.0001) + net.set_optimizer("ADAMW") # THE missing paired piece -- ngc-learn hard-codes + # Adam for every synapse. The larger +-0.3 init + # tested ALONE (with SGD) made things WORSE + # (73.31% vs 86.41%) -- likely because SGD's + # step size scales with raw gradient magnitude, + # unlike Adam's per-parameter normalization, + # which can handle a larger weight scale + # gracefully. Init range and optimizer were + # tuned TOGETHER in the original; testing one + # without the other may have been the mistake. net.compile() net.randomize_weights() - - net.summary() - - # Drop the base learning rate significantly for deep ADAMW - adjusted_lr = lr / 1000.0 - + net.set_mu_cache_threshold(0.0) # REVERTED from 0.05 -- that showed genuine + # instability under real, sustained AdamW + # training (energy trending UP, a 62-point + # accuracy crash at epoch 8), not just noise. + # Likely cause: small caching-approximation + # bias accumulating in Adam's momentum/ + # variance buffers over many updates -- + # something the earlier single-trajectory, + # no-weight-update speed test structurally + # could not have caught. threshold=0 is + # exact (proven bit-identical to no caching) + # and confirmed safe with this exact stack. + + # Override weight init to match ngc-learn's actual convention: fixed + # uniform range +-0.3, INDEPENDENT of layer size -- not our own + # size-scaled Gaussian (std ~0.039 for the 784->512 layer, ~8x + # smaller). This directly affects how informative the very first + # forward-projection pass is -- larger initial weights produce more + # differentiated activations from a single forward pass. Biases + # already default to 0 in Deepity (never touched by + # RandomizeWeights()), matching ngc-learn's own bias_init=constant(0). + rng_init = np.random.default_rng(7) + for layer in net.layers[:-1]: # skip the terminal -- it has no weights (nextSize=0) + w_shape = layer.weights.shape + layer.weights[:] = rng_init.uniform(-0.3, 0.3, w_shape).astype(np.float32) + + print(f"\nTraining with FORWARD-PROJECTION init: {EPOCHS} epochs, {STEPS} steps, " + f"lr={LR}, decay_rate={DECAY_RATE}...\n") + print("Reference (ngc-learn, real run): 26.91, 42.96, 60.12, 75.20, 84.68, 89.52,") + print(" 91.90, 93.45, 94.30, 94.80, 95.13, 95.38, 95.63, 95.74, 95.95 -- test 95.09%\n") + + rng = np.random.default_rng(42) + n_batches = len(X_train) // BATCH_SIZE start_time = perf_counter() - - # Train with I_avg caching and decay_rate=0.95 to prevent late-stage explosions - net.fit_iavg( - X=X_train, - Y=Y_train, - labels=y_train_labels, - epochs=epochs, - steps=train_steps, - per_class=batch_size // 10, - num_classes=10, - hidden_layer_index=1, - hidden_size=512, - initial_lr=adjusted_lr, - decay_rate=0.95, - reset_cache_per_epoch=True - ) + epoch_accs = [] + + for epoch in range(EPOCHS): + current_lr = LR * (DECAY_RATE ** epoch) + net.set_learning_rate(current_lr) + + indices = rng.permutation(len(X_train)) + X_shuf, Y_shuf = X_train[indices], Y_train[indices] + + correct = 0 + total = 0 + epoch_energy = 0.0 + + for b in range(n_batches): + X_batch = X_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + Y_batch = Y_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + + energy = net.train_step_with_projection(X_batch, Y_batch, STEPS) + epoch_energy += energy + + # REAL accuracy check -- genuine UNCLAMPED settle on a cheap + # subset, not reading beliefs right after unclamp_state() (which + # only flips a flag, never resets z -- that was reading back the + # clamped TARGET itself, trivially "matching" 100% every time). + N_ACC_BATCHES = 10 + for b in range(min(N_ACC_BATCHES, n_batches)): + X_batch = X_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + Y_batch = Y_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + + net.reset_state() + net.clamp_input(X_batch) + net.project_forward() + for _ in range(STEPS): # same step count as training, genuinely unclamped + net.calculate_state() + net.update_state() + + terminal_beliefs = np.array(net[-1].beliefs).reshape(BATCH_SIZE, 10) + pred = np.argmax(terminal_beliefs, axis=1) + true = np.argmax(Y_batch, axis=1) + correct += np.sum(pred == true) + total += BATCH_SIZE + + epoch_acc = 100.0 * correct / total + epoch_accs.append(epoch_acc) + avg_energy = epoch_energy / n_batches + elapsed = perf_counter() - start_time + print(f"Epoch {epoch+1}/{EPOCHS} | Time: {elapsed:.1f}s | Acc: {epoch_acc:.2f}% | Avg energy: {avg_energy:.4f}") train_time = perf_counter() - start_time - console.print(f"[bold green]Training complete in {train_time:.1f}s.[/bold green]\n") - - console.print(f"[bold yellow]Evaluating on held-out test set with {test_steps} inference steps...[/bold yellow]") - correct, total = 0, 0 - - for i in track(range(0, len(X_test), batch_size), description="[cyan]Evaluating..."): - X_batch = X_test[i:i + batch_size] - y_labels_batch = y_test_labels[i:i + batch_size] - if len(X_batch) != batch_size: continue - - # Use the longer zero-init test steps to let the network settle fully - pred_classes = np.argmax(net.predict(X_batch, steps=test_steps), axis=1) + print(f"\nTraining complete in {train_time:.1f}s.") + + print("\nRunning final test evaluation (with forward-projection init)...") + correct = 0 + total = 0 + for i in range(0, len(X_test), BATCH_SIZE): + X_batch = X_test[i:i + BATCH_SIZE] + y_labels_batch = y_test_labels[i:i + BATCH_SIZE] + if len(X_batch) != BATCH_SIZE: + continue + + net.reset_state() + net.clamp_input(X_batch) + net.project_forward() + for _ in range(300): # generous settle for final eval readout + net.calculate_state() + net.update_state() + + terminal_beliefs = np.array(net[-1].beliefs).reshape(BATCH_SIZE, 10) + pred_classes = np.argmax(terminal_beliefs, axis=1) correct += np.sum(pred_classes == y_labels_batch) - total += batch_size + total += BATCH_SIZE + + test_acc = 100.0 * correct / total + print(f"\n=== Result ===") + print(f"Deepity + forward-projection test accuracy: {test_acc:.2f}% (ngc-learn: 95.09%)") + print(f"Train time: {train_time:.1f}s") + print(f"\nDeepity per-epoch: {[round(a,2) for a in epoch_accs]}") + print(f"ngc-learn per-epoch: [26.91, 42.96, 60.12, 75.20, 84.68, 89.52, 91.90,") + print(f" 93.45, 94.30, 94.80, 95.13, 95.38, 95.63, 95.74, 95.95]") - console.print(f"\n[bold magenta]Test Accuracy: {correct}/{total} ({(correct/total)*100:.2f}%)[/bold magenta]") if __name__ == "__main__": - X_train, Y_train, X_test, y_test_labels, y_train_labels = load_full_mnist() - - # Decouple the steps: 6 for lightning-fast training, 60 for accurate prediction - train_and_evaluate( - X_train, Y_train, y_train_labels, X_test, y_test_labels, - train_steps=60, - test_steps=60, - lr=0.06, - epochs=50, - batch_size=250 - ) + main() \ No newline at end of file diff --git a/include/deepity/Activations.h b/include/deepity/Activations.h index 0e92c0f..40833f4 100644 --- a/include/deepity/Activations.h +++ b/include/deepity/Activations.h @@ -26,8 +26,9 @@ * Deep::tanh(array, arraysize) * * @note Separate implementations exist for AVX512F, AVX2, SSE, and naive. - * @version 1.0 - * @date 2026-06-21 + * Memory does NOT need to be aligned. + * @version 1.1 + * @date 2026-08-23 * @author Jack Rose */ @@ -137,40 +138,33 @@ namespace Deep #if defined(__AVX512F__) simd_end = n - (n % 16); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 16) { - __m512 x_512 = _mm512_load_ps(x + i); - + __m512 x_512 = _mm512_loadu_ps(x + i); __m512 res = Sleef_expf16_u10avx512f(x_512); - - _mm512_store_ps(x + i, res); + _mm512_storeu_ps(x + i, res); } #elif defined(__AVX2__) simd_end = n - (n % 8); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 8) { - __m256 x_256 = _mm256_load_ps(x + i); - + __m256 x_256 = _mm256_loadu_ps(x + i); __m256 res = Sleef_expf8_u10avx2(x_256); - - _mm256_store_ps(x + i, res); + _mm256_storeu_ps(x + i, res); } #elif defined(__SSE2__) simd_end = n - (n % 4); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 4) { - __m128 x_128 = _mm_load_ps(x + i); - + __m128 x_128 = _mm_loadu_ps(x + i); __m128 res = Sleef_expf4_u10sse2(x_128); - - _mm_store_ps(x + i, res); + _mm_storeu_ps(x + i, res); } - #endif for (size_t i = simd_end; i < n; ++i) @@ -185,37 +179,32 @@ namespace Deep #if defined(__AVX512F__) simd_end = n - (n % 16); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 16) { - __m512 x_512 = _mm512_load_ps(x + i); - + __m512 x_512 = _mm512_loadu_ps(x + i); __m512 res = Sleef_logf16_u10avx512f(x_512); - - _mm512_store_ps(x + i, res); + _mm512_storeu_ps(x + i, res); } #elif defined(__AVX2__) simd_end = n - (n % 8); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 8) { - __m256 x_256 = _mm256_load_ps(x + i); - + __m256 x_256 = _mm256_loadu_ps(x + i); __m256 res = Sleef_logf8_u10avx2(x_256); - - _mm256_store_ps(x + i, res); + _mm256_storeu_ps(x + i, res); } #elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) simd_end = n - (n % 4); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 4) { - __m128 x_128 = _mm_load_ps(x + i); + __m128 x_128 = _mm_loadu_ps(x + i); __m128 res = Sleef_logf4_u10sse2(x_128); - - _mm_store_ps(x + i, res); + _mm_storeu_ps(x + i, res); } #endif for (size_t i = simd_end; i < n; ++i) @@ -226,7 +215,7 @@ namespace Deep #pragma region relu /// @brief RELU(x) = MAX(0, x) for all x - /// @param x array, \em assumed to be properly aligned + /// @param x array, \em does not need to be aligned /// @param n x length static inline void relu(float *RESTRICT x, const size_t n) noexcept { @@ -239,87 +228,87 @@ namespace Deep __m512 zeros = _mm512_setzero_ps(); size_t simd_end4 = n - (n % 64); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end4); i += 64) { - __m512 x0 = _mm512_load_ps(x + i); - __m512 x1 = _mm512_load_ps(x + i + 16); - __m512 x2 = _mm512_load_ps(x + i + 32); - __m512 x3 = _mm512_load_ps(x + i + 48); + __m512 x0 = _mm512_loadu_ps(x + i); + __m512 x1 = _mm512_loadu_ps(x + i + 16); + __m512 x2 = _mm512_loadu_ps(x + i + 32); + __m512 x3 = _mm512_loadu_ps(x + i + 48); x0 = _mm512_max_ps(zeros, x0); x1 = _mm512_max_ps(zeros, x1); x2 = _mm512_max_ps(zeros, x2); x3 = _mm512_max_ps(zeros, x3); - _mm512_store_ps(x + i, x0); - _mm512_store_ps(x + i + 16, x1); - _mm512_store_ps(x + i + 32, x2); - _mm512_store_ps(x + i + 48, x3); + _mm512_storeu_ps(x + i, x0); + _mm512_storeu_ps(x + i + 16, x1); + _mm512_storeu_ps(x + i + 32, x2); + _mm512_storeu_ps(x + i + 48, x3); } simd_end = n - (n % 16); for (size_t i = simd_end4; i < simd_end; i += 16) { - _mm512_store_ps(x + i, _mm512_max_ps(zeros, _mm512_load_ps(x + i))); + _mm512_storeu_ps(x + i, _mm512_max_ps(zeros, _mm512_loadu_ps(x + i))); } #elif defined(__AVX2__) || defined(__AVX__) __m256 zeros = _mm256_setzero_ps(); size_t simd_end4 = n - (n % 32); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end4); i += 32) { - __m256 x0 = _mm256_load_ps(x + i); - __m256 x1 = _mm256_load_ps(x + i + 8); - __m256 x2 = _mm256_load_ps(x + i + 16); - __m256 x3 = _mm256_load_ps(x + i + 24); + __m256 x0 = _mm256_loadu_ps(x + i); + __m256 x1 = _mm256_loadu_ps(x + i + 8); + __m256 x2 = _mm256_loadu_ps(x + i + 16); + __m256 x3 = _mm256_loadu_ps(x + i + 24); x0 = _mm256_max_ps(zeros, x0); x1 = _mm256_max_ps(zeros, x1); x2 = _mm256_max_ps(zeros, x2); x3 = _mm256_max_ps(zeros, x3); - _mm256_store_ps(x + i, x0); - _mm256_store_ps(x + i + 8, x1); - _mm256_store_ps(x + i + 16, x2); - _mm256_store_ps(x + i + 24, x3); + _mm256_storeu_ps(x + i, x0); + _mm256_storeu_ps(x + i + 8, x1); + _mm256_storeu_ps(x + i + 16, x2); + _mm256_storeu_ps(x + i + 24, x3); } simd_end = n - (n % 8); for (size_t i = simd_end4; i < simd_end; i += 8) { - _mm256_store_ps(x + i, _mm256_max_ps(zeros, _mm256_load_ps(x + i))); + _mm256_storeu_ps(x + i, _mm256_max_ps(zeros, _mm256_loadu_ps(x + i))); } #elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) __m128 zeros = _mm_setzero_ps(); size_t simd_end4 = n - (n % 16); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end4); i += 16) { - __m128 x0 = _mm_load_ps(x + i); - __m128 x1 = _mm_load_ps(x + i + 4); - __m128 x2 = _mm_load_ps(x + i + 8); - __m128 x3 = _mm_load_ps(x + i + 12); + __m128 x0 = _mm_loadu_ps(x + i); + __m128 x1 = _mm_loadu_ps(x + i + 4); + __m128 x2 = _mm_loadu_ps(x + i + 8); + __m128 x3 = _mm_loadu_ps(x + i + 12); x0 = _mm_max_ps(zeros, x0); x1 = _mm_max_ps(zeros, x1); x2 = _mm_max_ps(zeros, x2); x3 = _mm_max_ps(zeros, x3); - _mm_store_ps(x + i, x0); - _mm_store_ps(x + i + 4, x1); - _mm_store_ps(x + i + 8, x2); - _mm_store_ps(x + i + 12, x3); + _mm_storeu_ps(x + i, x0); + _mm_storeu_ps(x + i + 4, x1); + _mm_storeu_ps(x + i + 8, x2); + _mm_storeu_ps(x + i + 12, x3); } simd_end = n - (n % 4); for (size_t i = simd_end4; i < simd_end; i += 4) { - _mm_store_ps(x + i, _mm_max_ps(zeros, _mm_load_ps(x + i))); + _mm_storeu_ps(x + i, _mm_max_ps(zeros, _mm_loadu_ps(x + i))); } #endif @@ -341,31 +330,31 @@ namespace Deep __m512 zeros = _mm512_setzero_ps(); size_t simd_end4 = n - (n % 64); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end4); i += 64) { - __m512 x0 = _mm512_load_ps(x + i); - __m512 x1 = _mm512_load_ps(x + i + 16); - __m512 x2 = _mm512_load_ps(x + i + 32); - __m512 x3 = _mm512_load_ps(x + i + 48); + __m512 x0 = _mm512_loadu_ps(x + i); + __m512 x1 = _mm512_loadu_ps(x + i + 16); + __m512 x2 = _mm512_loadu_ps(x + i + 32); + __m512 x3 = _mm512_loadu_ps(x + i + 48); __mmask16 m0 = _mm512_cmp_ps_mask(x0, zeros, _CMP_GT_OQ); __mmask16 m1 = _mm512_cmp_ps_mask(x1, zeros, _CMP_GT_OQ); __mmask16 m2 = _mm512_cmp_ps_mask(x2, zeros, _CMP_GT_OQ); __mmask16 m3 = _mm512_cmp_ps_mask(x3, zeros, _CMP_GT_OQ); - _mm512_store_ps(x + i, _mm512_mask_blend_ps(m0, zeros, ones)); - _mm512_store_ps(x + i + 16, _mm512_mask_blend_ps(m1, zeros, ones)); - _mm512_store_ps(x + i + 32, _mm512_mask_blend_ps(m2, zeros, ones)); - _mm512_store_ps(x + i + 48, _mm512_mask_blend_ps(m3, zeros, ones)); + _mm512_storeu_ps(x + i, _mm512_mask_blend_ps(m0, zeros, ones)); + _mm512_storeu_ps(x + i + 16, _mm512_mask_blend_ps(m1, zeros, ones)); + _mm512_storeu_ps(x + i + 32, _mm512_mask_blend_ps(m2, zeros, ones)); + _mm512_storeu_ps(x + i + 48, _mm512_mask_blend_ps(m3, zeros, ones)); } simd_end = n - (n % 16); for (size_t i = simd_end4; i < simd_end; i += 16) { - __m512 x0 = _mm512_load_ps(x + i); + __m512 x0 = _mm512_loadu_ps(x + i); __mmask16 m0 = _mm512_cmp_ps_mask(x0, zeros, _CMP_GT_OQ); - _mm512_store_ps(x + i, _mm512_mask_blend_ps(m0, zeros, ones)); + _mm512_storeu_ps(x + i, _mm512_mask_blend_ps(m0, zeros, ones)); } #elif defined(__AVX2__) || defined(__AVX__) @@ -373,30 +362,30 @@ namespace Deep __m256 zeros = _mm256_setzero_ps(); size_t simd_end4 = n - (n % 32); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end4); i += 32) { - __m256 x0 = _mm256_load_ps(x + i); - __m256 x1 = _mm256_load_ps(x + i + 8); - __m256 x2 = _mm256_load_ps(x + i + 16); - __m256 x3 = _mm256_load_ps(x + i + 24); + __m256 x0 = _mm256_loadu_ps(x + i); + __m256 x1 = _mm256_loadu_ps(x + i + 8); + __m256 x2 = _mm256_loadu_ps(x + i + 16); + __m256 x3 = _mm256_loadu_ps(x + i + 24); x0 = _mm256_and_ps(ones, _mm256_cmp_ps(x0, zeros, _CMP_GT_OQ)); x1 = _mm256_and_ps(ones, _mm256_cmp_ps(x1, zeros, _CMP_GT_OQ)); x2 = _mm256_and_ps(ones, _mm256_cmp_ps(x2, zeros, _CMP_GT_OQ)); x3 = _mm256_and_ps(ones, _mm256_cmp_ps(x3, zeros, _CMP_GT_OQ)); - _mm256_store_ps(x + i, x0); - _mm256_store_ps(x + i + 8, x1); - _mm256_store_ps(x + i + 16, x2); - _mm256_store_ps(x + i + 24, x3); + _mm256_storeu_ps(x + i, x0); + _mm256_storeu_ps(x + i + 8, x1); + _mm256_storeu_ps(x + i + 16, x2); + _mm256_storeu_ps(x + i + 24, x3); } simd_end = n - (n % 8); for (size_t i = simd_end4; i < simd_end; i += 8) { - __m256 x0 = _mm256_load_ps(x + i); - _mm256_store_ps(x + i, _mm256_and_ps(ones, _mm256_cmp_ps(x0, zeros, _CMP_GT_OQ))); + __m256 x0 = _mm256_loadu_ps(x + i); + _mm256_storeu_ps(x + i, _mm256_and_ps(ones, _mm256_cmp_ps(x0, zeros, _CMP_GT_OQ))); } #elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) @@ -404,30 +393,30 @@ namespace Deep __m128 zeros = _mm_setzero_ps(); size_t simd_end4 = n - (n % 16); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end4); i += 16) { - __m128 x0 = _mm_load_ps(x + i); - __m128 x1 = _mm_load_ps(x + i + 4); - __m128 x2 = _mm_load_ps(x + i + 8); - __m128 x3 = _mm_load_ps(x + i + 12); + __m128 x0 = _mm_loadu_ps(x + i); + __m128 x1 = _mm_loadu_ps(x + i + 4); + __m128 x2 = _mm_loadu_ps(x + i + 8); + __m128 x3 = _mm_loadu_ps(x + i + 12); x0 = _mm_and_ps(ones, _mm_cmpgt_ps(x0, zeros)); x1 = _mm_and_ps(ones, _mm_cmpgt_ps(x1, zeros)); x2 = _mm_and_ps(ones, _mm_cmpgt_ps(x2, zeros)); x3 = _mm_and_ps(ones, _mm_cmpgt_ps(x3, zeros)); - _mm_store_ps(x + i, x0); - _mm_store_ps(x + i + 4, x1); - _mm_store_ps(x + i + 8, x2); - _mm_store_ps(x + i + 12, x3); + _mm_storeu_ps(x + i, x0); + _mm_storeu_ps(x + i + 4, x1); + _mm_storeu_ps(x + i + 8, x2); + _mm_storeu_ps(x + i + 12, x3); } simd_end = n - (n % 4); for (size_t i = simd_end4; i < simd_end; i += 4) { - __m128 x0 = _mm_load_ps(x + i); - _mm_store_ps(x + i, _mm_and_ps(ones, _mm_cmpgt_ps(x0, zeros))); + __m128 x0 = _mm_loadu_ps(x + i); + _mm_storeu_ps(x + i, _mm_and_ps(ones, _mm_cmpgt_ps(x0, zeros))); } #endif @@ -460,42 +449,39 @@ namespace Deep #if defined(__AVX512F__) simd_end = n - (n % 16); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 16) { - __m512 x_512 = _mm512_load_ps(x + i); - // u10 guarantees 1.0 ULP accuracy (highly precise) + __m512 x_512 = _mm512_loadu_ps(x + i); __m512 res = Sleef_tanhf16_u10avx512f(x_512); - _mm512_store_ps(x + i, res); + _mm512_storeu_ps(x + i, res); } #elif defined(__AVX2__) simd_end = n - (n % 8); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 8) { - __m256 x_256 = _mm256_load_ps(x + i); + __m256 x_256 = _mm256_loadu_ps(x + i); __m256 res = Sleef_tanhf8_u10avx2(x_256); - _mm256_store_ps(x + i, res); + _mm256_storeu_ps(x + i, res); } #elif defined(__SSE4_1__) || defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) simd_end = n - (n % 4); -#pragma omp parallel for schedule(static) if (n > 65536) +#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 4) { - __m128 x_128 = _mm_load_ps(x + i); - // Fallback to sse4 or sse2 depending on what your SLEEF build exposes + __m128 x_128 = _mm_loadu_ps(x + i); #if defined(__SSE4_1__) __m128 res = Sleef_tanhf4_u10sse4(x_128); #else __m128 res = Sleef_tanhf4_u10sse2(x_128); #endif - _mm_store_ps(x + i, res); + _mm_storeu_ps(x + i, res); } #endif - // Scalar remainder for (size_t i = simd_end; i < n; i++) { x[i] = Sleef_tanhf_u10(x[i]); @@ -519,14 +505,14 @@ namespace Deep for (; i < simd_end; i += 16) { - __m512 t = _mm512_load_ps(x + i); + __m512 t = _mm512_loadu_ps(x + i); #ifdef __FMA__ __m512 res = _mm512_fnmadd_ps(t, t, ones); // 1 - t*t #else __m512 res = _mm512_sub_ps(ones, _mm512_mul_ps(t, t)); #endif - _mm512_store_ps(x + i, res); + _mm512_storeu_ps(x + i, res); } #elif defined(__AVX2__) @@ -535,13 +521,13 @@ namespace Deep for (; i < simd_end; i += 8) { - __m256 t = _mm256_load_ps(x + i); + __m256 t = _mm256_loadu_ps(x + i); #ifdef __FMA__ __m256 res = _mm256_fnmadd_ps(t, t, ones); // 1 - t*t #else __m256 res = _mm256_sub_ps(ones, _mm256_mul_ps(t, t)); #endif - _mm256_store_ps(x + i, res); + _mm256_storeu_ps(x + i, res); } #elif defined(__SSE4_1__) || defined(_M_AMD64) || defined(_M_X64) __m128 ones = _mm_set1_ps(1.0f); @@ -549,14 +535,14 @@ namespace Deep for (; i < simd_end; i += 4) { - __m128 t = _mm_load_ps(x + i); + __m128 t = _mm_loadu_ps(x + i); #ifdef __FMA__ __m128 res = _mm_fnmadd_ps(t, t, ones); // 1 - t*t #else __m128 res = _mm_sub_ps(ones, _mm_mul_ps(t, t)); #endif - _mm_store_ps(x + i, res); + _mm_storeu_ps(x + i, res); } #endif @@ -569,7 +555,7 @@ namespace Deep #pragma region sigmoid /// @brief Implements the \em Logistic \em Sigmoid approximation, i.e. `S(x) = 1 / (1 + e^(-x))` - /// @param x array, \em assumed to be 64-bit aligned! + /// @param x array, \em does not need to be aligned /// @param n x length static inline void sigmoid(float *RESTRICT x, const size_t n) noexcept { @@ -585,12 +571,12 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 16) { - __m512 x_512 = _mm512_load_ps(x + i); + __m512 x_512 = _mm512_loadu_ps(x + i); __m512 neg_x = _mm512_mul_ps(x_512, neg_one); __m512 exp_neg_x = Sleef_expf16_u10avx512f(neg_x); __m512 den = _mm512_add_ps(exp_neg_x, one); __m512 sig = _mm512_div_ps(one, den); - _mm512_store_ps(x + i, sig); + _mm512_storeu_ps(x + i, sig); } #elif defined(__AVX2__) __m256 one = _mm256_set1_ps(1.0f); @@ -599,12 +585,12 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 8) { - __m256 x_256 = _mm256_load_ps(x + i); + __m256 x_256 = _mm256_loadu_ps(x + i); __m256 neg_x = _mm256_mul_ps(x_256, neg_one); __m256 exp_neg_x = Sleef_expf8_u10avx2(neg_x); __m256 den = _mm256_add_ps(exp_neg_x, one); __m256 sig = _mm256_div_ps(one, den); - _mm256_store_ps(x + i, sig); + _mm256_storeu_ps(x + i, sig); } #elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) __m128 one = _mm_set1_ps(1.0f); @@ -613,12 +599,12 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 4) { - __m128 x_128 = _mm_load_ps(x + i); + __m128 x_128 = _mm_loadu_ps(x + i); __m128 neg_x = _mm_mul_ps(x_128, neg_one); __m128 exp_neg_x = Sleef_expf4_u10sse2(neg_x); __m128 den = _mm_add_ps(exp_neg_x, one); __m128 sig = _mm_div_ps(one, den); - _mm_store_ps(x + i, sig); + _mm_storeu_ps(x + i, sig); } #endif for (; i < n; i++) @@ -628,7 +614,7 @@ namespace Deep } /// @brief Implements the \em Elliot \em Sigmoid approximation, i.e. `S(x) = (1/2)((x / (1 + |x|)) + 1)` - /// @param x array, \em assumed to be 64-bit aligned! + /// @param x array, \em does not need to be aligned /// @param n x length inline void e_sigmoid(float *RESTRICT x, const size_t n) noexcept { @@ -644,14 +630,14 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 16) { - __m512 x_512 = _mm512_load_ps(x + i); + __m512 x_512 = _mm512_loadu_ps(x + i); __m512 den = _mm512_add_ps( _mm512_abs_ps(x_512), one); __m512 div = _mm512_div_ps(x_512, den); __m512 sig = _mm512_fmadd_ps(div, half, half); - _mm512_store_ps(x + i, sig); + _mm512_storeu_ps(x + i, sig); } #elif defined(__AVX2__) __m256 half = _mm256_set1_ps(0.5f); @@ -661,7 +647,7 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 8) { - __m256 x_256 = _mm256_load_ps(x + i); + __m256 x_256 = _mm256_loadu_ps(x + i); __m256 den = _mm256_add_ps( _mm256_and_ps(x_256, mask), one); @@ -672,7 +658,7 @@ namespace Deep __m256 sig = _mm256_add_ps(_mm256_mul_ps(div, half), half); #endif - _mm256_store_ps(x + i, sig); + _mm256_storeu_ps(x + i, sig); } #elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) __m128 half = _mm_set1_ps(0.5f); @@ -682,7 +668,7 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 4) { - __m128 x_128 = _mm_load_ps(x + i); + __m128 x_128 = _mm_loadu_ps(x + i); __m128 den = _mm_add_ps( _mm_and_ps(x_128, mask), one); @@ -693,7 +679,7 @@ namespace Deep __m128 sig = _mm_add_ps(_mm_mul_ps(div, half), half); #endif - _mm_store_ps(x + i, sig); + _mm_storeu_ps(x + i, sig); } #endif for (; i < n; i++) @@ -717,9 +703,9 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 16) { - __m512 x_512 = _mm512_load_ps(x + i); + __m512 x_512 = _mm512_loadu_ps(x + i); __m512 d = _mm512_fnmadd_ps(x_512, x_512, x_512); // d = x * (1 - x) = x - x^2 = -x*x + x - _mm512_store_ps(x + i, d); + _mm512_storeu_ps(x + i, d); } #elif defined(__AVX2__) @@ -727,27 +713,27 @@ namespace Deep size_t simd_end = n - r; for (; i < simd_end; i += 8) { - __m256 x_256 = _mm256_load_ps(x + i); + __m256 x_256 = _mm256_loadu_ps(x + i); #ifdef __FMA__ __m256 d = _mm256_fnmadd_ps(x_256, x_256, x_256); // d = x * (1 - x) = x - x^2 = -x*x + x #else __m256 d = _mm256_sub_ps(x_256, _mm256_mul_ps(x_256, x_256)); #endif - _mm256_store_ps(x + i, d); + _mm256_storeu_ps(x + i, d); } #elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) size_t r = n % 4; size_t simd_end = n - r; for (; i < simd_end; i += 4) { - __m128 x_128 = _mm_load_ps(x + i); + __m128 x_128 = _mm_loadu_ps(x + i); #ifdef __FMA__ __m128 d = _mm_fnmadd_ps(x_128, x_128, x_128); // d = x * (1 - x) = x - x^2 = -x*x + x #else __m128 d = _mm_sub_ps(x_128, _mm_mul_ps(x_128, x_128)); #endif - _mm_store_ps(x + i, d); + _mm_storeu_ps(x + i, d); } #endif @@ -764,4 +750,4 @@ namespace Deep { std::fill_n(x, n, 1.0f); } -} \ No newline at end of file +} diff --git a/include/deepity/networks/ConvPCNetwork.h b/include/deepity/networks/ConvPCNetwork.h index b6534db..0af2857 100644 --- a/include/deepity/networks/ConvPCNetwork.h +++ b/include/deepity/networks/ConvPCNetwork.h @@ -65,7 +65,7 @@ namespace Deep * @brief Destroys the network, freeing every owned ConvPCLayer * and releasing the backing MemoryArena. */ - ~ConvPCNetwork(); + ~ConvPCNetwork() = default; // No copy (owns raw pointers + a MemoryArena); move not implemented // either, matching DiscriminativePCNetwork's conventions. @@ -207,7 +207,7 @@ namespace Deep * @return Pointer to the last layer added via AddLayer(); this is * the layer with outChannels==0. */ - ConvPCLayer *GetTerminalLayer() noexcept { return layers.back(); } + ConvPCLayer *GetTerminalLayer() noexcept { return layers.back().get(); } /** * @brief Returns every layer in the network, in the order they @@ -215,7 +215,7 @@ namespace Deep * @return A const reference to the internal layer list. Valid for * the lifetime of this ConvPCNetwork. */ - const std::vector &GetLayers() const noexcept { return layers; } + const auto &GetLayers() const noexcept { return layers; } /** * @brief Returns the fixed batch size this network was @@ -257,7 +257,7 @@ namespace Deep private: /// @brief Every layer in the network, in the order added via /// AddLayer(); owned raw pointers, freed in the destructor. - std::vector layers; + std::vector> layers; /// @brief Single contiguous memory arena backing every layer's /// beliefs/errors/weights, bound during Compile(). diff --git a/include/deepity/networks/DiscriminativePCNetwork.h b/include/deepity/networks/DiscriminativePCNetwork.h index 97d3c78..3efe0a2 100644 --- a/include/deepity/networks/DiscriminativePCNetwork.h +++ b/include/deepity/networks/DiscriminativePCNetwork.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -33,7 +34,7 @@ namespace Deep /// @see https://arxiv.org/pdf/2506.06332 class DiscriminativePCNetwork { - std::vector layers; + std::vector> layers; int batchSize; bool autoSize = true; @@ -49,7 +50,7 @@ namespace Deep DiscriminativePCNetwork(int batchSize) : batchSize(batchSize), autoSize(false) {} /// @brief Default constructor; deletes each layer. - ~DiscriminativePCNetwork(); + ~DiscriminativePCNetwork() = default; DiscriminativePCNetwork(DiscriminativePCNetwork &&other) noexcept = default; DiscriminativePCNetwork &operator=(DiscriminativePCNetwork &&other) noexcept = default; @@ -91,7 +92,7 @@ namespace Deep void ResetState() noexcept; - const std::vector &GetLayers() const noexcept { return layers; } + const auto &GetLayers() const noexcept { return layers; } /// @brief Returns the batch size for the network's layers /// @return size_t batchSize @@ -101,27 +102,27 @@ namespace Deep { if (layers.empty()) return nullptr; - return layers.back(); + return layers.back().get(); } void SetLearningRate(float lr) { - for (auto layer : layers) + for (auto &layer : layers) layer->SetLearningRate(lr); } void SetInferenceRate(float ir) { - for (auto layer : layers) + for (auto &layer : layers) layer->SetLearningRate(ir); } void SetPrecisionRate(float pr) { - for (auto layer : layers) + for (auto &layer : layers) layer->SetPrecisionRate(pr); } void SetLambda(float l) { - for (auto layer : layers) + for (auto &layer : layers) layer->SetLambda(l); } diff --git a/include/deepity/networks/SimpleConvPCNetwork.h b/include/deepity/networks/SimpleConvPCNetwork.h index d490695..9ed80dd 100644 --- a/include/deepity/networks/SimpleConvPCNetwork.h +++ b/include/deepity/networks/SimpleConvPCNetwork.h @@ -34,11 +34,12 @@ namespace Deep { public: explicit SimpleConvPCNetwork(int batchSize) noexcept; - ~SimpleConvPCNetwork(); SimpleConvPCNetwork(const SimpleConvPCNetwork &) = delete; SimpleConvPCNetwork &operator=(const SimpleConvPCNetwork &) = delete; + ~SimpleConvPCNetwork() = default; + /// @brief Adds a convolutional layer. Pass outChannels=0 to mark a /// terminal layer (no outgoing prediction), matching /// SimpleConvPCLayer's nextSize=0 convention. @@ -77,8 +78,8 @@ namespace Deep /// against outChannels==0, so this is belt-and-suspenders). void UpdateWeights() noexcept; - SimpleConvPCLayer *GetTerminalLayer() noexcept { return layers.back(); } - const std::vector &GetLayers() const noexcept { return layers; } + SimpleConvPCLayer *GetTerminalLayer() noexcept { return layers.back().get(); } + const auto &GetLayers() const noexcept { return layers; } int GetBatchSize() const noexcept { return batchSize; } /// @brief Full train step: clamp input+target, settle for @@ -92,7 +93,7 @@ namespace Deep std::vector Predict(const std::vector &x, int inferenceSteps); private: - std::vector layers; + std::vector> layers; std::unique_ptr arena; int batchSize; OptimizerType pendingOpt = OptimizerType::SGD; diff --git a/include/deepity/networks/SimplePCNetwork.h b/include/deepity/networks/SimplePCNetwork.h index b0917dd..f2d10b9 100644 --- a/include/deepity/networks/SimplePCNetwork.h +++ b/include/deepity/networks/SimplePCNetwork.h @@ -37,12 +37,11 @@ namespace Deep /// @param batchSize Batch size explicit SimplePCNetwork(int batchSize) noexcept; - /// @brief Default destructor; deletes each layer. - ~SimplePCNetwork(); - SimplePCNetwork(const SimplePCNetwork &) = delete; SimplePCNetwork &operator=(const SimplePCNetwork &) = delete; + ~SimplePCNetwork() = default; + /// @brief Adds a layer to the network. /// @param size input size /// @param nextSize output size @@ -82,15 +81,15 @@ namespace Deep /// @brief Returns the network's terminal (final) layer. /// @return Pointer to the last layer added via AddLayer(). - SimplePCLayer *GetTerminalLayer() noexcept { return layers.back(); } + SimplePCLayer *GetTerminalLayer() noexcept { return layers.back().get(); } /// @brief Returns every layer in the network, in the order they were added. /// @return A reference to the internal layer list. - std::vector &GetLayers() noexcept { return layers; } + std::vector> &GetLayers() noexcept { return layers; } /// @brief Returns every layer in the network, in the order they were added. /// @return A const reference to the internal layer list. - const std::vector &GetLayers() const noexcept { return layers; } + const std::vector> &GetLayers() const noexcept { return layers; } /// @brief Returns the batch size for the network's layers /// @return size_t batchSize @@ -100,7 +99,7 @@ namespace Deep /// @param o The optimizer type to apply. void SetOptimizer(OptimizerType o) noexcept { - for (SimplePCLayer *layer : layers) + for (auto &layer : layers) layer->SetOptimizer(o); } @@ -134,12 +133,32 @@ namespace Deep /// point each layer's CalculateState() runs). Only mu is used. void ProjectForward() noexcept; + /// @brief Full train step WITH forward-projection initialization, + /// all in ONE call -- reset, clamp, project, settle, update weights, + /// unclamp. Matches TrainStep()'s signature/return convention exactly, + /// just with ProjectForward() inserted between clamping the input and + /// clamping the target. + /// + /// Exists specifically to eliminate the Python/pybind boundary- + /// crossing overhead of doing this same sequence via many separate + /// calls from Python (reset_state, clamp_input, project_forward, + /// clamp_state, then STEPS*2 individual calculate_state/update_state + /// calls, update_weights, unclamp_state -- over 40 individual + /// crossings per batch at STEPS=20). This does the whole sequence in + /// ONE crossing instead. + float TrainStepWithProjection(const std::vector &x, const std::vector &y, int inferenceSteps); + + /// @brief Sets mu-cache threshold on every layer -- see + /// SimplePCLayer::SetMuCacheThreshold() for semantics. Safe to call any + /// time after Compile(). + void SetMuCacheThreshold(float threshold) noexcept; + /// @brief Loads all layers into one contiguous block of memory. void Compile(); private: - std::vector layers; + std::vector> layers; std::unique_ptr arena; int batchSize; }; -} \ No newline at end of file +} diff --git a/logs/build.log b/logs/build.log index 6e73be5..0851ab6 100644 --- a/logs/build.log +++ b/logs/build.log @@ -1,4 +1,4 @@ ---- Deepity Build Log (Release) --- +--- Deepity Build Log (Release, arch=fast) --- === Compilation === ninja: no work to do. @@ -7,27 +7,27 @@ ninja: no work to do. === Tests === === Part 1: SGD weight-gradient check === - W[5]: delta=0.03608 numeric_dE/dW=-0.689149 MATCHES DESCENT rel_err=0.0023544 + W[5]: delta=0.03608 numeric_dE/dW=-0.68903 MATCHES DESCENT rel_err=0.00236346 W[17]: delta=0.0546692 numeric_dE/dW=-1.01769 MATCHES DESCENT rel_err=0.00371893 - W[6]: delta=-0.018854 numeric_dE/dW=0.368714 MATCHES DESCENT rel_err=0.0011345 - W[4]: delta=0.000938594 numeric_dE/dW=-0.026226 MATCHES DESCENT rel_err=0.0142114 - W[8]: delta=-0.0150366 numeric_dE/dW=0.299931 MATCHES DESCENT rel_err=0.000133743 - W[1]: delta=-0.0190711 numeric_dE/dW=0.371456 MATCHES DESCENT rel_err=0.00134147 + W[6]: delta=-0.018854 numeric_dE/dW=0.368834 MATCHES DESCENT rel_err=0.00111797 + W[4]: delta=0.000938594 numeric_dE/dW=-0.0263453 MATCHES DESCENT rel_err=0.0143733 + W[8]: delta=-0.0150366 numeric_dE/dW=0.30005 MATCHES DESCENT rel_err=0.000113825 + W[1]: delta=-0.0190711 numeric_dE/dW=0.371337 MATCHES DESCENT rel_err=0.00135795 W[14]: delta=0.0116533 numeric_dE/dW=-0.317812 MATCHES DESCENT rel_err=0.0133326 - W[11]: delta=0.0684187 numeric_dE/dW=-1.29008 MATCHES DESCENT rel_err=0.00303433 -Worst relative error: 0.0142114 + W[11]: delta=0.0684187 numeric_dE/dW=-1.28984 MATCHES DESCENT rel_err=0.00304413 +Worst relative error: 0.0143733 PASS === Part 2: feedback-term (Col2Im) verification, SimpleConvPCLayer === - z[3]: implied_dz_dt=0.0045494 -numeric_dE/dz=0.00455976 rel_err=0.00227074 - z[19]: implied_dz_dt=-0.0106531 -numeric_dE/dz=-0.0106394 rel_err=0.00128848 + z[3]: implied_dz_dt=0.0045494 -numeric_dE/dz=0.00452995 rel_err=0.00429182 + z[19]: implied_dz_dt=-0.0106531 -numeric_dE/dz=-0.0106096 rel_err=0.00410083 z[15]: implied_dz_dt=0.0041806 -numeric_dE/dz=0.00420213 rel_err=0.00512289 - z[26]: implied_dz_dt=-0.00197962 -numeric_dE/dz=-0.00199676 rel_err=0.00857779 + z[26]: implied_dz_dt=-0.00197962 -numeric_dE/dz=-0.00196695 rel_err=0.00643612 z[1]: implied_dz_dt=-0.0406229 -numeric_dE/dz=-0.0406206 rel_err=5.87843e-05 - z[5]: implied_dz_dt=-0.0247908 -numeric_dE/dz=-0.0248253 rel_err=0.00139243 + z[5]: implied_dz_dt=-0.0247908 -numeric_dE/dz=-0.0247955 rel_err=0.000192225 z[29]: implied_dz_dt=0.0272506 -numeric_dE/dz=0.0272691 rel_err=0.000677503 z[12]: implied_dz_dt=0.0424355 -numeric_dE/dz=0.0424385 rel_err=7.01353e-05 -Worst relative error: 0.00857779 +Worst relative error: 0.00643612 PASS === Part 3: AdamW weight-gradient check (NEW port, checking SIGN first) === diff --git a/pydeepity/ConvolutionalPCN.py b/pydeepity/ConvolutionalPCN.py new file mode 100644 index 0000000..1ed1962 --- /dev/null +++ b/pydeepity/ConvolutionalPCN.py @@ -0,0 +1,79 @@ +from ._backend import dy +from .utils import _PCNMixin +from typing import Optional +import numpy as np +import numpy.typing as npt + +class ConvolutionalPCN(dy.ConvPCNetwork, _PCNMixin): + """ + A Convolutional Predictive Coding Network (PCN) wrapper for the Deepity C++ backend. + """ + def __init__(self, batch_size: int) -> None: + super().__init__(batch_size) + self._last_shape: Optional[tuple[int, int, int]] = None + + def add_layer( + self, + out_channels: int, + kernel_h: int, + kernel_w: int, + in_channels: Optional[int] = None, + in_height: Optional[int] = None, + in_width: Optional[int] = None, + stride_h: int = 1, + stride_w: int = 1, + pad_h: int = 0, + pad_w: int = 0, + lr: float = 1e-6, + ir: float = 0.1, + pr: float = 0.0, + lmbda: float = 1e-4, + act: str = "relu", + ) -> None: + shape_args = (in_channels, in_height, in_width) + n_given = sum(a is not None for a in shape_args) + + if n_given == 0: + if self._last_shape is None: + raise ValueError( + "First add_layer() call must specify in_channels, in_height, and in_width explicitly." + ) + in_channels, in_height, in_width = self._last_shape + elif n_given != 3: + raise ValueError("in_channels/in_height/in_width must be given ALL together or OMITTED all together.") + + super().add_layer( + in_channels, out_channels, in_height, in_width, + kernel_h, kernel_w, stride_h=stride_h, stride_w=stride_w, pad_h=pad_h, pad_w=pad_w, + lr=lr, ir=ir, pr=pr, lmbda=lmbda, activation=act, activation_deriv="d" + act, + ) + + if out_channels > 0: + added = self[-1] + self._last_shape = (added.out_channels, added.out_height, added.out_width) + else: + self._last_shape = None + + def compile(self) -> None: + super().compile() + + def randomize_weights(self) -> None: + super().randomize_weights() + + def set_learning_rate(self, lr: float) -> None: + for layer in self.layers: + layer.set_learning_rate(lr) + + def set_inference_rate(self, ir: float) -> None: + for layer in self.layers: + layer.set_inference_rate(ir) + + def set_precision_rate(self, pr: float) -> None: + for layer in self.layers: + layer.set_precision_rate(pr) + + def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: + return super().train_step(X.flatten(), Y.flatten(), steps) + + def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: + return super().predict(X.flatten(), steps) diff --git a/pydeepity/SequentialPCN.py b/pydeepity/SequentialPCN.py new file mode 100644 index 0000000..665a199 --- /dev/null +++ b/pydeepity/SequentialPCN.py @@ -0,0 +1,72 @@ +from ._backend import dy +from typing import Optional +import numpy as np +import numpy.typing as npt +from .utils import _PCNMixin + +class SequentialPCN(dy.DiscriminativePCNetwork, _PCNMixin): + """ + A Sequential Predictive Coding Network wrapper for the Deepity C++ backend. + """ + def __init__(self, batch_size: Optional[int] = None) -> None: + bsz = dy.auto_batch_size() if batch_size is None else batch_size + super().__init__(bsz) + + def add_layer( + self, + in_features: int, + out_features: int, + lr: float, + ir: float, + pr: float, + act: str, + lmbda: float = 0.0001 + ) -> None: + super().add_layer( + in_features, out_features, lr=lr, ir=ir, pr=pr, + lmbda=lmbda, activation=act, activation_deriv='d' + act + ) + + def set_learning_rate(self, lr: float) -> None: + super().set_learning_rate(lr) + + def compile(self) -> None: + super().compile() + + def randomize_weights(self) -> None: + super().randomize_weights() + + def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: + """Executes a full forward clamp, state relaxation, and weight update cycle.""" + self.reset_state() + self.clamp_input(X) + self[-1].clamp_state(Y) + + total_energy: float = 0.0 + for _ in range(steps): + total_energy += self.calculate_state() + self.update_state() + + self.update_weights() + self.update_precision() + self[-1].unclamp_state() + + return total_energy + + def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: + """Runs generative/discriminative inference on the provided input data.""" + self.reset_state() + self.clamp_input(X.flatten()) + + for _ in range(steps): + self.calculate_state() + self.update_state() + + return np.array(self[-1].beliefs) + + def save(self, dir_path: str) -> bool: + return super().save(dir_path) + + def load(self, dir_path: str) -> bool: + return super().load(dir_path) + diff --git a/pydeepity/SimpleConvolutionalPCN.py b/pydeepity/SimpleConvolutionalPCN.py new file mode 100644 index 0000000..c86f237 --- /dev/null +++ b/pydeepity/SimpleConvolutionalPCN.py @@ -0,0 +1,84 @@ +from ._backend import dy +from .utils import _PCNMixin +from typing import Optional +import numpy as np +import numpy.typing as npt + +class SimpleConvolutionalPCN(dy.SimpleConvPCNetwork, _PCNMixin): + """ + A Convolutional Predictive Coding Network built from precision-free, + AdamW-capable SimpleConvPCLayers. Mirrors ConvolutionalPCN's + shape-inference convenience (in_channels/in_height/in_width can be + omitted after the first add_layer() call, inferred from the previous + layer's output shape) -- minus precision, which doesn't exist here. + """ + def __init__(self, batch_size: int) -> None: + super().__init__(batch_size) + self._last_shape: Optional[tuple[int, int, int]] = None + + def add_layer( + self, + out_channels: int, + kernel_h: int, + kernel_w: int, + in_channels: Optional[int] = None, + in_height: Optional[int] = None, + in_width: Optional[int] = None, + stride_h: int = 1, + stride_w: int = 1, + pad_h: int = 0, + pad_w: int = 0, + lr: float = 1e-6, + ir: float = 0.1, + lmbda: float = 1e-4, + act: str = "relu", + ) -> None: + shape_args = (in_channels, in_height, in_width) + n_given = sum(a is not None for a in shape_args) + + if n_given == 0: + if self._last_shape is None: + raise ValueError( + "First add_layer() call must specify in_channels, in_height, and in_width explicitly." + ) + in_channels, in_height, in_width = self._last_shape + elif n_given != 3: + raise ValueError("in_channels/in_height/in_width must be given ALL together or OMITTED all together.") + + super().add_layer( + in_channels, out_channels, in_height, in_width, + kernel_h, kernel_w, stride_h=stride_h, stride_w=stride_w, pad_h=pad_h, pad_w=pad_w, + lr=lr, ir=ir, lmbda=lmbda, activation=act, activation_deriv="d" + act, + ) + + if out_channels > 0: + added = self[-1] + self._last_shape = (added.out_channels, added.out_height, added.out_width) + else: + self._last_shape = None + + def set_optimizer(self, optimizer: str) -> None: + """Sets the optimizer: ADAM, ADAMW, or SGD. Call BEFORE compile() -- + see SimpleConvPCNetwork's C++ docs; buffer sizing depends on this + being set before Compile() allocates the shared arena.""" + super().set_optimizer(optimizer) + + def compile(self) -> None: + super().compile() + + def randomize_weights(self) -> None: + super().randomize_weights() + + def set_learning_rate(self, lr: float) -> None: + for layer in self.layers: + layer.set_learning_rate(lr) + + def set_inference_rate(self, ir: float) -> None: + for layer in self.layers: + layer.set_inference_rate(ir) + + def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: + return super().train_step(X.flatten(), Y.flatten(), steps) + + def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: + return super().predict(X.flatten(), steps) \ No newline at end of file diff --git a/pydeepity/SimplePCN.py b/pydeepity/SimplePCN.py new file mode 100644 index 0000000..10cd912 --- /dev/null +++ b/pydeepity/SimplePCN.py @@ -0,0 +1,223 @@ +from ._backend import dy +import numpy as np +import numpy.typing as npt +from typing import Optional + +from rich.console import Console +from rich.progress import ( + Progress, + SpinnerColumn, + BarColumn, + TextColumn, + MofNCompleteColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) +from .utils import _PCNMixin + +class SimplePCN(dy.SimplePCNetwork, _PCNMixin): + """ + A Sequential Predictive Coding Network built from precision-stripped SimplePCLayers. + """ + def __init__(self, batch_size: Optional[int] = None) -> None: + bsz = dy.auto_batch_size() if batch_size is None else batch_size + super().__init__(bsz) + + def add_layer( + self, + in_features: int, + out_features: int, + lr: float, + ir: float, + act: str, + lmbda: float = 0.0001 + ) -> None: + super().add_layer( + in_features, out_features, lr=lr, ir=ir, + lmbda=lmbda, activation=act, activation_deriv='d' + act + ) + + def set_learning_rate(self, lr: float) -> None: + for layer in self.layers: + layer.set_learning_rate(lr) + + def compile(self) -> None: + super().compile() + + def randomize_weights(self) -> None: + super().randomize_weights() + + def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: + self.reset_state() + self.clamp_input(X) + self[-1].clamp_state(Y) + + total_energy: float = 0.0 + for _ in range(steps): + total_energy += self.calculate_state() + self.update_state() + + self.update_weights() + self[-1].unclamp_state() + + return total_energy + + def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: + self.reset_state() + self.clamp_input(X.flatten()) + + for _ in range(steps): + self.calculate_state() + self.update_state() + + return np.array(self[-1].beliefs) + + def set_mu_cache_threshold(self, threshold: float) -> None: + """Sets the mu-cache staleness threshold on every layer: -1 disables + caching (default), 0 reproduces the exact clamped-only behavior + (safe, no approximation), >0 extends caching to unclamped layers + too as a genuine approximation -- validated via a real settling- + trajectory sweep, threshold~0.05-0.1 gave ~56-60% speedup with + final energy staying close to the true baseline. NOT yet validated + for its effect on real multi-epoch training accuracy -- that's a + separate, necessary check before trusting a given threshold in + production training, the same discipline every other optimization + this session has gone through.""" + for layer in self.layers: + layer.set_mu_cache_threshold(threshold) + + def fit_iavg( + self, + X: npt.NDArray[np.float32], + Y: npt.NDArray[np.float32], + labels: npt.NDArray[np.int32], + epochs: int, + steps: int, + per_class: int, + num_classes: int, + cache_layers: list[tuple[int, int]], + initial_lr: float = 0.01, + decay_rate: float = 1.0, + reset_cache_per_epoch: bool = True + ) -> "SimplePCN": + """ + Specialized training loop for deep networks that utilizes Class-Average + Caching (I_avg). Manages the StreamAlignedBatcher and dynamically seeds + MULTIPLE hidden layers simultaneously -- a generalization of the + original single hidden_layer_index/hidden_size version, which could + only ever cache one layer. + + @param cache_layers List of (layer_index, layer_size) pairs -- every + layer listed gets its own independent per-class cache, seeded and + captured every batch. layer_index is the index into self.layers + (0 = the first, input-adjacent layer; the terminal layer, always + the last index, should NOT be included here -- it's clamped to + the target directly, never cached). + """ + console = Console() + bsz = self.batch_size + + batcher = dy.StreamAlignedBatcher( + X, Y, labels, + X.shape[1], Y.shape[1], + num_classes, per_class, 42 + ) + n_batches = batcher.num_batches_per_epoch() + + layer_names = ", ".join(f"L{idx}({sz})" for idx, sz in cache_layers) + console.print( + f"\n[bold cyan]Training I_avg[/bold cyan] [dim]|[/dim] {epochs} epochs [dim]|[/dim] " + f"{steps} inference steps [dim]|[/dim] {n_batches} batches/epoch [dim]|[/dim] " + f"caching: {layer_names}\n" + ) + + progress = Progress( + SpinnerColumn(style="cyan"), + TextColumn("[bold blue]{task.description}"), + BarColumn(bar_width=40, style="blue", complete_style="cyan"), + MofNCompleteColumn(), + TextColumn("[dim]•[/dim]"), + TimeElapsedColumn(), + TextColumn("[dim]•[/dim]"), + TimeRemainingColumn(), + TextColumn("[magenta]{task.fields[stats]}"), + console=console, + ) + + # caches[layer_index] = {class: avg_vector} -- one independent cache + # per listed layer, not a single shared one. + caches: dict[int, dict[int, npt.NDArray[np.float32]]] = {idx: {} for idx, _ in cache_layers} + + with progress: + epoch_task = progress.add_task("[bold]Epochs", total=epochs, stats="") + batch_task = progress.add_task(" Batches", total=n_batches, stats="") + + for epoch in range(epochs): + if reset_cache_per_epoch: + for layer_idx in caches: + caches[layer_idx].clear() + + current_lr = initial_lr * (decay_rate ** epoch) + self.set_learning_rate(current_lr) + + epoch_energy = 0.0 + progress.reset(batch_task, total=n_batches) + + for b in range(n_batches): + X_batch, Y_batch, _ = batcher.get_batch() + self.reset_state() + + # 1. Apply EVERY listed layer's cache independently -- + # each layer seeds from its OWN cache, not a shared one. + for layer_idx, layer_size in cache_layers: + cache = caches[layer_idx] + if cache: + init_beliefs = np.zeros((bsz, layer_size), dtype=np.float32) + for c in range(num_classes): + if c in cache: + init_beliefs[c * per_class : (c + 1) * per_class] = cache[c] + + layer = self.layers[layer_idx] + np.copyto(layer.beliefs, init_beliefs) + + self.clamp_input(X_batch.flatten()) + self[-1].clamp_state(Y_batch.flatten()) + + # 2. Settle & Update + energy = 0.0 + for _ in range(steps): + energy += self.calculate_state() + self.update_state() + + self.update_weights() + + # 3. Cache EVERY listed layer's newly settled beliefs + # independently. + for layer_idx, layer_size in cache_layers: + settled = np.array(self.layers[layer_idx].beliefs, copy=False).reshape(bsz, layer_size) + for c in range(num_classes): + caches[layer_idx][c] = settled[ + c * per_class : (c + 1) * per_class + ].mean(axis=0, dtype=np.float32).astype(np.float32, copy=False) + + self[-1].unclamp_state() + + # 4. Progress tracking + epoch_energy += energy + avg_so_far = epoch_energy / (b + 1) + + progress.update( + batch_task, + advance=1, + stats=f"lr={current_lr:.5f} energy={energy:8.2f} avg={avg_so_far:8.2f}", + ) + + progress.update( + epoch_task, + advance=1, + stats=f"epoch {epoch + 1} avg energy = {epoch_energy / n_batches:.4f}", + ) + + console.print("\n[bold green]✓ I_avg Training complete.[/bold green]\n") + return self + diff --git a/pydeepity/__init__.py b/pydeepity/__init__.py index f550736..046f921 100644 --- a/pydeepity/__init__.py +++ b/pydeepity/__init__.py @@ -1,552 +1,49 @@ -import numpy as np -import numpy.typing as npt -from typing import Optional - -# Import the raw compiled C++ bindings -from . import pydeepity as dy - -from rich.console import Console -from rich.progress import ( - Progress, - SpinnerColumn, - BarColumn, - TextColumn, - MofNCompleteColumn, - TimeElapsedColumn, - TimeRemainingColumn, -) - -__all__ = ["SequentialPCN", "SimplePCN", "ConvolutionalPCN"] - -# ============================================================================ -# Shared Progress & Training Utilities -# ============================================================================ - -def _fit_with_progress( - net, - X: npt.NDArray[np.float32], - Y: npt.NDArray[np.float32], - epochs: int, - steps: int, - initial_lr: float = 0.01, - decay_rate: float = 1.0, - shuffle: bool = True, -) -> None: - """ - Shared training-loop implementation used by all PCN classes. - It delegates to the specific class's `train_step()` method. - """ - console = Console() - n = len(X) - bsz = net.batch_size - n_batches = n // bsz - - console.print( - f"\n[bold cyan]Training[/bold cyan] [dim]|[/dim] {epochs} epochs [dim]|[/dim] " - f"{steps} inference steps [dim]|[/dim] {n_batches} batches/epoch (batch_size={bsz})\n" - ) - - progress = Progress( - SpinnerColumn(style="cyan"), - TextColumn("[bold blue]{task.description}"), - BarColumn(bar_width=40, style="blue", complete_style="cyan"), - MofNCompleteColumn(), - TextColumn("[dim]•[/dim]"), - TimeElapsedColumn(), - TextColumn("[dim]•[/dim]"), - TimeRemainingColumn(), - TextColumn("[magenta]{task.fields[stats]}"), - console=console, - ) - - with progress: - epoch_task = progress.add_task("[bold]Epochs", total=epochs, stats="") - batch_task = progress.add_task(" Batches", total=n_batches, stats="") - - for epoch in range(epochs): - current_lr = initial_lr * (decay_rate ** epoch) - net.set_learning_rate(current_lr) - - if shuffle: - indices = np.random.permutation(n) - X_shuf, Y_shuf = X[indices], Y[indices] - else: - X_shuf, Y_shuf = X, Y - - epoch_energy = 0.0 - progress.reset(batch_task, total=n_batches) - - for b in range(n_batches): - X_batch = X_shuf[b * bsz : (b + 1) * bsz] - Y_batch = Y_shuf[b * bsz : (b + 1) * bsz] - - energy = net.train_step(X_batch, Y_batch, steps) - epoch_energy += energy - avg_so_far = epoch_energy / (b + 1) - - progress.update( - batch_task, - advance=1, - stats=f"lr={current_lr:.5f} energy={energy:8.2f} avg={avg_so_far:8.2f}", - ) - - progress.update( - epoch_task, - advance=1, - stats=f"epoch {epoch + 1} avg energy = {epoch_energy / n_batches:.4f}", - ) - - console.print("\n[bold green]✓ Training complete.[/bold green]\n") - - -class _PCNMixin: - """ - Provides shared top-level Python functionality (like fit()) to all Network wrappers. - """ - def fit( - self, - X: npt.NDArray[np.float32], - Y: npt.NDArray[np.float32], - epochs: int, - steps: int, - initial_lr: float = 0.01, - decay_rate: float = 1.0, - shuffle: bool = True, - ): - """ - Runs a full multi-epoch training loop with a live rich progress display. - Delegates per-batch execution to the class's `train_step()` method. - """ - _fit_with_progress(self, X, Y, epochs, steps, initial_lr, decay_rate, shuffle) - return self - - -# ============================================================================ -# Core Network Wrappers -# ============================================================================ - -class SequentialPCN(dy.DiscriminativePCNetwork, _PCNMixin): - """ - A Sequential Predictive Coding Network wrapper for the Deepity C++ backend. - """ - def __init__(self, batch_size: Optional[int] = None) -> None: - bsz = dy.auto_batch_size() if batch_size is None else batch_size - super().__init__(bsz) +__version__ = "1.0.0" + +from .SequentialPCN import SequentialPCN +from .SimplePCN import SimplePCN +from .ConvolutionalPCN import ConvolutionalPCN +from .SimpleConvolutionalPCN import SimpleConvolutionalPCN + +from ._backend import dy + +StreamAlignedBatcher = dy.StreamAlignedBatcher + +# Hardware & Threading Utilities +get_l2_cache_bytes = dy.get_l2_cache_bytes +auto_batch_size = dy.auto_batch_size +dynamic_thread = dy.dynamic_thread +omp_max_threads = dy.omp_max_threads +omp_num_procs = dy.omp_num_procs + +# In-place SIMD Activation Functions +relu = dy.relu +drelu = dy.drelu +tanh = dy.tanh +dtanh = dy.dtanh +sigmoid = dy.sigmoid +dsigmoid = dy.dsigmoid + +__all__ = [ + # Networks + "SequentialPCN", + "SimplePCN", + "ConvolutionalPCN", + # "SimpleConvolutionalPCN", - def add_layer( - self, - in_features: int, - out_features: int, - lr: float, - ir: float, - pr: float, - act: str, - lmbda: float = 0.0001 - ) -> None: - super().add_layer( - in_features, out_features, lr=lr, ir=ir, pr=pr, - lmbda=lmbda, activation=act, activation_deriv='d' + act - ) - - def set_learning_rate(self, lr: float) -> None: - super().set_learning_rate(lr) - - def compile(self) -> None: - super().compile() - - def randomize_weights(self) -> None: - super().randomize_weights() - - def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: - """Executes a full forward clamp, state relaxation, and weight update cycle.""" - self.reset_state() - self.clamp_input(X) - self[-1].clamp_state(Y) - - total_energy: float = 0.0 - for _ in range(steps): - total_energy += self.calculate_state() - self.update_state() - - self.update_weights() - self.update_precision() - self[-1].unclamp_state() - - return total_energy - - def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: - """Runs generative/discriminative inference on the provided input data.""" - self.reset_state() - self.clamp_input(X.flatten()) - - for _ in range(steps): - self.calculate_state() - self.update_state() - - return np.array(self[-1].beliefs) - - def save(self, dir_path: str) -> bool: - return super().save(dir_path) - - def load(self, dir_path: str) -> bool: - return super().load(dir_path) - - -class SimplePCN(dy.SimplePCNetwork, _PCNMixin): - """ - A Sequential Predictive Coding Network built from precision-stripped SimplePCLayers. - """ - def __init__(self, batch_size: Optional[int] = None) -> None: - bsz = dy.auto_batch_size() if batch_size is None else batch_size - super().__init__(bsz) + # Utilities + "StreamAlignedBatcher", + "get_l2_cache_bytes", + "auto_batch_size", + "dynamic_thread", + "omp_max_threads", + "omp_num_procs", - def add_layer( - self, - in_features: int, - out_features: int, - lr: float, - ir: float, - act: str, - lmbda: float = 0.0001 - ) -> None: - super().add_layer( - in_features, out_features, lr=lr, ir=ir, - lmbda=lmbda, activation=act, activation_deriv='d' + act - ) - - def set_learning_rate(self, lr: float) -> None: - for layer in self.layers: - layer.set_learning_rate(lr) - - def compile(self) -> None: - super().compile() - - def randomize_weights(self) -> None: - super().randomize_weights() - - def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: - self.reset_state() - self.clamp_input(X) - self[-1].clamp_state(Y) - - total_energy: float = 0.0 - for _ in range(steps): - total_energy += self.calculate_state() - self.update_state() - - self.update_weights() - self[-1].unclamp_state() - - return total_energy - - def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: - self.reset_state() - self.clamp_input(X.flatten()) - - for _ in range(steps): - self.calculate_state() - self.update_state() - - return np.array(self[-1].beliefs) - - def set_mu_cache_threshold(self, threshold: float) -> None: - """Sets the mu-cache staleness threshold on every layer: -1 disables - caching (default), 0 reproduces the exact clamped-only behavior - (safe, no approximation), >0 extends caching to unclamped layers - too as a genuine approximation -- validated via a real settling- - trajectory sweep, threshold~0.05-0.1 gave ~56-60% speedup with - final energy staying close to the true baseline. NOT yet validated - for its effect on real multi-epoch training accuracy -- that's a - separate, necessary check before trusting a given threshold in - production training, the same discipline every other optimization - this session has gone through.""" - for layer in self.layers: - layer.set_mu_cache_threshold(threshold) - - def fit_iavg( - self, - X: npt.NDArray[np.float32], - Y: npt.NDArray[np.float32], - labels: npt.NDArray[np.int32], - epochs: int, - steps: int, - per_class: int, - num_classes: int, - cache_layers: list[tuple[int, int]], - initial_lr: float = 0.01, - decay_rate: float = 1.0, - reset_cache_per_epoch: bool = True - ) -> "SimplePCN": - """ - Specialized training loop for deep networks that utilizes Class-Average - Caching (I_avg). Manages the StreamAlignedBatcher and dynamically seeds - MULTIPLE hidden layers simultaneously -- a generalization of the - original single hidden_layer_index/hidden_size version, which could - only ever cache one layer. - - @param cache_layers List of (layer_index, layer_size) pairs -- every - layer listed gets its own independent per-class cache, seeded and - captured every batch. layer_index is the index into self.layers - (0 = the first, input-adjacent layer; the terminal layer, always - the last index, should NOT be included here -- it's clamped to - the target directly, never cached). - """ - console = Console() - bsz = self.batch_size - - batcher = dy.StreamAlignedBatcher( - X, Y, labels, - X.shape[1], Y.shape[1], - num_classes, per_class, 42 - ) - n_batches = batcher.num_batches_per_epoch() - - layer_names = ", ".join(f"L{idx}({sz})" for idx, sz in cache_layers) - console.print( - f"\n[bold cyan]Training I_avg[/bold cyan] [dim]|[/dim] {epochs} epochs [dim]|[/dim] " - f"{steps} inference steps [dim]|[/dim] {n_batches} batches/epoch [dim]|[/dim] " - f"caching: {layer_names}\n" - ) - - progress = Progress( - SpinnerColumn(style="cyan"), - TextColumn("[bold blue]{task.description}"), - BarColumn(bar_width=40, style="blue", complete_style="cyan"), - MofNCompleteColumn(), - TextColumn("[dim]•[/dim]"), - TimeElapsedColumn(), - TextColumn("[dim]•[/dim]"), - TimeRemainingColumn(), - TextColumn("[magenta]{task.fields[stats]}"), - console=console, - ) - - # caches[layer_index] = {class: avg_vector} -- one independent cache - # per listed layer, not a single shared one. - caches: dict[int, dict[int, npt.NDArray[np.float32]]] = {idx: {} for idx, _ in cache_layers} - - with progress: - epoch_task = progress.add_task("[bold]Epochs", total=epochs, stats="") - batch_task = progress.add_task(" Batches", total=n_batches, stats="") - - for epoch in range(epochs): - if reset_cache_per_epoch: - for layer_idx in caches: - caches[layer_idx].clear() - - current_lr = initial_lr * (decay_rate ** epoch) - self.set_learning_rate(current_lr) - - epoch_energy = 0.0 - progress.reset(batch_task, total=n_batches) - - for b in range(n_batches): - X_batch, Y_batch, _ = batcher.get_batch() - self.reset_state() - - # 1. Apply EVERY listed layer's cache independently -- - # each layer seeds from its OWN cache, not a shared one. - for layer_idx, layer_size in cache_layers: - cache = caches[layer_idx] - if cache: - init_beliefs = np.zeros((bsz, layer_size), dtype=np.float32) - for c in range(num_classes): - if c in cache: - init_beliefs[c * per_class : (c + 1) * per_class] = cache[c] - - layer = self.layers[layer_idx] - np.copyto(layer.beliefs, init_beliefs) - - self.clamp_input(X_batch.flatten()) - self[-1].clamp_state(Y_batch.flatten()) - - # 2. Settle & Update - energy = 0.0 - for _ in range(steps): - energy += self.calculate_state() - self.update_state() - - self.update_weights() - - # 3. Cache EVERY listed layer's newly settled beliefs - # independently. - for layer_idx, layer_size in cache_layers: - settled = np.array(self.layers[layer_idx].beliefs, copy=False).reshape(bsz, layer_size) - for c in range(num_classes): - caches[layer_idx][c] = settled[ - c * per_class : (c + 1) * per_class - ].mean(axis=0, dtype=np.float32).astype(np.float32, copy=False) - - self[-1].unclamp_state() - - # 4. Progress tracking - epoch_energy += energy - avg_so_far = epoch_energy / (b + 1) - - progress.update( - batch_task, - advance=1, - stats=f"lr={current_lr:.5f} energy={energy:8.2f} avg={avg_so_far:8.2f}", - ) - - progress.update( - epoch_task, - advance=1, - stats=f"epoch {epoch + 1} avg energy = {epoch_energy / n_batches:.4f}", - ) - - console.print("\n[bold green]✓ I_avg Training complete.[/bold green]\n") - return self - -class ConvolutionalPCN(dy.ConvPCNetwork, _PCNMixin): - """ - A Convolutional Predictive Coding Network (PCN) wrapper for the Deepity C++ backend. - """ - def __init__(self, batch_size: int) -> None: - super().__init__(batch_size) - self._last_shape: Optional[tuple[int, int, int]] = None - - def add_layer( - self, - out_channels: int, - kernel_h: int, - kernel_w: int, - in_channels: Optional[int] = None, - in_height: Optional[int] = None, - in_width: Optional[int] = None, - stride_h: int = 1, - stride_w: int = 1, - pad_h: int = 0, - pad_w: int = 0, - lr: float = 1e-6, - ir: float = 0.1, - pr: float = 0.0, - lmbda: float = 1e-4, - act: str = "relu", - ) -> None: - shape_args = (in_channels, in_height, in_width) - n_given = sum(a is not None for a in shape_args) - - if n_given == 0: - if self._last_shape is None: - raise ValueError( - "First add_layer() call must specify in_channels, in_height, and in_width explicitly." - ) - in_channels, in_height, in_width = self._last_shape - elif n_given != 3: - raise ValueError("in_channels/in_height/in_width must be given ALL together or OMITTED all together.") - - super().add_layer( - in_channels, out_channels, in_height, in_width, - kernel_h, kernel_w, stride_h=stride_h, stride_w=stride_w, pad_h=pad_h, pad_w=pad_w, - lr=lr, ir=ir, pr=pr, lmbda=lmbda, activation=act, activation_deriv="d" + act, - ) - - if out_channels > 0: - added = self[-1] - self._last_shape = (added.out_channels, added.out_height, added.out_width) - else: - self._last_shape = None - - def compile(self) -> None: - super().compile() - - def randomize_weights(self) -> None: - super().randomize_weights() - - def set_learning_rate(self, lr: float) -> None: - for layer in self.layers: - layer.set_learning_rate(lr) - - def set_inference_rate(self, ir: float) -> None: - for layer in self.layers: - layer.set_inference_rate(ir) - - def set_precision_rate(self, pr: float) -> None: - for layer in self.layers: - layer.set_precision_rate(pr) - - def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: - return super().train_step(X.flatten(), Y.flatten(), steps) - - def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: - return super().predict(X.flatten(), steps) - -class SimpleConvolutionalPCN(dy.SimpleConvPCNetwork, _PCNMixin): - """ - A Convolutional Predictive Coding Network built from precision-free, - AdamW-capable SimpleConvPCLayers. Mirrors ConvolutionalPCN's - shape-inference convenience (in_channels/in_height/in_width can be - omitted after the first add_layer() call, inferred from the previous - layer's output shape) -- minus precision, which doesn't exist here. - """ - def __init__(self, batch_size: int) -> None: - super().__init__(batch_size) - self._last_shape: Optional[tuple[int, int, int]] = None - - def add_layer( - self, - out_channels: int, - kernel_h: int, - kernel_w: int, - in_channels: Optional[int] = None, - in_height: Optional[int] = None, - in_width: Optional[int] = None, - stride_h: int = 1, - stride_w: int = 1, - pad_h: int = 0, - pad_w: int = 0, - lr: float = 1e-6, - ir: float = 0.1, - lmbda: float = 1e-4, - act: str = "relu", - ) -> None: - shape_args = (in_channels, in_height, in_width) - n_given = sum(a is not None for a in shape_args) - - if n_given == 0: - if self._last_shape is None: - raise ValueError( - "First add_layer() call must specify in_channels, in_height, and in_width explicitly." - ) - in_channels, in_height, in_width = self._last_shape - elif n_given != 3: - raise ValueError("in_channels/in_height/in_width must be given ALL together or OMITTED all together.") - - super().add_layer( - in_channels, out_channels, in_height, in_width, - kernel_h, kernel_w, stride_h=stride_h, stride_w=stride_w, pad_h=pad_h, pad_w=pad_w, - lr=lr, ir=ir, lmbda=lmbda, activation=act, activation_deriv="d" + act, - ) - - if out_channels > 0: - added = self[-1] - self._last_shape = (added.out_channels, added.out_height, added.out_width) - else: - self._last_shape = None - - def set_optimizer(self, optimizer: str) -> None: - """Sets the optimizer: ADAM, ADAMW, or SGD. Call BEFORE compile() -- - see SimpleConvPCNetwork's C++ docs; buffer sizing depends on this - being set before Compile() allocates the shared arena.""" - super().set_optimizer(optimizer) - - def compile(self) -> None: - super().compile() - - def randomize_weights(self) -> None: - super().randomize_weights() - - def set_learning_rate(self, lr: float) -> None: - for layer in self.layers: - layer.set_learning_rate(lr) - - def set_inference_rate(self, ir: float) -> None: - for layer in self.layers: - layer.set_inference_rate(ir) - - def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: - return super().train_step(X.flatten(), Y.flatten(), steps) - - def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: - return super().predict(X.flatten(), steps) \ No newline at end of file + # Activations + "relu", + "drelu", + "tanh", + "dtanh", + "sigmoid", + "dsigmoid", +] \ No newline at end of file diff --git a/pydeepity/__pycache__/__init__.cpython-312.pyc b/pydeepity/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 02c23c3..0000000 Binary files a/pydeepity/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/pydeepity/__pycache__/__init__.cpython-314.pyc b/pydeepity/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index c0045ae..0000000 Binary files a/pydeepity/__pycache__/__init__.cpython-314.pyc and /dev/null differ diff --git a/pydeepity/_backend.py b/pydeepity/_backend.py new file mode 100644 index 0000000..b1d8839 --- /dev/null +++ b/pydeepity/_backend.py @@ -0,0 +1,7 @@ +try: + from . import pydeepity as dy +except ImportError as e: + raise ImportError( + "Could not load the compiled Deepity C++ backend. " + "Ensure the package was installed correctly or compiled for your architecture." + ) from e \ No newline at end of file diff --git a/pydeepity/utils.py b/pydeepity/utils.py new file mode 100644 index 0000000..92d57cb --- /dev/null +++ b/pydeepity/utils.py @@ -0,0 +1,113 @@ +import numpy as np +import numpy.typing as npt + +from rich.console import Console +from rich.progress import ( + Progress, + SpinnerColumn, + BarColumn, + TextColumn, + MofNCompleteColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) + +def _fit_with_progress( + net, + X: npt.NDArray[np.float32], + Y: npt.NDArray[np.float32], + epochs: int, + steps: int, + initial_lr: float = 0.01, + decay_rate: float = 1.0, + shuffle: bool = True, +) -> None: + """ + Shared training-loop implementation used by all PCN classes. + It delegates to the specific class's `train_step()` method. + """ + console = Console() + n = len(X) + bsz = net.batch_size + n_batches = n // bsz + + console.print( + f"\n[bold cyan]Training[/bold cyan] [dim]|[/dim] {epochs} epochs [dim]|[/dim] " + f"{steps} inference steps [dim]|[/dim] {n_batches} batches/epoch (batch_size={bsz})\n" + ) + + progress = Progress( + SpinnerColumn(style="cyan"), + TextColumn("[bold blue]{task.description}"), + BarColumn(bar_width=40, style="blue", complete_style="cyan"), + MofNCompleteColumn(), + TextColumn("[dim]•[/dim]"), + TimeElapsedColumn(), + TextColumn("[dim]•[/dim]"), + TimeRemainingColumn(), + TextColumn("[magenta]{task.fields[stats]}"), + console=console, + ) + + with progress: + epoch_task = progress.add_task("[bold]Epochs", total=epochs, stats="") + batch_task = progress.add_task(" Batches", total=n_batches, stats="") + + for epoch in range(epochs): + current_lr = initial_lr * (decay_rate ** epoch) + net.set_learning_rate(current_lr) + + if shuffle: + indices = np.random.permutation(n) + X_shuf, Y_shuf = X[indices], Y[indices] + else: + X_shuf, Y_shuf = X, Y + + epoch_energy = 0.0 + progress.reset(batch_task, total=n_batches) + + for b in range(n_batches): + X_batch = X_shuf[b * bsz : (b + 1) * bsz] + Y_batch = Y_shuf[b * bsz : (b + 1) * bsz] + + energy = net.train_step(X_batch, Y_batch, steps) + epoch_energy += energy + avg_so_far = epoch_energy / (b + 1) + + progress.update( + batch_task, + advance=1, + stats=f"lr={current_lr:.5f} energy={energy:8.2f} avg={avg_so_far:8.2f}", + ) + + progress.update( + epoch_task, + advance=1, + stats=f"epoch {epoch + 1} avg energy = {epoch_energy / n_batches:.4f}", + ) + + console.print("\n[bold green]✓ Training complete.[/bold green]\n") + + +class _PCNMixin: + """ + Provides shared top-level Python functionality (like fit()) to all Network wrappers. + """ + def fit( + self, + X: npt.NDArray[np.float32], + Y: npt.NDArray[np.float32], + epochs: int, + steps: int, + initial_lr: float = 0.01, + decay_rate: float = 1.0, + shuffle: bool = True, + ): + """ + Runs a full multi-epoch training loop with a live rich progress display. + Delegates per-batch execution to the class's `train_step()` method. + """ + _fit_with_progress(self, X, Y, epochs, steps, initial_lr, decay_rate, shuffle) + return self + + diff --git a/pyproject.toml b/pyproject.toml index 8fe2f47..5bf206f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,89 @@ [build-system] -requires = ["setuptools>=42", "wheel"] -build-backend = "setuptools.build_meta" +requires = [ + "scikit-build-core>=0.10", + "pybind11>=2.13.0", + "cmake>=3.21", + "ninja", +] +build-backend = "scikit_build_core.build" + +[project] +name = "pydeepity" +version = "1.0.0" +description = "A high-performance Predictive Coding library." +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "numpy", + "rich", +] + +# ══════════════════════════════════════════════════════════════════════════ +# scikit-build-core: how `pip install .` / `pip wheel .` invokes CMake +# ══════════════════════════════════════════════════════════════════════════ +[tool.scikit-build] +minimum-version = "build-system.requires" +build-dir = "build/{wheel_tag}" + +# Pure-Python sources under pydeepity/ (__init__.py, utils.py, etc.) get +# merged with whatever CMake's install() step places under pydeepity/ (the +# compiled pydeepity*.so) into a single wheel. +wheel.packages = ["pydeepity"] + +sdist.exclude = [ + "tests", + "experiments", + "logs", + "build", + ".github", +] + +[tool.scikit-build.cmake.define] +# every flag here is chosen for maximum portability, not maximum speed. +# See CMakeLists.txt's DEEPITY_ARCH_FLAGS/DEEPITY_MSVC_ARCH_FLAGS comments, +# and build.py's --distributed profile, which mirrors this exact config for +# local testing (`python build.py Release --distributed --no-cuda`). +DEEPITY_BUILD_TESTS = "OFF" +DEEPITY_BUILD_PYTHON_BINDINGS = "ON" +DEEPITY_ENABLE_CUDA = "OFF" +DEEPITY_ARCH_FLAGS = "-march=x86-64-v2 -mtune=generic" +DEEPITY_MSVC_ARCH_FLAGS = "" + +# ══════════════════════════════════════════════════════════════════════════ +# cibuildwheel: which wheels get built, and what each platform needs +# installed before CMake can configure successfully. +# ══════════════════════════════════════════════════════════════════════════ +[tool.cibuildwheel] +build = "cp39-* cp310-* cp311-* cp312-* cp313-*" +skip = "*-musllinux* *-win32 *-manylinux_i686" +build-verbosity = 1 + +test-requires = ["numpy"] +test-command = "python -c \"import pydeepity; print('pydeepity OK:', pydeepity.__file__)\"" + +[tool.cibuildwheel.linux] +# manylinux2014's stock GCC is too old for -std=c++20; the _2_28 images +# (AlmaLinux 8 based) ship GCC 12+, which is required here. +manylinux-x86_64-image = "manylinux_2_28" +before-all = "dnf install -y openblas-devel libgomp" + +[tool.cibuildwheel.macos] +# NOT currently exercised by CI (see ci.yml's build-wheels matrix comment): +# macos-latest runners are arm64, and CMakeLists.txt's SIMD flags are +# x86-only with no arch-conditional branching yet, so a wheel build here +# fails before it gets anywhere near this config actually mattering. Left +# in place as the starting point for whoever picks that fix up. +before-all = "brew install openblas libomp" +# find_package(BLAS) / OpenMP need an explicit nudge on macOS since Homebrew +# installs outside the paths Apple's toolchain searches by default. +environment = { CMAKE_PREFIX_PATH = "/usr/local/opt/openblas:/usr/local/opt/libomp:/opt/homebrew/opt/openblas:/opt/homebrew/opt/libomp" } +# x86_64 only for now: DEEPITY_ARCH_FLAGS above is an x86 -march string, and +# CMakeLists.txt's SLEEF config force-enables AVX2/AVX512F unconditionally - +# neither means anything when cross-compiling for arm64. Building real +# arm64/Apple Silicon wheels needs CMakeLists.txt itself to branch SIMD +# flags on target architecture, not just this file. Tracked as follow-up. +archs = ["x86_64"] + +[tool.cibuildwheel.windows] +before-build = "pip install delvewheel" +repair-wheel-command = "delvewheel repair --add-path pydeepity -w {dest_dir} {wheel}" \ No newline at end of file diff --git a/setup.py b/setup.py index 42c6fbd..89992f3 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,41 @@ -from setuptools import setup, find_packages +import os +import subprocess +import sys +from setuptools import setup, Extension, find_packages +from setuptools.command.build_ext import build_ext + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=""): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + +class CMakeBuild(build_ext): + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + if not extdir.endswith(os.path.sep): + extdir += os.path.sep + + os.makedirs(self.build_temp, exist_ok=True) + + # Tell CMake to output the compiled pybind11 library directly to the extdir + cmake_args = [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + "-DCMAKE_BUILD_TYPE=Release" + ] + + # Build using all available CPU cores + build_args = ["--config", "Release", "-j", str(os.cpu_count())] + + # Run CMake configure and build + subprocess.check_call(["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp) + subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp) setup( name="pydeepity", version="1.0.0", - author="Ra4ster", - description="A high-performance Predictive Coding Network C++ engine", - packages=find_packages(), - - package_data={ - "pydeepity": ["*.so", "*.pyd", "*.dylib"], - }, - include_package_data=True, - python_requires=">=3.8", -) + packages=find_packages(include=["pydeepity", "pydeepity.*"]), + ext_modules=[CMakeExtension("pydeepity.pydeepity")], + cmdclass={"build_ext": CMakeBuild}, + zip_safe=False, +) \ No newline at end of file diff --git a/src/ConvPCNetwork.cpp b/src/ConvPCNetwork.cpp index 117956b..509db78 100644 --- a/src/ConvPCNetwork.cpp +++ b/src/ConvPCNetwork.cpp @@ -4,12 +4,6 @@ namespace Deep { ConvPCNetwork::ConvPCNetwork(int batchSize) noexcept : batchSize(batchSize) {} - ConvPCNetwork::~ConvPCNetwork() - { - for (auto *l : layers) - delete l; - } - void ConvPCNetwork::AddLayer(int inChannels, int outChannels, int inHeight, int inWidth, int kernelH, int kernelW, @@ -18,39 +12,39 @@ namespace Deep float lr, float ir, float pr, float lmbda, ActivationType aType, ActivationType dType) { - ConvPCLayer *l = new ConvPCLayer( + auto l = std::make_unique( inChannels, outChannels, inHeight, inWidth, kernelH, kernelW, strideH, strideW, padH, padW, batchSize, lr, ir, pr, lmbda, aType, dType); if (!layers.empty()) { - layers.back()->SetLayerAbove(l); - l->SetLayerBelow(layers.back()); + layers.back()->SetLayerAbove(l.get()); + l->SetLayerBelow(layers.back().get()); } - layers.push_back(l); + layers.push_back(std::move(l)); } void ConvPCNetwork::Compile() { size_t total = 0; - for (auto *l : layers) + for (auto &l : layers) total += l->GetRequiredFloats(); arena = std::make_unique(total); - for (auto *l : layers) + for (auto &l : layers) l->BindMemory(*arena); } void ConvPCNetwork::RandomizeWeights(std::mt19937 &rng) noexcept { - for (auto *l : layers) + for (auto &l : layers) l->RandomizeWeights(rng); } void ConvPCNetwork::ResetState() noexcept { - for (auto *l : layers) + for (auto &l : layers) l->ResetState(); } @@ -62,14 +56,14 @@ namespace Deep float ConvPCNetwork::CalculateState() noexcept { float e = 0.0f; - for (auto *l : layers) + for (auto &l : layers) e += l->CalculateState(); return e; } void ConvPCNetwork::UpdateState() noexcept { - for (auto *l : layers) + for (auto &l : layers) l->UpdateState(); } diff --git a/src/DiscriminativePCNetwork.cpp b/src/DiscriminativePCNetwork.cpp index 56921f1..4d75cf6 100644 --- a/src/DiscriminativePCNetwork.cpp +++ b/src/DiscriminativePCNetwork.cpp @@ -7,14 +7,6 @@ namespace Deep { - DiscriminativePCNetwork::~DiscriminativePCNetwork() - { - for (auto l : layers) - delete l; - - layers.clear(); - } - void DiscriminativePCNetwork::AddLayer(int size, int nextSize, float lr, float ir, float pr, float lmbda, void (*act)(float *, size_t), void (*dAct)(float *, size_t, bool)) { @@ -25,15 +17,14 @@ namespace Deep DynamicThread(batchSize); } - // Pass raw pointers directly to Constructor 2 - DiscriminativePCLayer *l = new DiscriminativePCLayer(size, nextSize, batchSize, lr, ir, pr, lmbda, act, dAct); + auto l = std::make_unique(size, nextSize, batchSize, lr, ir, pr, lmbda, act, dAct); if (!layers.empty()) { - layers.back()->SetLayerAbove(l); - l->SetLayerBelow(layers.back()); + layers.back()->SetLayerAbove(l.get()); + l->SetLayerBelow(layers.back().get()); } - layers.push_back(l); + layers.push_back(std::move(l)); } void DiscriminativePCNetwork::AddLayer(int size, int nextSize, float lr, float ir, float pr, float lmbda, @@ -47,25 +38,25 @@ namespace Deep } // Pass enums directly to Constructor 1 - DiscriminativePCLayer *l = new DiscriminativePCLayer(size, nextSize, batchSize, lr, ir, pr, lmbda, aType, dType); + auto l = std::make_unique(size, nextSize, batchSize, lr, ir, pr, lmbda, aType, dType); if (!layers.empty()) { - layers.back()->SetLayerAbove(l); - l->SetLayerBelow(layers.back()); + layers.back()->SetLayerAbove(l.get()); + l->SetLayerBelow(layers.back().get()); } - layers.push_back(l); + layers.push_back(std::move(l)); } void DiscriminativePCNetwork::RandomizeWeights(std::mt19937 &rng) { - for (auto l : layers) + for (auto &l : layers) l->RandomizeWeights(rng); } void DiscriminativePCNetwork::ResetState() noexcept { - for (auto l : layers) + for (auto &l : layers) l->ResetState(); } @@ -84,7 +75,7 @@ namespace Deep void DiscriminativePCNetwork::UpdateState() { - for (auto l : layers) + for (auto &l : layers) l->UpdateState(); } @@ -159,20 +150,13 @@ namespace Deep { size_t total_floats_needed = 0; - // 1. Calculate the exact footprint of the entire network - for (auto *layer : layers) - { + for (auto &layer : layers) total_floats_needed += layer->GetRequiredFloats(); - } - // 2. Allocate the single contiguous block of memory arena = std::make_unique(total_floats_needed); - // 3. Bind every layer sequentially into the arena - for (auto *layer : layers) - { + for (auto &layer : layers) layer->BindMemory(*arena); - } } bool DiscriminativePCNetwork::Save(const std::string &filename) const noexcept diff --git a/src/ModelIO.cpp b/src/ModelIO.cpp index 97e8ddd..c16f7d1 100644 --- a/src/ModelIO.cpp +++ b/src/ModelIO.cpp @@ -52,7 +52,7 @@ namespace Deep std::vector writeBuffer(BUFFER_SIZE); wStream.rdbuf()->pubsetbuf(writeBuffer.data(), BUFFER_SIZE); - for (const auto *layer : layers) + for (const auto &layer : layers) { size_t inputSize = layer->GetInputSize(); size_t outputSize = layer->GetOutputSize(); @@ -91,7 +91,7 @@ namespace Deep for (size_t i = 0; i < layers.size(); ++i) { - const auto *layer = layers[i]; + const auto &layer = layers[i]; mStream << " {\n"; mStream << " \"index\": " << i << ",\n"; mStream << " \"input_size\": " << layer->GetInputSize() << ",\n"; @@ -129,7 +129,7 @@ namespace Deep for (size_t i = 0; i < layers.size(); ++i) { - const auto *layer = layers[i]; + const auto &layer = layers[i]; rStream << "| " << i << " | " << layer->GetInputSize() << " | " << layer->GetOutputSize() << " | " @@ -160,7 +160,7 @@ namespace Deep std::vector readBuffer(BUFFER_SIZE); wStream.rdbuf()->pubsetbuf(readBuffer.data(), BUFFER_SIZE); - for (auto *layer : net.GetLayers()) + for (auto &layer : net.GetLayers()) { size_t inputSize = layer->GetInputSize(); size_t outputSize = layer->GetOutputSize(); diff --git a/src/SimpleConvPCNetwork.cpp b/src/SimpleConvPCNetwork.cpp index 58dfd91..4364f65 100644 --- a/src/SimpleConvPCNetwork.cpp +++ b/src/SimpleConvPCNetwork.cpp @@ -4,12 +4,6 @@ namespace Deep { SimpleConvPCNetwork::SimpleConvPCNetwork(int batchSize) noexcept : batchSize(batchSize) {} - SimpleConvPCNetwork::~SimpleConvPCNetwork() - { - for (auto *l : layers) - delete l; - } - void SimpleConvPCNetwork::AddLayer(int inChannels, int outChannels, int inHeight, int inWidth, int kernelH, int kernelW, @@ -18,17 +12,17 @@ namespace Deep float lr, float ir, float lmbda, ActivationType aType, ActivationType dType) { - SimpleConvPCLayer *l = new SimpleConvPCLayer( + auto l = std::make_unique( inChannels, outChannels, inHeight, inWidth, kernelH, kernelW, strideH, strideW, padH, padW, batchSize, lr, ir, lmbda, aType, dType); if (!layers.empty()) { - layers.back()->SetLayerAbove(l); - l->SetLayerBelow(layers.back()); + layers.back()->SetLayerAbove(l.get()); + l->SetLayerBelow(layers.back().get()); } - layers.push_back(l); + layers.push_back(std::move(l)); } void SimpleConvPCNetwork::SetOptimizer(OptimizerType opt) noexcept @@ -43,27 +37,27 @@ namespace Deep void SimpleConvPCNetwork::Compile() { - for (auto *l : layers) + for (auto &l : layers) l->SetOptimizer(pendingOpt); size_t total = 0; - for (auto *l : layers) + for (auto &l : layers) total += l->GetRequiredFloats(); arena = std::make_unique(total); - for (auto *l : layers) + for (auto &l : layers) l->BindMemory(*arena); } void SimpleConvPCNetwork::RandomizeWeights(std::mt19937 &rng) noexcept { - for (auto *l : layers) + for (auto &l : layers) l->RandomizeWeights(rng); } void SimpleConvPCNetwork::ResetState() noexcept { - for (auto *l : layers) + for (auto &l : layers) l->ResetState(); } @@ -75,14 +69,14 @@ namespace Deep float SimpleConvPCNetwork::CalculateState() noexcept { float e = 0.0f; - for (auto *l : layers) + for (auto &l : layers) e += l->CalculateState(); return e; } void SimpleConvPCNetwork::UpdateState() noexcept { - for (auto *l : layers) + for (auto &l : layers) l->UpdateState(); } diff --git a/src/SimplePCLayer.cpp b/src/SimplePCLayer.cpp index 5596ef6..8ff97b7 100644 --- a/src/SimplePCLayer.cpp +++ b/src/SimplePCLayer.cpp @@ -53,7 +53,7 @@ namespace Deep for (auto &s : seeds) s = seedDist(seedGenerator); -#pragma omp parallel +#pragma omp parallel if(!omp_in_parallel()) { std::mt19937 rng(seeds[omp_get_thread_num()]); std::normal_distribution dist(0.0f, limit); @@ -78,12 +78,9 @@ namespace Deep cblas_saxpy(N, -1.0f, layerBelow->mu, 1, e, 1); } - // No precision weighting: E = 0.5 * sum(e^2), plain SSE. No p - // multiply, no -0.5*log(p) term -- both were exactly inert at - // pr=0.0, which was the only value ever actually used. float totalEnergy = 0.0f; -#pragma omp parallel for schedule(static) reduction(+ : totalEnergy) +#pragma omp parallel for schedule(static) reduction(+ : totalEnergy) if(batchSize > 4 && !omp_in_parallel()) for (int batch = 0; batch < batchSize; ++batch) { const size_t offset = (size_t)batch * size; @@ -143,32 +140,10 @@ namespace Deep size_t Nout = (size_t)batchSize * nextSize; size_t N = (size_t)batchSize * size; - // Mu-caching: recompute only when z has moved enough SINCE THE - // LAST RECOMPUTE (not just the immediately preceding step -- - // comparing against prevZ, updated only on real recomputes, - // catches cumulative drift across several skipped steps that a - // step-to-step comparison would miss). - // - // threshold=0 (or a CLAMPED layer, whose z never changes at - // all) makes the ratio always exactly 0 -- reproducing the - // original, EXACT, validated behavior. threshold>0 extends - // this to UNCLAMPED layers too, as a genuine APPROXIMATION -- - // correctness there means "close enough for real training," - // not "bit-identical," and needs accuracy-impact validation, - // not just a single-batch trajectory diff. - // - // mu itself is NOT a stable buffer between steps -- UpdateState() - // mutates it in place (converts it to its own derivative, for - // the feedback GEMM) -- so a skip must copy a PRESERVED value - // back into mu, not just skip writing to mu. bool shouldRecompute = true; if (muCacheThreshold >= 0.0f && muCacheValid) { float zNorm = cblas_snrm2((int)N, prevZ, 1); - // Reuse dz_dt as scratch -- safe: UpdateState() overwrites - // it fresh every step and nothing reads it between here - // and that call, avoiding a per-call heap allocation on - // this hot path. cblas_scopy((int)N, z, 1, dz_dt, 1); cblas_saxpy((int)N, -1.0f, prevZ, 1, dz_dt, 1); float deltaNorm = cblas_snrm2((int)N, dz_dt, 1); @@ -197,9 +172,6 @@ namespace Deep } else { - // Restore the preserved, correctly-activated prediction -- - // mu was left holding last step's DERIVATIVE by - // UpdateState(), not the prediction itself. cblas_scopy((int)Nout, cachedMu, 1, mu, 1); } } @@ -220,9 +192,7 @@ namespace Deep if (isClamped) return; - // Own term: dz_dt = -e (was -p*e; p=1 always made this a no-op - // multiply, so the multiplication itself is simply removed). -#pragma omp parallel for schedule(static) +#pragma omp parallel for schedule(static) if(batchSize > 4 && !omp_in_parallel()) for (int batch = 0; batch < batchSize; ++batch) { size_t offset = (size_t)batch * size; @@ -263,9 +233,7 @@ namespace Deep { const float *e_above = layerAbove->GetErrors(); - // bottom_up = e_above * mu(f') -- was e_above * p_above * mu(f'); - // p_above=1 always made that multiply a no-op, removed. -#pragma omp parallel for schedule(static) +#pragma omp parallel for schedule(static) if(batchSize > 4 && !omp_in_parallel()) for (int batch = 0; batch < batchSize; ++batch) { size_t offset = (size_t)batch * nextSize; @@ -320,8 +288,7 @@ namespace Deep const float *e_above = layerAbove->GetErrors(); float *local_grad = bottom_up; - // 1. Compute the local gradient delta (shared across all optimizers) -#pragma omp parallel for schedule(static) +#pragma omp parallel for schedule(static) if(batchSize > 4 && !omp_in_parallel()) for (int batch = 0; batch < batchSize; ++batch) { size_t offset = (size_t)batch * nextSize; @@ -358,7 +325,6 @@ namespace Deep local_grad[offset + f] = e_above[offset + f] * mu[offset + f]; } - // 2. Dispatch to the selected optimizer switch (opt) { case OptimizerType::SGD: @@ -366,7 +332,6 @@ namespace Deep if (lmbda > 0.0f) cblas_sscal((size_t)nextSize * size, 1.0f - lmbda, W, 1); - // Writes scaled updates directly into W via beta=1.0f cblas_sgemm( CblasRowMajor, CblasTrans, CblasNoTrans, nextSize, size, batchSize, @@ -382,29 +347,21 @@ namespace Deep case OptimizerType::ADAM: case OptimizerType::ADAMW: { - t++; // Increment layer-wide timestep ONCE + t++; size_t num_weights = (size_t)nextSize * size; - - // NOTE: Using 1.0f / batchSize instead of 1.0f so the gradient - // magnitude doesn't drastically shift Adam's variance estimates - // if you change your batch size. float grad_scale = -1.0f / batchSize; - // Compute raw gradients for W into grad_W - // beta=0.0f overwrites grad_W cleanly, no memset needed cblas_sgemm( CblasRowMajor, CblasTrans, CblasNoTrans, nextSize, size, batchSize, grad_scale, local_grad, nextSize, z, size, 0.0f, grad_W, size); - // Compute raw gradients for biases into grad_b std::memset(grad_b, 0, nextSize * sizeof(float)); for (int batch = 0; batch < batchSize; batch++) cblas_saxpy(nextSize, grad_scale, local_grad + batch * nextSize, 1, grad_b, 1); - // Apply specific Adam flavor to Weights if (opt == OptimizerType::ADAMW) { Deep::AdamWUpdate(W, grad_W, m_W, v_W, num_weights, t, lr, lmbda); @@ -414,9 +371,7 @@ namespace Deep Deep::AdamUpdate(W, grad_W, m_W, v_W, num_weights, t, lr); } - // Apply plain Adam to Biases (biases almost never use weight decay) Deep::AdamUpdate(b, grad_b, m_b, v_b, nextSize, t, lr); - break; } } @@ -433,7 +388,7 @@ namespace Deep size_t copySize = std::min(inputData.size(), (size_t)(batchSize * size)) * sizeof(float); memcpy(z, inputData.data(), copySize); isClamped = true; - muCacheValid = false; // fresh data this batch -- must recompute at least once + muCacheValid = false; } void SimplePCLayer::UnclampState() noexcept @@ -449,7 +404,6 @@ namespace Deep size_t total = 0; size_t own_state_size = (size_t)batchSize * size; - // z, e, dz_dt -- NO p/log_p buffers at all total += pad16(own_state_size) * 3; if (nextSize > 0) @@ -457,16 +411,15 @@ namespace Deep size_t out_state_size = (size_t)batchSize * nextSize; size_t w_size = (size_t)size * nextSize; - total += pad16(w_size); // W - total += pad16(nextSize); // b - total += pad16(out_state_size) * 3; // mu, cachedMu, bottom_up - total += pad16(own_state_size); // prevZ (own_state_size, matches z) + total += pad16(w_size); + total += pad16(nextSize); + total += pad16(out_state_size) * 3; + total += pad16(own_state_size); - // Conditionally allocate Adam variables to save space for SGD users if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { - total += pad16(w_size) * 3; // grad_W, m_W, v_W - total += pad16(nextSize) * 3; // grad_b, m_b, v_b + total += pad16(w_size) * 3; + total += pad16(nextSize) * 3; } } @@ -503,7 +456,6 @@ namespace Deep std::memset(bottom_up, 0, out_state_size * sizeof(float)); std::memset(prevZ, 0, own_state_size * sizeof(float)); - // Conditionally bind Adam variables if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { grad_W = arena.AllocateFloats(w_size); @@ -514,13 +466,11 @@ namespace Deep m_b = arena.AllocateFloats(nextSize); v_b = arena.AllocateFloats(nextSize); - // Adam moments must begin at zero std::memset(m_W, 0, w_size * sizeof(float)); std::memset(v_W, 0, w_size * sizeof(float)); std::memset(m_b, 0, nextSize * sizeof(float)); std::memset(v_b, 0, nextSize * sizeof(float)); - // It's good practice to zero the gradients as well std::memset(grad_W, 0, w_size * sizeof(float)); std::memset(grad_b, 0, nextSize * sizeof(float)); } @@ -531,4 +481,4 @@ namespace Deep localArena.reset(); } } -} \ No newline at end of file +} diff --git a/src/SimplePCNetwork.cpp b/src/SimplePCNetwork.cpp index c7e2484..9fee9f1 100644 --- a/src/SimplePCNetwork.cpp +++ b/src/SimplePCNetwork.cpp @@ -4,48 +4,41 @@ namespace Deep { SimplePCNetwork::SimplePCNetwork(int batchSize) noexcept : batchSize(batchSize) {} - SimplePCNetwork::~SimplePCNetwork() - { - for (auto l : layers) - delete l; - layers.clear(); - } - void SimplePCNetwork::AddLayer(int size, int nextSize, float lr, float ir, float lmbda, void (*act)(float *, size_t), void (*dAct)(float *, size_t, bool)) { - SimplePCLayer *l = new SimplePCLayer(size, nextSize, batchSize, lr, ir, lmbda, act, dAct); + std::unique_ptr l = std::make_unique(size, nextSize, batchSize, lr, ir, lmbda, act, dAct); if (!layers.empty()) { - layers.back()->SetLayerAbove(l); - l->SetLayerBelow(layers.back()); + layers.back()->SetLayerAbove(l.get()); + l->SetLayerBelow(layers.back().get()); } - layers.push_back(l); + layers.push_back(std::move(l)); } void SimplePCNetwork::AddLayer(int size, int nextSize, float lr, float ir, float lmbda, ActivationType aType, ActivationType dType) { - SimplePCLayer *l = new SimplePCLayer(size, nextSize, batchSize, lr, ir, lmbda, aType, dType); + std::unique_ptr l = std::make_unique(size, nextSize, batchSize, lr, ir, lmbda, aType, dType); if (!layers.empty()) { - layers.back()->SetLayerAbove(l); - l->SetLayerBelow(layers.back()); + layers.back()->SetLayerAbove(l.get()); + l->SetLayerBelow(layers.back().get()); } - layers.push_back(l); + layers.push_back(std::move(l)); } void SimplePCNetwork::RandomizeWeights(std::mt19937 &rng) { - for (auto l : layers) + for (auto &l : layers) l->RandomizeWeights(rng); } void SimplePCNetwork::ResetState() noexcept { - for (auto l : layers) + for (auto &l : layers) l->ResetState(); } @@ -64,7 +57,7 @@ namespace Deep void SimplePCNetwork::UpdateState() { - for (auto l : layers) + for (auto &l : layers) l->UpdateState(); } @@ -127,14 +120,40 @@ namespace Deep } } + float SimplePCNetwork::TrainStepWithProjection(const std::vector &x, const std::vector &y, int inferenceSteps) + { + ResetState(); + Clamp(x); + ProjectForward(); + GetTerminalLayer()->ClampState(y); + + float finalEnergy = 0.0f; + for (int t = 0; t < inferenceSteps; ++t) + { + finalEnergy = CalculateState(); + UpdateState(); + } + + UpdateWeights(); + GetTerminalLayer()->UnclampState(); + + return finalEnergy; + } + + void SimplePCNetwork::SetMuCacheThreshold(float threshold) noexcept + { + for (auto &l : layers) + l->SetMuCacheThreshold(threshold); + } + void SimplePCNetwork::Compile() { size_t total_floats_needed = 0; - for (auto *layer : layers) + for (auto &layer : layers) total_floats_needed += layer->GetRequiredFloats(); arena = std::make_unique(total_floats_needed); - for (auto *layer : layers) + for (auto &layer : layers) layer->BindMemory(*arena); } } diff --git a/temp.py b/temp.py index 769909c..acbf087 100644 --- a/temp.py +++ b/temp.py @@ -4,27 +4,32 @@ from pydeepity import SimplePCN from time import perf_counter -# Tests forward-projection initialization -- the real, structural finding -# from reading ngc-learn's actual pcn_model.py source: EVERY batch, they -# run a pure feedforward pass through CURRENT weights first, and seed the -# settling loop's hidden states from THAT, not from zero. This is -# fundamentally different from (and likely much stronger than) I_avg's -# stale class-average caching -- fresh, per-example, current-weights, -# every single batch. +# FINAL VALIDATION -- the complete stack found today: +# - Forward-projection initialization (seeds hidden layers from a real +# forward pass through current weights, not zero-init) +# - +-0.3 uniform weight init (matching ngc-learn's actual convention) +# - AdamW, lr=0.001 (matching ngc-learn's hard-coded Adam + eta=0.001 -- +# the init range and optimizer needed to be changed TOGETHER, not +# independently; tested and confirmed: 73.31% alone with SGD vs +# 93.96% paired with AdamW) +# - mu_cache_threshold=0.05 -- confirmed +30.6% faster than ngc-learn's +# real, measured per-batch time in an isolated speed test. THIS RUN +# checks whether that speed holds without costing the 93.96% accuracy +# already confirmed at threshold=disabled. +# - train_step_with_projection() -- single C++ call per batch, not a +# manual Python loop (confirmed real, if smaller, speedup from +# eliminating ~40 Python/pybind boundary crossings per batch) # -# Matches ngc-learn's real settings as closely as possible now that we -# have the actual source: [0,1] normalization, labels clipped to -# [0.001, 0.999] (not raw one-hot), tanh activation (confirmed as their -# actual default), 784->512->512->10, T=20, plain SGD (isolating THIS -# specific change -- forward-projection init -- from the separate, -# already-known Adam question). +# [0,1] normalization, labels clipped to [0.001, 0.999], tanh activation, +# 784->512->512->10, T=20 -- all matching ngc-learn's actual, real source +# as closely as possible. def load_full_mnist(): print("Fetching full MNIST dataset (70,000 images)...") X, y = fetch_openml('mnist_784', version=1, return_X_y=True, as_frame=False, parser='auto') X = X.astype(np.float32) / 255.0 # [0,1], matching ngc-learn - y = y.astype(int) + y = y.astype(int) # type: ignore # Clipped one-hot, matching ngc-learn's `jnp.clip(lab, eps, 1-eps)` eps = 0.001 @@ -73,6 +78,18 @@ def main() -> None: # without the other may have been the mistake. net.compile() net.randomize_weights() + net.set_mu_cache_threshold(0.0) # REVERTED from 0.05 -- that showed genuine + # instability under real, sustained AdamW + # training (energy trending UP, a 62-point + # accuracy crash at epoch 8), not just noise. + # Likely cause: small caching-approximation + # bias accumulating in Adam's momentum/ + # variance buffers over many updates -- + # something the earlier single-trajectory, + # no-weight-update speed test structurally + # could not have caught. threshold=0 is + # exact (proven bit-identical to no caching) + # and confirmed safe with this exact stack. # Override weight init to match ngc-learn's actual convention: fixed # uniform range +-0.3, INDEPENDENT of layer size -- not our own @@ -112,23 +129,7 @@ def main() -> None: X_batch = X_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] Y_batch = Y_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] - # Manual loop instead of train_step() -- need ProjectForward() - # inserted between clamp_input() and the settling loop, which - # the existing train_step() convenience method doesn't do. - net.reset_state() - net.clamp_input(X_batch) - net.project_forward() # THE new step -- seeds hidden layers - # from a genuine forward pass, not zero - net[-1].clamp_state(Y_batch) - - energy = 0.0 - for _ in range(STEPS): - energy += net.calculate_state() - net.update_state() - - net.update_weights() - net[-1].unclamp_state() - + energy = net.train_step_with_projection(X_batch, Y_batch, STEPS) epoch_energy += energy # REAL accuracy check -- genuine UNCLAMPED settle on a cheap diff --git a/tests/t513.cpp b/tests/t513.cpp new file mode 100644 index 0000000..d26f77a --- /dev/null +++ b/tests/t513.cpp @@ -0,0 +1,90 @@ +// Confirms the cache-aliasing theory directly: SAME isolated-layer test +// as tLayer1Isolate.cpp, but at size=513 instead of 512 -- a single +// dimension off from a power of two. If the ~1.5ms/call slowness was +// genuinely caused by 512's power-of-two stride aliasing in the CPU +// cache, 513 should show DRAMATICALLY better throughput despite being +// virtually the same amount of actual work (~0.4% more FLOPs). +#include +#include +#include +#include +#include + +using namespace Deep; + +int main() +{ + const int BATCH = 256; + const int STEPS = 20; + const int N_REPS = 40; + const int SIZE = 513; // ONLY difference from tLayer1Isolate.cpp: 512 -> 513 + + SimplePCLayer below(SIZE, SIZE, BATCH, 0.001f, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + SimplePCLayer layer1(SIZE, SIZE, BATCH, 0.001f, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + SimplePCLayer above(SIZE, 10, BATCH, 0.001f, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + + below.SetLayerAbove(&layer1); + layer1.SetLayerBelow(&below); + layer1.SetLayerAbove(&above); + above.SetLayerBelow(&layer1); + + std::mt19937 rng(7); + below.RandomizeWeights(rng); + layer1.RandomizeWeights(rng); + above.RandomizeWeights(rng); + + std::uniform_real_distribution initDist(-0.3f, 0.3f); + auto reinit = [&](SimplePCLayer &l, size_t inSz, size_t outSz) + { + if (outSz == 0) return; + float *W = l.GetWeights(); + for (size_t i = 0; i < inSz * outSz; ++i) W[i] = initDist(rng); + }; + reinit(below, SIZE, SIZE); + reinit(layer1, SIZE, SIZE); + reinit(above, SIZE, 10); + + std::vector belowInput((size_t)BATCH * SIZE); + std::mt19937 dataRng(123); + std::uniform_real_distribution dataDist(-1.0f, 1.0f); + for (auto &v : belowInput) v = dataDist(dataRng); + + below.ClampState(belowInput); + + double t_calc = 0, t_update = 0; + auto totalStart = std::chrono::steady_clock::now(); + + for (int rep = 0; rep < N_REPS; ++rep) + { + below.CalculateState(); + + for (int step = 0; step < STEPS; ++step) + { + auto t0 = std::chrono::steady_clock::now(); + layer1.CalculateState(); + auto t1 = std::chrono::steady_clock::now(); + t_calc += std::chrono::duration(t1 - t0).count(); + + t0 = t1; + layer1.UpdateState(); + t1 = std::chrono::steady_clock::now(); + t_update += std::chrono::duration(t1 - t0).count(); + } + } + + auto totalEnd = std::chrono::steady_clock::now(); + double totalTime = std::chrono::duration(totalEnd - totalStart).count(); + + std::cout << "=== Isolated " << SIZE << "->" << SIZE << " layer (non-power-of-2), " + << N_REPS << " reps x " << STEPS << " steps ===\n\n"; + std::cout << "Total: " << totalTime << "s\n"; + std::cout << "calculate_state: " << 1000*t_calc/(N_REPS*STEPS) << " ms/call\n"; + std::cout << "update_state: " << 1000*t_update/(N_REPS*STEPS) << " ms/call\n\n"; + std::cout << "Compare against size=512's measured: calc=1.670ms, update=1.487ms\n"; + std::cout << "If 513 is DRAMATICALLY faster despite ~0.4% MORE actual work,\n"; + std::cout << "that confirms cache-associativity aliasing at the 512 power-of-2\n"; + std::cout << "stride as the real cause -- a well-known, well-understood HPC\n"; + std::cout << "pathology with a simple, standard fix (pad the leading dimension).\n"; + + return 0; +} diff --git a/tests/tProfile.cpp b/tests/tProfile.cpp index 6687e9d..637593f 100644 --- a/tests/tProfile.cpp +++ b/tests/tProfile.cpp @@ -1,108 +1,95 @@ -// tProfile.cpp -- SIMPLIFIED for this investigation. -// -// The original per-phase instrumentation (PCN_TIME() calls inside -// DiscriminativePCLayer.cpp) was never carried forward through today's -// restructuring -- the registry in Profile.h is empty, confirmed by the -// missing per-layer sections in the last run's output. Rather than -// re-instrument the whole library again, this measures the two coarse -// phases that actually matter for "where is the Windows slowdown coming -// from" directly at the call site: the settling loop (CalculateState + -// UpdateState, repeated INFERENCE_STEPS times) vs UpdateWeights(). No -// changes to any library source file needed -- just std::chrono around -// the existing public methods. -#include -#include +// Isolates ONE 512->512 SimplePCLayer, standalone, matching Layer 1's +// real situation from the granular profile (fed by a 512-wide "layer +// below", feeding into a 10-wide "layer above") but stripped of +// everything else in the network -- to find exactly what's +// pathologically slow about this specific shape. 30.82ms calc / 27.06ms +// update measured in the real network is ~5x more than its FLOP count +// vs Layer 0 would predict. +#include #include +#include +#include #include -#include using namespace Deep; -int main(void) +int main() { - std::cout << "OpenBLAS selected CPU kernel: " << openblas_get_corename() << "\n\n"; - const int BATCH_SIZE = 256; - const int INFERENCE_STEPS = 150; - const int N_BATCHES = 5; - - std::mt19937 rng(42); - std::uniform_real_distribution dist(-1.0f, 1.0f); - - DiscriminativePCNetwork net(BATCH_SIZE); - net.AddLayer(784, 512, 0.002f, 0.05f, 0.0f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); - net.AddLayer(512, 10, 0.002f, 0.05f, 0.0f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); - net.AddLayer(10, 0, 0.002f, 0.05f, 0.0f, 0.0001f, ActivationType::LINEAR, ActivationType::dLINEAR); - net.Compile(); - net.RandomizeWeights(rng); - - std::vector X((size_t)BATCH_SIZE * 784); - std::vector Y((size_t)BATCH_SIZE * 10); - for (auto &v : X) v = dist(rng); - for (auto &v : Y) v = dist(rng); + const int BATCH = 256; + const int STEPS = 20; + const int N_REPS = 40; + + // Matches Layer 1 exactly: 512 in, 512 out, same activation, wired + // to a layer below (512-wide, standing in for Layer 0) and a layer + // above (10-wide, standing in for Layer 2). + SimplePCLayer below(512, 512, BATCH, 0.001f, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + SimplePCLayer layer1(512, 512, BATCH, 0.001f, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + SimplePCLayer above(512, 10, BATCH, 0.001f, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + + below.SetLayerAbove(&layer1); + layer1.SetLayerBelow(&below); + layer1.SetLayerAbove(&above); + above.SetLayerBelow(&layer1); + + std::mt19937 rng(7); + below.RandomizeWeights(rng); + layer1.RandomizeWeights(rng); + above.RandomizeWeights(rng); + + std::uniform_real_distribution initDist(-0.3f, 0.3f); + auto reinit = [&](SimplePCLayer &l, size_t inSz, size_t outSz) + { + if (outSz == 0) return; + float *W = l.GetWeights(); + for (size_t i = 0; i < inSz * outSz; ++i) W[i] = initDist(rng); + }; + reinit(below, 512, 512); + reinit(layer1, 512, 512); + reinit(above, 512, 10); - std::cout << "Profiling (coarse, call-site timing): " << N_BATCHES << " batches, batch_size=" - << BATCH_SIZE << ", inference_steps=" << INFERENCE_STEPS << "\n\n"; + std::vector belowInput((size_t)BATCH * 512); + std::mt19937 dataRng(123); + std::uniform_real_distribution dataDist(-1.0f, 1.0f); + for (auto &v : belowInput) v = dataDist(dataRng); - double totalSettle = 0.0; - double totalCalcState = 0.0; - double totalUpdateState = 0.0; - double totalUpdateWeights = 0.0; - double totalResetClamp = 0.0; + below.ClampState(belowInput); // "below" acts as a clamped input stand-in - auto wallStart = std::chrono::steady_clock::now(); + double t_calc = 0, t_update = 0; + auto totalStart = std::chrono::steady_clock::now(); - for (int b = 0; b < N_BATCHES; ++b) + for (int rep = 0; rep < N_REPS; ++rep) { - auto t0 = std::chrono::steady_clock::now(); - net.ResetState(); - net.Clamp(X); - net.GetTerminalLayer()->ClampState(Y); - auto t1 = std::chrono::steady_clock::now(); - totalResetClamp += std::chrono::duration(t1 - t0).count(); + below.CalculateState(); // seed below's own mu once, matching project_forward's role - for (int s = 0; s < INFERENCE_STEPS; ++s) + for (int step = 0; step < STEPS; ++step) { - auto cs0 = std::chrono::steady_clock::now(); - net.CalculateState(); - auto cs1 = std::chrono::steady_clock::now(); - totalCalcState += std::chrono::duration(cs1 - cs0).count(); - - net.UpdateState(); - auto us1 = std::chrono::steady_clock::now(); - totalUpdateState += std::chrono::duration(us1 - cs1).count(); + auto t0 = std::chrono::steady_clock::now(); + layer1.CalculateState(); + auto t1 = std::chrono::steady_clock::now(); + t_calc += std::chrono::duration(t1 - t0).count(); + + t0 = t1; + layer1.UpdateState(); + t1 = std::chrono::steady_clock::now(); + t_update += std::chrono::duration(t1 - t0).count(); } - - auto t2 = std::chrono::steady_clock::now(); - net.UpdateWeights(); - auto t3 = std::chrono::steady_clock::now(); - totalUpdateWeights += std::chrono::duration(t3 - t2).count(); - - net.GetTerminalLayer()->UnclampState(); - - std::cout << " batch " << (b + 1) << "/" << N_BATCHES << " done\n"; } - auto wallEnd = std::chrono::steady_clock::now(); - double wallSeconds = std::chrono::duration(wallEnd - wallStart).count(); - totalSettle = totalCalcState + totalUpdateState; - - std::cout << "\n=== Coarse phase breakdown (" << N_BATCHES << " batches, " - << INFERENCE_STEPS << " steps each) ===\n"; - std::cout << " Wall clock total: " << wallSeconds << " s\n"; - std::cout << " ResetState+Clamp: " << totalResetClamp << " s (" - << (100.0 * totalResetClamp / wallSeconds) << "%)\n"; - std::cout << " CalculateState (sum): " << totalCalcState << " s (" - << (100.0 * totalCalcState / wallSeconds) << "%)\n"; - std::cout << " UpdateState (sum): " << totalUpdateState << " s (" - << (100.0 * totalUpdateState / wallSeconds) << "%)\n"; - std::cout << " UpdateWeights (sum): " << totalUpdateWeights << " s (" - << (100.0 * totalUpdateWeights / wallSeconds) << "%)\n"; - std::cout << " --------\n"; - std::cout << " Settle loop total: " << totalSettle << " s (" - << (100.0 * totalSettle / wallSeconds) << "%)\n"; - std::cout << " Per-step avg (Calc+Upd): " << (totalSettle / (N_BATCHES * INFERENCE_STEPS)) * 1000.0 << " ms\n"; - std::cout << " Per-step CalculateState: " << (totalCalcState / (N_BATCHES * INFERENCE_STEPS)) * 1000.0 << " ms\n"; - std::cout << " Per-step UpdateState: " << (totalUpdateState / (N_BATCHES * INFERENCE_STEPS)) * 1000.0 << " ms\n"; + auto totalEnd = std::chrono::steady_clock::now(); + double totalTime = std::chrono::duration(totalEnd - totalStart).count(); + + std::cout << "=== Isolated 512->512 layer, " << N_REPS << " reps x " << STEPS << " steps ===\n\n"; + std::cout << "Total: " << totalTime << "s\n"; + std::cout << "calculate_state: " << 1000*t_calc/(N_REPS*STEPS) << " ms/call (avg per single call)\n"; + std::cout << "update_state: " << 1000*t_update/(N_REPS*STEPS) << " ms/call (avg per single call)\n\n"; + std::cout << "Compare against the real network's measured Layer 1 costs:\n"; + std::cout << " calc: 30.82ms/batch / 20 steps = 1.541 ms/call\n"; + std::cout << " update: 27.06ms/batch / 20 steps = 1.353 ms/call\n"; + std::cout << "\nIf THIS isolated test also shows ~1.5ms/call, the slowness is\n"; + std::cout << "intrinsic to a 512x512 SimplePCLayer regardless of context. If\n"; + std::cout << "it's much faster here, something about the FULL 4-layer network's\n"; + std::cout << "context (thread contention between layers, memory layout, etc)\n"; + std::cout << "is the real cause -- not the layer itself in isolation.\n"; return 0; } diff --git a/tests/tSimple.cpp b/tests/tSimple.cpp index 32f11f2..ed69753 100644 --- a/tests/tSimple.cpp +++ b/tests/tSimple.cpp @@ -1,103 +1,148 @@ -#include -#include -#include +// Granular profiling of the EXACT configuration currently under test: +// 784->512->512->10, ADAMW, mu_cache_threshold=0, forward-projection +// init, batch=256, steps=20. Isolates where the remaining ~12ms/batch +// gap vs ngc-learn (82ms measured vs their 70.31ms) actually lives -- +// project_forward specifically, settling loop (per-layer), weight +// updates, or fixed per-region overhead (thread spin-up, virtual +// dispatch) that doesn't show up as "real compute" anywhere. +#include #include -#include -#include -#include -#include -#include +#include +#include +#include -float WeightNorm(Deep::DiscriminativePCLayer *layer, size_t count) -{ - const float *w = layer->GetWeights(); - float sumSq = 0.0f; - for (size_t i = 0; i < count; ++i) - sumSq += w[i] * w[i]; - return std::sqrt(sumSq); -} +using namespace Deep; -int main(void) +int main() { - Deep::DiscriminativePCNetwork net(4); - Timer timer; - - net.AddLayer(2, 8, 0.05f, 0.3f, 0.000f, 0.0001f, Deep::tanh, Deep::dTanh); - net.AddLayer(8, 1, 0.05f, 0.3f, 0.000f, 0.0001f, Deep::tanh, Deep::dTanh); - net.AddLayer(1, 0, 0.05f, 0.3f, 0.000f, 0.0001f, Deep::linear, Deep::dLinear); - - std::mt19937 rng(42); + const int BATCH = 256; + const float LR = 0.001f; + const int STEPS = 20; + const int N_BATCHES = 40; // matches the earlier timing test's sample size + + SimplePCNetwork net(BATCH); + net.AddLayer(784, 512, LR, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + net.AddLayer(512, 512, LR, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + net.AddLayer(512, 10, LR, 0.08f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); + net.AddLayer(10, 0, LR, 0.08f, 0.0001f, ActivationType::LINEAR, ActivationType::dLINEAR); + net.SetOptimizer(OptimizerType::ADAMW); + net.Compile(); + + std::mt19937 rng(7); net.RandomizeWeights(rng); - auto &layers = net.GetLayers(); - auto *layer0 = static_cast(layers[0]); - auto *layer1 = static_cast(layers[1]); - float initialW0Norm = WeightNorm(layer0, 2 * 8); - - std::vector flatX = { - -1.0f, -1.0f, - -1.0f, +1.0f, - +1.0f, -1.0f, - +1.0f, +1.0f}; - std::vector flatY = { - -1.0f, - +1.0f, - +1.0f, - -1.0f}; - - int epochs = 5000; - int inferenceSteps = 50; - int reportEvery = 100; - - std::cout << "Starting Discriminative PC XOR Test (Clean API)...\n"; - - double start = timer.elapsed(); - for (int epoch = 0; epoch < epochs; epoch++) + // Match the real +-0.3 uniform init directly in C++ this time + std::uniform_real_distribution initDist(-0.3f, 0.3f); + for (auto *layer : net.GetLayers()) { - // One clean method to handle the entire training step - float energy = net.TrainStep(flatX, flatY, inferenceSteps); + if (layer->GetOutputSize() == 0) continue; // terminal has no weights + size_t wsz = layer->GetInputSize() * layer->GetOutputSize(); + float *W = layer->GetWeights(); + for (size_t i = 0; i < wsz; ++i) + W[i] = initDist(rng); + } - if (epoch % reportEvery == 0) - { - float w0norm = WeightNorm(layer0, 2 * 8); - float w1norm = WeightNorm(layer1, 8 * 1); + net.SetMuCacheThreshold(0.0f); + + std::mt19937 dataRng(123); + std::uniform_real_distribution dataDist(0.0f, 1.0f); + std::vector X((size_t)BATCH * 784); + std::vector Y((size_t)BATCH * 10, 0.001f); + for (auto &v : X) v = dataDist(dataRng); + for (int b = 0; b < BATCH; ++b) + Y[b * 10 + (b % 10)] = 0.999f; // fake one-hot-ish labels, real values don't matter for timing - std::cout << "Epoch " << std::setw(5) << epoch - << " | Energy: " << std::fixed << std::setprecision(4) << energy / 4.0f - << " | W0 norm: " << w0norm - << " | W1 norm: " << w1norm - << " | Elapsed: " << timer.elapsed() - start << "s\n"; - start = timer.elapsed(); + // Granular phase timers + double t_reset = 0, t_clamp_input = 0, t_project = 0, t_clamp_target = 0; + double t_calc[4] = {0, 0, 0, 0}; + double t_update[4] = {0, 0, 0, 0}; + double t_weights = 0, t_unclamp = 0; - if (w0norm > 10.0f * initialW0Norm && w0norm > 20.0f) + auto &layers = net.GetLayers(); + + auto totalStart = std::chrono::steady_clock::now(); + + for (int rep = 0; rep < N_BATCHES; ++rep) + { + auto t0 = std::chrono::steady_clock::now(); + net.ResetState(); + auto t1 = std::chrono::steady_clock::now(); + t_reset += std::chrono::duration(t1 - t0).count(); + + t0 = t1; + net.Clamp(X); + t1 = std::chrono::steady_clock::now(); + t_clamp_input += std::chrono::duration(t1 - t0).count(); + + t0 = t1; + net.ProjectForward(); + t1 = std::chrono::steady_clock::now(); + t_project += std::chrono::duration(t1 - t0).count(); + + t0 = t1; + layers.back()->ClampState(Y); + t1 = std::chrono::steady_clock::now(); + t_clamp_target += std::chrono::duration(t1 - t0).count(); + + for (int step = 0; step < STEPS; ++step) + { + for (size_t i = 0; i < layers.size(); ++i) { - std::cout << " *** WARNING: W0 norm jumped far beyond init scale ***\n"; + t0 = std::chrono::steady_clock::now(); + layers[i]->CalculateState(); + t1 = std::chrono::steady_clock::now(); + t_calc[i] += std::chrono::duration(t1 - t0).count(); + } + for (size_t i = 0; i < layers.size(); ++i) + { + t0 = std::chrono::steady_clock::now(); + layers[i]->UpdateState(); + t1 = std::chrono::steady_clock::now(); + t_update[i] += std::chrono::duration(t1 - t0).count(); } } - } - - std::cout << "\n=== Predictions ===\n"; - // Effortless prediction API - auto preds = net.Predict(flatX, inferenceSteps); + t0 = std::chrono::steady_clock::now(); + net.UpdateWeights(); + t1 = std::chrono::steady_clock::now(); + t_weights += std::chrono::duration(t1 - t0).count(); - int correct = 0; - for (int i = 0; i < 4; i++) - { - float pred = preds[i]; - float target = flatY[i]; - bool signCorrect = (pred > 0 && target > 0) || (pred < 0 && target < 0); - if (signCorrect) - correct++; - - std::cout << "Input: [" << std::setw(2) << flatX[i * 2] << ", " << std::setw(2) << flatX[i * 2 + 1] << "]" - << " | Target: " << std::showpos << target - << " | Pred: " << std::noshowpos << std::setprecision(4) << pred - << (signCorrect ? " OK" : " WRONG") << "\n"; + t0 = t1; + layers.back()->UnclampState(); + t1 = std::chrono::steady_clock::now(); + t_unclamp += std::chrono::duration(t1 - t0).count(); } - std::cout << "\nAccuracy: " << correct << "/4\n"; - std::cout << (correct == 4 ? "XOR LEARNED SUCCESSFULLY!" : "XOR NOT FULLY LEARNED.") << "\n"; + auto totalEnd = std::chrono::steady_clock::now(); + double totalTime = std::chrono::duration(totalEnd - totalStart).count(); + double msPerBatch = 1000.0 * totalTime / N_BATCHES; + + double calcSum = 0, updateSum = 0; + for (int i = 0; i < 4; ++i) { calcSum += t_calc[i]; updateSum += t_update[i]; } + + std::cout << "=== Granular breakdown, " << N_BATCHES << " batches ===\n\n"; + std::cout << "Total: " << totalTime << "s (" << msPerBatch << " ms/batch)\n"; + std::cout << "ngc-learn reference: 70.31 ms/batch\n\n"; + + auto pct = [&](double t) { return 100.0 * t / totalTime; }; + + std::cout << "reset_state: " << 1000*t_reset/N_BATCHES << " ms/batch (" << pct(t_reset) << "%)\n"; + std::cout << "clamp_input: " << 1000*t_clamp_input/N_BATCHES << " ms/batch (" << pct(t_clamp_input) << "%)\n"; + std::cout << "project_forward: " << 1000*t_project/N_BATCHES << " ms/batch (" << pct(t_project) << "%)\n"; + std::cout << "clamp_target: " << 1000*t_clamp_target/N_BATCHES << " ms/batch (" << pct(t_clamp_target) << "%)\n"; + std::cout << "settling (calc): " << 1000*calcSum/N_BATCHES << " ms/batch (" << pct(calcSum) << "%)\n"; + for (int i = 0; i < 4; ++i) + std::cout << " layer " << i << " calc: " << 1000*t_calc[i]/N_BATCHES << " ms/batch\n"; + std::cout << "settling (update):" << 1000*updateSum/N_BATCHES << " ms/batch (" << pct(updateSum) << "%)\n"; + for (int i = 0; i < 4; ++i) + std::cout << " layer " << i << " update: " << 1000*t_update[i]/N_BATCHES << " ms/batch\n"; + std::cout << "update_weights: " << 1000*t_weights/N_BATCHES << " ms/batch (" << pct(t_weights) << "%)\n"; + std::cout << "unclamp: " << 1000*t_unclamp/N_BATCHES << " ms/batch (" << pct(t_unclamp) << "%)\n"; + + double accountedFor = t_reset + t_clamp_input + t_project + t_clamp_target + calcSum + updateSum + t_weights + t_unclamp; + double unaccounted = totalTime - accountedFor; + std::cout << "\nUnaccounted (measurement overhead, fixed costs not captured above): " + << 1000*unaccounted/N_BATCHES << " ms/batch (" << pct(unaccounted) << "%)\n"; return 0; -} \ No newline at end of file +} diff --git a/tests/tThreadSweep.cpp b/tests/tThreadSweep.cpp deleted file mode 100644 index 83fbccd..0000000 --- a/tests/tThreadSweep.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -using namespace Deep; - -double RunConfig(int batchSize, int threads, int steps, int nBatches) -{ - omp_set_num_threads(threads); - openblas_set_num_threads(threads); - - std::mt19937 rng(42); - std::uniform_real_distribution dist(-1.0f, 1.0f); - - DiscriminativePCNetwork net(batchSize); - net.AddLayer(784, 512, 0.002f, 0.05f, 0.0f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); - net.AddLayer(512, 10, 0.002f, 0.05f, 0.0f, 0.0001f, ActivationType::TANH, ActivationType::dTANH); - net.AddLayer(10, 0, 0.002f, 0.05f, 0.0f, 0.0001f, ActivationType::LINEAR, ActivationType::dLINEAR); - net.Compile(); - - // Force thread count again AFTER Compile() -- DynamicThread() runs - // inside AddLayer()'s layer constructors and would otherwise override - // this with its own (currently hardcoded) choice. - omp_set_num_threads(threads); - openblas_set_num_threads(threads); - - net.RandomizeWeights(rng); - - std::vector X((size_t)batchSize * 784); - std::vector Y((size_t)batchSize * 10); - for (auto &v : X) - v = dist(rng); - for (auto &v : Y) - v = dist(rng); - - auto start = std::chrono::steady_clock::now(); - for (int b = 0; b < nBatches; ++b) - net.TrainStep(X, Y, steps); - auto end = std::chrono::steady_clock::now(); - - double totalSeconds = std::chrono::duration(end - start).count(); - return (totalSeconds / (nBatches * steps)) * 1000.0; // ms/step -} - -int main() -{ - const int STEPS = 60; // matches real training's confirmed value, not the profiler's 150 - const int N_BATCHES = 3; // short -- just enough for a stable reading per cell - - std::vector batchSizes = {256, 512, 1024}; - std::vector threadCounts = {4, 8, 12, 16, 24, 32, 48}; - - std::cout << "Sweeping batch_size x thread_count (" << STEPS << " steps, " - << N_BATCHES << " batches per cell)...\n\n"; - - std::cout << "batch\\threads"; - for (int t : threadCounts) - std::cout << "\t" << t; - std::cout << "\n"; - - for (int bsz : batchSizes) - { - std::cout << bsz; - for (int t : threadCounts) - { - double msPerStep = RunConfig(bsz, t, STEPS, N_BATCHES); - std::cout << "\t" << msPerStep; - std::cout.flush(); - } - std::cout << "\n"; - } - - std::cout << "\nLook for the LOWEST ms/step value in the whole grid -- that's the\n"; - std::cout << "real Pareto-optimal (batch_size, thread_count) combination for this\n"; - std::cout << "machine, not necessarily matching today's batch=256/threads=8 guess.\n"; - - return 0; -} \ No newline at end of file