diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad20d03..c94e4ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,18 +45,18 @@ jobs: cmake \ ninja-build \ libopenblas-dev \ - pybind11-dev \ python3-dev \ ${{ matrix.compiler == 'clang' && 'clang libomp-dev' || '' }} - name: Install Python build dependencies - # pybind11's CMake config package is needed even though libpybind11-dev - # is installed above, in case find_package(pybind11) resolves via pip's - # copy instead - installing both covers either path. - run: pip install pybind11 + run: python -m pip install nanobind rich - - name: Build and test (Release) - run: python build.py Release + - name: Build (Release) + # --no-tests: CMakeLists.txt no longer registers any CTest tests + # (the old test/benchmark executables were removed), so running + # ctest here would just fail on "no tests found" for no useful + # reason. + run: python build.py Release --no-tests - name: Upload build log if: always() @@ -105,7 +105,7 @@ jobs: !C:/vcpkg/buildtrees !C:/vcpkg/packages !C:/vcpkg/downloads - key: vcpkg-windows-openblas-pybind11-v1 + key: vcpkg-windows-openblas-v1 - name: Bootstrap vcpkg if: steps.vcpkg-cache.outputs.cache-hit != 'true' @@ -114,14 +114,10 @@ jobs: C:/vcpkg/bootstrap-vcpkg.bat - name: Install vcpkg dependencies - # Mirrors CMakeLists.txt: only OpenBLAS and pybind11 come from - # vcpkg. SLEEF and Google Benchmark are fetched from source by CMake - # itself (see FetchContent_Declare in CMakeLists.txt) and need no - # package manager step at all. - run: C:/vcpkg/vcpkg.exe install openblas pybind11 --triplet x64-windows + run: C:/vcpkg/vcpkg.exe install openblas --triplet x64-windows - name: Install Python build dependencies - run: pip install pybind11 + run: python -m pip install nanobind rich - name: Configure (CMake, using the windows-clang preset) # Uses CMakePresets.json's "windows-clang" preset directly, rather @@ -133,14 +129,6 @@ jobs: - name: Build run: cmake --build build/Release --config Release --parallel 2>&1 | Tee-Object -FilePath windows-build.log - - name: Run tests - run: | - $exe = Get-ChildItem -Path build/Release -Recurse -Filter "DeepityTests.exe" | Select-Object -First 1 - if (-not $exe) { - throw "DeepityTests.exe not found under build/Release - check windows-build.log" - } - & $exe.FullName - - name: Upload logs if: always() uses: actions/upload-artifact@v4 diff --git a/CMakeLists.txt b/CMakeLists.txt index fd99f43..5dcdd7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,12 @@ cmake_minimum_required(VERSION 3.21) project(deepity C CXX) -enable_testing() set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) option(DEEPITY_ENABLE_CUDA "Build with CUDA support" ON) -option(DEEPITY_BUILD_TESTS "Build the DeepityTests target" OFF) option(DEEPITY_BUILD_PYTHON_BINDINGS "Build the Python bindings target" ON) # Recommended if on Intel CPUs option(DEEPITY_USE_MKL "Use Intel MKL instead of OpenBLAS (Intel CPUs only -- MKL has a documented history of deliberately worse performance on non-Intel CPUs via runtime dispatch)" OFF) @@ -259,29 +257,6 @@ set(CMAKE_REQUIRED_QUIET ON) FetchContent_MakeAvailable(sleef) set(CMAKE_REQUIRED_QUIET OFF) -# --- Google Benchmark ----------------------------------------------- - -if(DEEPITY_BUILD_TESTS) - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - add_compile_options(-Wno-c2y-extensions) - endif() - - set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE) - set(BENCHMARK_ENABLE_PEDANTIC OFF CACHE BOOL "" FORCE) - set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) - set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE) - - FetchContent_Declare( - benchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.1 - ) - - set(CMAKE_REQUIRED_QUIET ON) - FetchContent_MakeAvailable(benchmark) - set(CMAKE_REQUIRED_QUIET OFF) -endif() - # --- Deepity library ------------------------------------------------ add_library(Deepity @@ -351,28 +326,6 @@ set_target_properties(Deepity PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${DEEPITY_IPO_SUPPORTED} ) -# --- Runtime DLL copying (Windows) ----------------------------------- -# $ only knows about DLLs CMake can see a full -# IMPORTED_LOCATION for (e.g. openblas_dll above). It can't discover -# libomp.dll on its own, so DEEPITY_OMP_DLL (resolved earlier) is copied -# alongside it explicitly. Centralized here instead of repeating both -# copy steps for every executable below. -function(deepity_copy_runtime_dlls target) - add_custom_command(TARGET ${target} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E $>,copy_if_different,true> - $ - $ - COMMAND_EXPAND_LISTS - ) - if(DEEPITY_OMP_DLL) - add_custom_command(TARGET ${target} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${DEEPITY_OMP_DLL}" - $ - ) - endif() -endfunction() - # --- pydeepity ------------------------------------------------------ if(DEEPITY_BUILD_PYTHON_BINDINGS AND NOT CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -424,113 +377,3 @@ if(WIN32) install(FILES ${DEEPITY_OMP_DLL} DESTINATION pydeepity) endif() endif() - -# --- Tests ---------------------------------------------------------- - -if(DEEPITY_BUILD_TESTS) - add_executable(DirectKPVerify tests/tDirectKPVerify.cpp) - target_link_libraries(DirectKPVerify PRIVATE Deepity) - set_target_properties(DirectKPVerify PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - add_test(NAME DirectKPVerify COMMAND DirectKPVerify) - - if(WIN32) - deepity_copy_runtime_dlls(DirectKPVerify) - endif() -endif() - -# --- Profiling ------------------------------------------------------ - -if(DEEPITY_BUILD_TESTS) - add_executable(GaussSeidelMiddleLayerVerify tests/tGaussSeidelMiddleLayerVerify.cpp) - target_link_libraries(GaussSeidelMiddleLayerVerify PRIVATE Deepity) - set_target_properties(GaussSeidelMiddleLayerVerify PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - add_test(NAME GaussSeidelMiddleLayerVerify COMMAND GaussSeidelMiddleLayerVerify) - - if(WIN32) - deepity_copy_runtime_dlls(GaussSeidelMiddleLayerVerify) - endif() -endif() - -if(DEEPITY_BUILD_TESTS) - add_library(DeepityProfiled STATIC - src/DiscriminativePCLayer.cpp - src/RBLayer.cpp - src/ConvPCLayer.cpp - src/SimpleConvPCLayer.cpp - src/SimplePCLayer.cpp - src/DiscriminativePCNetwork.cpp - src/ConvPCNetwork.cpp - src/SimplePCNetwork.cpp - src/ModelIO.cpp - src/StreamAlignedBatcher.cpp - ) - - if(DEEPITY_USE_MKL AND DEEPITY_BLAS_RESOLVED) - target_compile_definitions(DeepityProfiled PUBLIC DEEPITY_USE_MKL) - endif() - - target_include_directories(DeepityProfiled PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${BLAS_INCLUDE_DIR} - ) - target_include_directories(DeepityProfiled SYSTEM PUBLIC - ${sleef_SOURCE_DIR}/include - ${sleef_BINARY_DIR}/include - ) - target_link_libraries(DeepityProfiled PUBLIC - ${BLAS_LIBRARIES} - OpenMP::OpenMP_CXX - sleef - ) - - target_compile_definitions(DeepityProfiled PUBLIC SLEEF_STATIC_LIBS PCN_PROFILE) - - if(WIN32) - target_compile_definitions(DeepityProfiled PUBLIC NOMINMAX) - endif() - - if(MSVC) - target_compile_options(DeepityProfiled PRIVATE /O2 /fp:fast /openmp:llvm) - elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(DeepityProfiled PRIVATE -O3 -ffast-math -fvectorize -fslp-vectorize) - else() - target_compile_options(DeepityProfiled PRIVATE -O3 -ffast-math -ftree-vectorize -ftree-slp-vectorize) - endif() - - add_executable(DeepityProfile tests/tProfile.cpp) - target_link_libraries(DeepityProfile PRIVATE DeepityProfiled) - target_compile_definitions(DeepityProfile PRIVATE PCN_PROFILE) - - set_target_properties(DeepityProfile PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin - ) - add_test(NAME DeepityProfile COMMAND DeepityProfile) - - if(WIN32) - deepity_copy_runtime_dlls(DeepityProfile) - endif() - - # --- README benchmark: std-library vs Deep:: activations --------- - add_executable(ActivationBenchmark tests/tActivations.cpp) - target_link_libraries(ActivationBenchmark PRIVATE DeepityProfiled benchmark) - - set_target_properties(ActivationBenchmark PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin - ) - - if(WIN32) - deepity_copy_runtime_dlls(ActivationBenchmark) - endif() - - # --- README benchmark: sustained GFLOPS during a real train step - - add_executable(GflopsBenchmark tests/tGflopsBenchmark.cpp) - target_link_libraries(GflopsBenchmark PRIVATE DeepityProfiled benchmark) - - set_target_properties(GflopsBenchmark PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin - ) - - if(WIN32) - deepity_copy_runtime_dlls(GflopsBenchmark) - endif() -endif() \ No newline at end of file diff --git a/deepity_build/cmake_runner.py b/deepity_build/cmake_runner.py index 1ee3237..f1bdcd7 100644 --- a/deepity_build/cmake_runner.py +++ b/deepity_build/cmake_runner.py @@ -15,12 +15,7 @@ def find_generator() -> tuple[str | None, str]: ninja = shutil.which("ninja") return ninja, "Ninja" if ninja else "CMake" - def configure_command(config: BuildConfig, ninja: str | None, pgo_phase: str | None = None) -> list[str]: - """pgo_phase: None (no PGO), "GENERATE", or "USE" -- distinguishes - which pass of the two-pass PGO workflow this configure call is for. - A single config.pgo boolean can't express this on its own, since - both passes share the same BuildConfig.""" profile = config.profile cmd = [ @@ -35,6 +30,9 @@ def configure_command(config: BuildConfig, ninja: str | None, pgo_phase: str | N f"-DDEEPITY_ARCH_FLAGS={profile.unix_flags}", ] + if ninja: + cmd.extend(["-G", "Ninja"]) + if pgo_phase in ("GENERATE", "USE"): cmd.append(f"-DDEEPITY_PGO_MODE={pgo_phase}") cmd.append(f"-DDEEPITY_PGO_DATA_DIR={config.pgo_data_dir}") @@ -42,40 +40,35 @@ def configure_command(config: BuildConfig, ninja: str | None, pgo_phase: str | N if profile.msvc_flags: cmd.append(f"-DDEEPITY_MSVC_ARCH_FLAGS={profile.msvc_flags}") - if sys.platform == "win32": - if ninja: - cmd.extend(["-G", "Ninja"]) - - # CC being unset doesn't mean clang isn't in play -- CMake can - # auto-detect and pick it up on its own (as this project's own - # builds have shown), so check what's actually on PATH rather - # than an environment variable that may never have been set. - clang_path = shutil.which("clang") - cc_env = os.environ.get("CC", "").lower() - using_clang = clang_path is not None or "clang" in cc_env - - if using_clang: - # Locate libomp.lib next to the clang.exe on PATH - omp_lib = None - if clang_path: - llvm_lib_dir = Path(clang_path).parent.parent / "lib" - candidate = llvm_lib_dir / "libomp.lib" - if candidate.is_file(): - omp_lib = candidate.as_posix() - - cmd.extend([ - f"-DCMAKE_C_COMPILER=clang", - f"-DCMAKE_CXX_COMPILER=clang++", - "-DOpenMP_C_FLAGS=-fopenmp", - "-DOpenMP_CXX_FLAGS=-fopenmp", - "-DOpenMP_C_LIB_NAMES=omp", - "-DOpenMP_CXX_LIB_NAMES=omp", - ]) - if omp_lib: - cmd.append(f"-DOpenMP_omp_LIBRARY={omp_lib}") + if sys.platform == "win32" and ninja: + # CC being unset doesn't mean clang isn't in play -- CMake can + # auto-detect and pick it up on its own (as this project's own + # builds have shown), so check what's actually on PATH rather + # than an environment variable that may never have been set. + clang_path = shutil.which("clang") + cc_env = os.environ.get("CC", "").lower() + using_clang = clang_path is not None or "clang" in cc_env + + if using_clang: + omp_lib = None + if clang_path: + llvm_lib_dir = Path(clang_path).parent.parent / "lib" + candidate = llvm_lib_dir / "libomp.lib" + if candidate.is_file(): + omp_lib = candidate.as_posix() + + cmd.extend([ + f"-DCMAKE_C_COMPILER=clang", + f"-DCMAKE_CXX_COMPILER=clang++", + "-DOpenMP_C_FLAGS=-fopenmp", + "-DOpenMP_CXX_FLAGS=-fopenmp", + "-DOpenMP_C_LIB_NAMES=omp", + "-DOpenMP_CXX_LIB_NAMES=omp", + ]) + if omp_lib: + cmd.append(f"-DOpenMP_omp_LIBRARY={omp_lib}") return cmd - def build_command(config: BuildConfig) -> list[str]: return [ "cmake", @@ -98,4 +91,4 @@ def test_command(config: BuildConfig) -> list[str]: "--output-on-failure", "-j", str(config.jobs), - ] \ No newline at end of file + ] diff --git a/examples/benchmark.py b/examples/benchmark.py index 19893a1..f296e49 100644 --- a/examples/benchmark.py +++ b/examples/benchmark.py @@ -6,9 +6,9 @@ def load_full_mnist(): import gzip import urllib.request - + print("Fetching canonical MNIST dataset (idx-ubyte, matching ngc-learn exactly)...") base_url = "https://storage.googleapis.com/cvdf-datasets/mnist/" - files = { + files: dict[str, str] = { "x_train": "train-images-idx3-ubyte.gz", "y_train": "train-labels-idx1-ubyte.gz", "x_test": "t10k-images-idx3-ubyte.gz", @@ -16,31 +16,26 @@ def load_full_mnist(): } data_dir = "./data" os.makedirs(data_dir, exist_ok=True) - paths = {} + paths: dict[str, str] = {} for key, fname in files.items(): filepath = os.path.join(data_dir, fname) paths[key] = filepath if not os.path.exists(filepath): + print(f"Downloading {fname}...") urllib.request.urlretrieve(base_url + fname, filepath) - - # Standardized Normalization with gzip.open(paths["x_train"], 'rb') as f: X_train_raw = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 784) - X_train = (X_train_raw.astype(np.float32) / 255.0 - 0.1307) / 0.3081 - with gzip.open(paths["x_test"], 'rb') as f: X_test_raw = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 784) - X_test = (X_test_raw.astype(np.float32) / 255.0 - 0.1307) / 0.3081 - with gzip.open(paths["y_train"], 'rb') as f: y_train_labels = np.frombuffer(f.read(), np.uint8, offset=8) with gzip.open(paths["y_test"], 'rb') as f: y_test_labels = np.frombuffer(f.read(), np.uint8, offset=8) - + X_train = X_train_raw.astype(np.float32) / 255.0 + X_test = X_test_raw.astype(np.float32) / 255.0 eps = 0.001 Y_train = np.full((y_train_labels.shape[0], 10), eps, dtype=np.float32) Y_train[np.arange(y_train_labels.shape[0]), y_train_labels] = 1.0 - eps - return X_train, Y_train, X_test, y_test_labels def run_benchmark(name, net, X_train, Y_train, X_test, y_test_labels, lr, steps, epochs=3, decay=0.9): diff --git a/examples/mnist.py b/examples/mnist.py index cf73926..4497a55 100644 --- a/examples/mnist.py +++ b/examples/mnist.py @@ -7,10 +7,9 @@ def load_full_mnist(): import gzip import urllib.request - print("Fetching canonical MNIST dataset (idx-ubyte, matching ngc-learn exactly)...") base_url = "https://storage.googleapis.com/cvdf-datasets/mnist/" - files = { + files: dict[str, str] = { "x_train": "train-images-idx3-ubyte.gz", "y_train": "train-labels-idx1-ubyte.gz", "x_test": "t10k-images-idx3-ubyte.gz", @@ -18,14 +17,13 @@ def load_full_mnist(): } data_dir = "./data" os.makedirs(data_dir, exist_ok=True) - paths = {} + paths: dict[str, str] = {} for key, fname in files.items(): filepath = os.path.join(data_dir, fname) paths[key] = filepath if not os.path.exists(filepath): print(f"Downloading {fname}...") urllib.request.urlretrieve(base_url + fname, filepath) - with gzip.open(paths["x_train"], 'rb') as f: X_train_raw = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 784) with gzip.open(paths["x_test"], 'rb') as f: @@ -34,17 +32,13 @@ def load_full_mnist(): y_train_labels = np.frombuffer(f.read(), np.uint8, offset=8) with gzip.open(paths["y_test"], 'rb') as f: y_test_labels = np.frombuffer(f.read(), np.uint8, offset=8) - X_train = X_train_raw.astype(np.float32) / 255.0 X_test = X_test_raw.astype(np.float32) / 255.0 - eps = 0.001 Y_train = np.full((y_train_labels.shape[0], 10), eps, dtype=np.float32) Y_train[np.arange(y_train_labels.shape[0]), y_train_labels] = 1.0 - eps - return X_train, Y_train, X_test, y_test_labels - def main() -> None: SEED = int(sys.argv[1]) if len(sys.argv) > 1 else 7 EPOCHS = int(sys.argv[2]) if len(sys.argv) > 2 else 15 diff --git a/examples/test_ffnn.py b/examples/test_ffnn.py index a3716ec..9d691ee 100644 --- a/examples/test_ffnn.py +++ b/examples/test_ffnn.py @@ -6,24 +6,17 @@ from sklearn.model_selection import train_test_split from time import perf_counter -# Standard backprop feedforward baseline -- matches SequentialPCN's EXACT -# architecture (784 -> 512 -> 10, single hidden layer, tanh) for a true -# apples-to-apples comparison: same network shape, same data, different -# training algorithm (ordinary backprop vs. predictive coding settling). -# -# Worth knowing going in: PC networks are generally understood in the -# literature to be substantially slower than backprop to reach comparable -# accuracy, because of the iterative settling cost per batch (T_infer steps -# of forward+backward-like computation, vs. backprop's single forward + -# single backward pass). This script exists to measure that gap directly -# for this specific architecture/dataset, not to assume the answer. - +# Standard backprop feedforward baseline -- matches DKPPCN EXACTLY +# architecture (784 -> 512 -> 10, single hidden layer, sigmoid) for a true +# apples-to-apples comparison: same network shape, same data, same LR decay, +# same batch size, but ordinary backprop vs. predictive coding settling. def load_full_mnist_torch(): print("Fetching full MNIST dataset (70,000 images)...") X, y = fetch_openml('mnist_784', version=1, return_X_y=True, as_frame=False, parser='auto') - X = (X.astype(np.float32) / 127.5) - 1.0 + # MATCHED: Scale from 0 to 1 exactly like the C++ script + X = X.astype(np.float32) / 255.0 y = y.astype(int) X_train, X_test, y_train, y_test = train_test_split( @@ -38,16 +31,12 @@ def load_full_mnist_torch(): class FFNN(nn.Module): - """784 -> 512 -> 10, tanh hidden activation -- matches SequentialPCN's - add_layer(784, 512, act="tanh") / add_layer(512, 10, act="tanh") / - add_layer(10, 0, act="linear") exactly in shape and hidden activation. - Output layer is linear (raw logits), standard for CrossEntropyLoss.""" - + """784 -> 512 -> 10, sigmoid hidden activation.""" def __init__(self): super().__init__() self.net = nn.Sequential( nn.Linear(784, 512), - nn.Tanh(), + nn.Sigmoid(), # MATCHED: act="sigmoid" from DKPPCN nn.Linear(512, 10), ) @@ -56,9 +45,6 @@ def forward(self, x): def evaluate(model, test_loader, device): - """Full test-set accuracy. Cheap here since inference is a single - forward pass (no iterative settling like the PC variants) -- running - this every epoch adds negligible overhead relative to training time.""" model.eval() correct = 0 total = 0 @@ -76,22 +62,26 @@ def evaluate(model, test_loader, device): def main(): X_train_t, y_train_t, X_test_t, y_test_t = load_full_mnist_torch() - BATCH_SIZE = 256 - EPOCHS = 50 # matching the PC run's epoch count for direct comparison + BATCH_SIZE = 250 # MATCHED: DKPPCN uses 250 + EPOCHS = 50 print(f"\nBuilding PyTorch DataLoader (batch_size={BATCH_SIZE})...") train_loader = DataLoader(TensorDataset(X_train_t, y_train_t), batch_size=BATCH_SIZE, shuffle=True) test_loader = DataLoader(TensorDataset(X_test_t, y_test_t), batch_size=BATCH_SIZE, shuffle=False) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # MATCHED: Hardcode to CPU to compare against C++ CPU performance + device = torch.device("cpu") print(f"Using device: {device}") model = FFNN().to(device) - optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + + # MATCHED: Initial LR and Decay Rate from DKPPCN + optimizer = torch.optim.Adam(model.parameters(), lr=0.00373) + scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.94) criterion = nn.CrossEntropyLoss() print(f"\nTraining: {EPOCHS} epochs, batch_size={BATCH_SIZE}, " - f"architecture=784->512->10 (matches SequentialPCN exactly)...\n") + f"architecture=784->512->10 (matches DKPPCN exactly)...\n") start_time = perf_counter() epoch_accs = [] @@ -112,6 +102,9 @@ def main(): epoch_loss += loss.item() n_batches += 1 + + # MATCHED: Step the LR decay per epoch + scheduler.step() avg_loss = epoch_loss / n_batches epoch_acc = evaluate(model, test_loader, device) @@ -127,11 +120,10 @@ def main(): final_acc = epoch_accs[-1] print(f"\nTest Accuracy: {final_acc:.2f}%") print(f"\n--- Summary ---") - print(f"Architecture: 784 -> 512 -> 10 (tanh hidden)") + print(f"Architecture: 784 -> 512 -> 10 (sigmoid hidden)") print(f"Training time: {train_time:.1f}s for {EPOCHS} epochs") print(f"Test accuracy: {final_acc:.2f}%") print(f"\nPer-epoch test accuracy: {[round(a, 2) for a in epoch_accs]}") - if __name__ == "__main__": main() diff --git a/include/deepity/utils/AdamOptimizer.h b/include/deepity/utils/AdamOptimizer.h index 7bfe84d..43db769 100644 --- a/include/deepity/utils/AdamOptimizer.h +++ b/include/deepity/utils/AdamOptimizer.h @@ -389,7 +389,7 @@ namespace Deep size_t r = n % 4; size_t simd_end = n - r; #pragma omp parallel for schedule(static) if(n >= 4096 && !omp_in_parallel()) - for (ptrdiff_t; i < (ptrdiff_t)simd_end; i += 4) + for (ptrdiff_t i=0; i < (ptrdiff_t)simd_end; i += 4) { __m128 g = _mm_loadu_ps(&grad[i]); __m128 m_old = _mm_loadu_ps(&m[i]); diff --git a/include/deepity/utils/Optimize.h b/include/deepity/utils/Optimize.h index 40e8b44..cbeb666 100644 --- a/include/deepity/utils/Optimize.h +++ b/include/deepity/utils/Optimize.h @@ -1,6 +1,9 @@ #pragma once +#include #include #include +#include +#include #include #ifdef __linux__ @@ -47,9 +50,17 @@ namespace Deep #elif defined(_WIN32) DWORD bufSize = 0; GetLogicalProcessorInformation(nullptr, &bufSize); - auto buf = std::make_unique(bufSize / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)); - GetLogicalProcessorInformation(buf.get(), &bufSize); - for (DWORD i = 0; i < bufSize / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); i++) + + // Computed once, by name, instead of repeating `bufSize / + // sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)` inline at both the + // allocation site and the loop condition -- and std::vector instead + // of make_unique(count), since that's the far more common, + // heavily-exercised pattern for a runtime-sized buffer like this. + const size_t count = static_cast(bufSize) / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + std::vector buf(count); + GetLogicalProcessorInformation(buf.data(), &bufSize); + + for (size_t i = 0; i < count; ++i) { if (buf[i].Relationship == RelationCache && buf[i].Cache.Level == 2) return buf[i].Cache.Size; diff --git a/mnist.py b/mnist.py index 5316181..879160d 100644 --- a/mnist.py +++ b/mnist.py @@ -7,10 +7,9 @@ def load_full_mnist(): import gzip import urllib.request - print("Fetching canonical MNIST dataset (idx-ubyte, matching ngc-learn exactly)...") base_url = "https://storage.googleapis.com/cvdf-datasets/mnist/" - files = { + files: dict[str, str] = { "x_train": "train-images-idx3-ubyte.gz", "y_train": "train-labels-idx1-ubyte.gz", "x_test": "t10k-images-idx3-ubyte.gz", @@ -18,14 +17,13 @@ def load_full_mnist(): } data_dir = "./data" os.makedirs(data_dir, exist_ok=True) - paths = {} + paths: dict[str, str] = {} for key, fname in files.items(): filepath = os.path.join(data_dir, fname) paths[key] = filepath if not os.path.exists(filepath): print(f"Downloading {fname}...") urllib.request.urlretrieve(base_url + fname, filepath) - with gzip.open(paths["x_train"], 'rb') as f: X_train_raw = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 784) with gzip.open(paths["x_test"], 'rb') as f: @@ -34,14 +32,11 @@ def load_full_mnist(): y_train_labels = np.frombuffer(f.read(), np.uint8, offset=8) with gzip.open(paths["y_test"], 'rb') as f: y_test_labels = np.frombuffer(f.read(), np.uint8, offset=8) - X_train = X_train_raw.astype(np.float32) / 255.0 X_test = X_test_raw.astype(np.float32) / 255.0 - eps = 0.001 Y_train = np.full((y_train_labels.shape[0], 10), eps, dtype=np.float32) Y_train[np.arange(y_train_labels.shape[0]), y_train_labels] = 1.0 - eps - return X_train, Y_train, X_test, y_test_labels def train_step_dfa(net, X, Y, inference_steps): diff --git a/numpyPCN.py b/numpyPCN.py index 32134fd..e736cd0 100644 --- a/numpyPCN.py +++ b/numpyPCN.py @@ -1,13 +1,21 @@ import numpy as np +import numpy.typing as npt +from numpy.typing import NDArray import os from time import perf_counter +# Module-level so it's a plain, bare-name-resolvable reference wherever it's +# used as a list placeholder below -- a class-body attribute isn't visible +# as a bare name inside a method (only via self./ClassName.), which is why +# this used to raise NameError the first time process() actually ran. +NDArrayF32 = npt.NDArray[np.float32] + def load_canonical_mnist(): import gzip import urllib.request print("Fetching canonical MNIST dataset (idx-ubyte)...") base_url = "https://storage.googleapis.com/cvdf-datasets/mnist/" - files = { + files: dict[str, str] = { "x_train": "train-images-idx3-ubyte.gz", "y_train": "train-labels-idx1-ubyte.gz", "x_test": "t10k-images-idx3-ubyte.gz", @@ -15,7 +23,7 @@ def load_canonical_mnist(): } data_dir = "./data" os.makedirs(data_dir, exist_ok=True) - paths = {} + paths: dict[str, str] = {} for key, fname in files.items(): filepath = os.path.join(data_dir, fname) paths[key] = filepath @@ -61,12 +69,17 @@ def __init__(self, layer_sizes, seed=1234): self.mb = [np.zeros_like(b) for b in self.b] self.vb = [np.zeros_like(b) for b in self.b] self.t = 0 - + def process(self, X, Y=None, steps=20, ir=0.04, lr=0.001, train=True): B = X.shape[0] - z = [None] * (self.L + 1) - mu = [None] * (self.L + 1) - e = [None] * (self.L + 1) + # Real placeholder arrays, not the NDArrayF32 type object itself -- + # every slot gets overwritten below before being read, but typing + # these as actual arrays (rather than putting the type alias into + # the list as a literal value) is what lets pyright track them as + # arrays through the rest of this method. + z: list[NDArrayF32] = [np.empty(0, dtype=np.float32) for _ in range(self.L + 1)] + mu: list[NDArrayF32] = [np.empty(0, dtype=np.float32) for _ in range(self.L + 1)] + e: list[NDArrayF32] = [np.empty(0, dtype=np.float32) for _ in range(self.L + 1)] # --- PROJECTION PASS --- z[0] = X @@ -84,7 +97,7 @@ def process(self, X, Y=None, steps=20, ir=0.04, lr=0.001, train=True): # --- E-STEP (SETTLING) --- for step in range(steps): - dz = [None] * self.L + dz: list[NDArrayF32] = [np.empty(0, dtype=np.float32) for _ in range(self.L)] # 1. Update states (Linearized approximation: NO phi_prime!) for i in range(1, self.L): diff --git a/pgo_workload.py b/pgo_workload.py index f9e0e3b..3943d0a 100644 --- a/pgo_workload.py +++ b/pgo_workload.py @@ -29,13 +29,13 @@ def load_pgo_subset(): print("PGO workload: fetching MNIST (cached after first run)...") base_url = "https://storage.googleapis.com/cvdf-datasets/mnist/" - files = { + files: dict[str, str] = { "x_train": "train-images-idx3-ubyte.gz", "y_train": "train-labels-idx1-ubyte.gz", } data_dir = "./data" os.makedirs(data_dir, exist_ok=True) - paths = {} + paths: dict[str, str] = {} for key, fname in files.items(): filepath = os.path.join(data_dir, fname) paths[key] = filepath diff --git a/pydeepity/SequentialPCN.py b/pydeepity/SequentialPCN.py index 193ec68..e8f6947 100644 --- a/pydeepity/SequentialPCN.py +++ b/pydeepity/SequentialPCN.py @@ -3,6 +3,16 @@ import numpy as np import numpy.typing as npt from .utils import _fit_with_progress +from rich.console import Console +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) class SequentialPCN(dy.DiscriminativePCNetwork): """ diff --git a/pydeepity/SimpleConvolutionalPCN.py b/pydeepity/SimpleConvolutionalPCN.py index 47fe470..b20f9e0 100644 --- a/pydeepity/SimpleConvolutionalPCN.py +++ b/pydeepity/SimpleConvolutionalPCN.py @@ -100,14 +100,8 @@ def fit( _fit_with_progress(self, X, Y, epochs, steps, initial_lr, decay_rate, shuffle) return self - def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: - return super().train_step(X.flatten(), Y.flatten(), steps) - def train_step_with_projection(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: return super().train_step_with_projection(X.flatten(), Y.flatten(), steps) - def predict(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: - return super().predict(X.flatten(), steps) - def predict_with_projection(self, X: npt.NDArray[np.float32], steps: int) -> npt.NDArray[np.float32]: return super().predict_with_projection(X.flatten(), steps) diff --git a/pyrightconfig.json b/pyrightconfig.json index d404148..7670270 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -6,7 +6,10 @@ "build", "docs", "experiments", - "examples/test_pcn_torch.py", "**/node_modules", "**/.*", ".venv" + "examples", + "**/node_modules", + "**/.*", + ".venv" ], "executionEnvironments": [ @@ -19,4 +22,4 @@ "reportAttributeAccessIssue": "none" } ] -} \ No newline at end of file +} diff --git a/temp.py b/temp.py deleted file mode 100644 index b51cb1e..0000000 --- a/temp.py +++ /dev/null @@ -1,181 +0,0 @@ -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import numpy as np -import logging - -logging.getLogger('matplotlib.font_manager').setLevel(logging.ERROR) - -# --- FONT UPDATE: Added Alliance No.2 to the front --- -plt.rcParams['font.sans-serif'] = ['Alliance No.2', 'Alliance No 2', 'Alliance', 'Arial', 'Helvetica', 'DejaVu Sans'] -plt.rcParams['font.family'] = 'sans-serif' - -# --- Data --- -dkp_epochs = np.arange(1, 51) -dkp_acc = [89.36, 93.44, 94.16, 94.48, 95.44, 96.52, 96.40, 96.56, 96.84, 97.52, - 97.68, 97.32, 97.72, 97.76, 97.96, 97.80, 98.44, 98.32, 97.80, 98.80, - 98.80, 98.44, 98.64, 98.68, 98.32, 98.32, 98.64, 99.04, 98.80, 99.16, - 98.96, 99.24, 99.28, 99.12, 98.96, 99.20, 99.40, 99.04, 99.32, 99.36, - 98.72, 99.32, 98.96, 99.24, 99.28, 99.36, 99.20, 99.12, 99.36, 99.48] -dkp_test = 97.73 -dkp_time = 60 - -ngc_epochs = np.arange(1, 16) -ngc_acc = [26.91, 42.96, 60.12, 75.20, 84.68, 89.52, 91.90, - 93.45, 94.30, 94.80, 95.13, 95.38, 95.63, 95.74, 95.95] -ngc_test = 95.09 - -pytorch_ffnn_test = 98.27 -old_impl_test = 92.42 -old_time_min = 50 -speedup = round(old_time_min * 60 / dkp_time) - -# --- Theme (Palantir-style Light Mode) --- -BG = "#F8FAFC" -TEXT = "#0F172A" -MUTED = "#64748B" -GRID = "#CBD5E1" -CURVE = "#0284C7" -GOLD = "#D97706" -GRAY = "#94A3B8" -RED = "#DC2626" -CARD_BG = "#FFFFFF" - -fig = plt.figure(figsize=(16, 9), dpi=300, facecolor=BG) - -# --- Enhanced 3D Studio Room (Infinity Cove) --- -bg_ax = fig.add_axes([0, 0, 1, 1], zorder=-1) -bg_ax.axis('off') - -horizon = 0.15 # Aligns with the bottom of the graph axes - -# 1. Cyclorama base -for i in range(200): - y1, y2 = i/200.0, (i+1)/200.0 - if y1 <= horizon: - progress = y1 / horizon - c = 0.93 + 0.07 * progress - else: - progress = (y1 - horizon) / (1.0 - horizon) - c = 1.00 - 0.05 * progress - bg_ax.fill_between([0, 1], y1, y2, color=(c, c, c), zorder=1) - -# 2. Seamless Full-Width Background Grid -# This replaces `ax.grid()` to fix the "vertical rectangle box" issue. -y_ticks_manual = [80, 85, 90, 95, 100] -for y_val in y_ticks_manual: - # Map graph coordinates to figure coordinates (75 to 102 mapped into 0.15 to 0.75 height) - y_fig = horizon + (y_val - 75) / (102 - 75) * 0.60 - # Draw line from entirely left to entirely right edge of image - bg_ax.plot([0, 1], [y_fig, y_fig], color=GRID, alpha=0.4, linewidth=1, linestyle="--", zorder=2) - -# 3. Floor/Wall spotlights -for i in range(40): - radius = 0.6 * (40 - i) / 40 - alpha = 0.04 * (1 - i/40) - bg_ax.add_patch(matplotlib.patches.Ellipse((0.5, horizon), radius*2, radius*0.4, - color='#FFFFFF', alpha=alpha, transform=bg_ax.transAxes, ec="none", zorder=3)) - bg_ax.add_patch(plt.Circle((0.5, 0.5), radius*1.2, color='#FFFFFF', alpha=alpha*0.8, - transform=bg_ax.transAxes, ec="none", zorder=3)) - - -# --- Main Plot --- -ax = fig.add_axes([0.08, 0.15, 0.84, 0.60]) -ax.set_facecolor('none') - -# --- 1 & 2. Background Reference Lines (Flat Transparent Indicators) --- -ax.axhline(pytorch_ffnn_test, color=GOLD, linewidth=2.5, linestyle=":", alpha=0.25, zorder=2) -ax.axhline(old_impl_test, color=RED, linewidth=2, linestyle="-.", alpha=0.25, zorder=2) - -ax.plot(ngc_epochs, ngc_acc, color=GRAY, linewidth=3.5, linestyle="--", solid_capstyle="round", alpha=0.25, zorder=4) - -# --- 3. Headline Curve (DKPPCN) Translucent Depth (Frosted Glass & Depth of Field) --- -dx = 1.0 # Deeper X stretch -dy = -1.5 # Deeper Y stretch -micro_steps = 250 - -for i in range(micro_steps): - prop = i / micro_steps - alpha = 0.05 * (1 - prop)**2 - lw = 3.5 + (prop * 8) - shift_x = dkp_epochs + dx * prop - shift_y = [val + dy * prop for val in dkp_acc] - ax.plot(shift_x, shift_y, color=CURVE, linewidth=lw, linestyle="-", solid_capstyle="round", alpha=alpha, zorder=5) - -ax.fill_between(dkp_epochs, dkp_acc, 75, color=CURVE, alpha=0.03, zorder=6) - -ax.plot(dkp_epochs, dkp_acc, color=CURVE, linewidth=4.5, solid_capstyle="round", zorder=7) -ax.plot(dkp_epochs, dkp_acc, color='#BAE6FD', linewidth=1.2, solid_capstyle="round", zorder=8) -# Just a tiny touch of white for that glass edge reflection -ax.plot(dkp_epochs, [v + 0.1 for v in dkp_acc], color='#FFFFFF', linewidth=0.6, solid_capstyle="round", zorder=9) - -# --- 4. End Marker (Flat Indicator) --- -ax.scatter(dkp_epochs[-1], dkp_acc[-1], s=180, color=CURVE, edgecolors=CARD_BG, linewidth=2.5, zorder=10) - -# --- Direct line labels --- -# We keep labels fully opaque for readability -ax.text(3, 99, f'PyTorch Backprop: {pytorch_ffnn_test}%', - color=GOLD, fontsize=13, va='bottom', ha='left', weight='bold') -ax.text(15.5, 91.8, f'Previous Implementation: {old_impl_test}% (~{old_time_min} min)', - color=RED, fontsize=13, va='top', ha='left', weight='bold', alpha=0.9) -ax.text(15.5, 96, f'ngc-learn: {ngc_test}%', - color=MUTED, fontsize=12.5, va='top', ha='left') - -# --- Direct label for the headline curve --- -ax.text(48.5, 101.3, f'Deepity DKPPCN: {dkp_test}% ({dkp_time}s)', - color=CURVE, fontsize=15, weight="bold", ha='right', va='top', zorder=11) - -# --- Axes styling --- -ax.set_xlim(0, 52) -ax.set_ylim(75, 102) -# Force our ticks to match the background seamless grid lines -ax.set_yticks(y_ticks_manual) - -for spine in ax.spines.values(): - spine.set_visible(False) -# Removed inner ax.grid() to prevent the "vertical rectangle" artifact completely -ax.tick_params(colors=MUTED, labelsize=12, length=0, pad=10) -ax.set_xlabel("Training Epoch", color=TEXT, fontsize=14, labelpad=15, weight="bold") -ax.set_ylabel("MNIST Test Accuracy (%)", color=TEXT, fontsize=14, labelpad=15, weight="bold") - -# --- Header --- -fig.text(0.08, 0.88, "An Alternative to Backprop Just Closed the Gap", - color=TEXT, fontsize=34, weight="bold") -fig.text(0.08, 0.82, "Deepity DKPPCN • Predictive Coding Network Trained Locally in C++", - color=CURVE, fontsize=16, weight="bold", alpha=0.95) - -# --- Stats card --- -card_text = ( - f"{dkp_test}% Test Accuracy\n" - f"{dkp_time}s Total Training Time\n" - f"{speedup}x Faster than Previous Version\n\n" - "✓ 60,000 training images\n" - "✓ Trained locally in C++\n" - "✓ No backpropagation required" -) - -# Soft shadow for card -ax.text( - 0.903, 0.292, card_text, - transform=ax.transAxes, color=(0,0,0,0), fontsize=13.5, linespacing=1.6, - va="center", ha="right", - bbox=dict(boxstyle="round,pad=1.1", facecolor='black', edgecolor='none', alpha=0.04), zorder=12, -) -# Top Card -ax.text( - 0.90, 0.30, card_text, - transform=ax.transAxes, color=TEXT, fontsize=13.5, linespacing=1.6, - va="center", ha="right", - bbox=dict(boxstyle="round,pad=1.1", facecolor=CARD_BG, edgecolor=GRID, linewidth=1.5, alpha=0.95), zorder=13, -) - -# --- Footer --- -fig.text(0.08, 0.05, - f"DKPPCN closes to within {pytorch_ffnn_test - dkp_test:.2f} points of backprop while training {speedup}x faster than the previous implementation.", - fontsize=13, color=MUTED) -fig.text(0.92, 0.05, "LinkedIn: @jack-c-rose | github.com/Ra4ster", - fontsize=12, color=MUTED, alpha=0.7, ha="right") - -plt.savefig("resources/dkppcn_chart.png", facecolor=fig.get_facecolor(), bbox_inches="tight", pad_inches=0.4) -plt.close(fig) -print("Saved resources/dkppcn_chart.png") \ No newline at end of file