From 7696ec621b8ce9b65c583ed7b617deac78d89090 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:46:11 +0100 Subject: [PATCH 01/19] adding AES-256 --- .vscode/c_cpp_properties.json | 29 +- CMakeLists.txt | 6 + README.rst | 9 +- cmake/FindLibstored.cmake.tmpl | 3 + cmake/LibstoredStoresConfig.cmake.in | 3 +- cmake/libstored.cmake | 53 ++- dist/common/FindTinyAES.cmake | 74 +++++ dist/common/build.sh | 38 ++- dist/ubuntu/test_config.sh | 2 + dist/win32/build.cmd | 44 ++- .../1_hello_submodule/CMakeLists.txt | 4 +- include/libstored/aes.h | 144 ++++++++ include/stored.h | 4 +- python/CMakeLists.txt | 3 +- sphinx/Doxyfile.in | 3 +- sphinx/doc/cpp_protocol.rst | 6 + src/aes.cpp | 309 ++++++++++++++++++ tests/test_protocol.cpp | 27 ++ 18 files changed, 713 insertions(+), 48 deletions(-) create mode 100644 dist/common/FindTinyAES.cmake create mode 100644 include/libstored/aes.h create mode 100644 src/aes.cpp diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 4e3ae6aa..4a1f1ce8 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -1,13 +1,26 @@ { + "env": { + "includePath": [ + "${workspaceFolder}/include", + "${workspaceFolder}/dist/*/build/*-extern-prefix/src/**", + "${workspaceFolder}/examples/**", + "${workspaceFolder}/tests/**" + ], + "defines": [ + "STORED_HAVE_ZMQ", + "STORED_HAVE_AES", + "STORED_HAVE_HEATSHRINK", + "AES256" + ] + }, "configurations": [ { "name": "Linux", "includePath": [ - "${workspaceFolder}/include", - "${workspaceFolder}/**" + "${env:includePath}" ], "defines": [ - "STORED_HAVE_ZMQ" + "${env:defines}" ], "compilerPath": "/usr/bin/clang", "cStandard": "c17", @@ -17,11 +30,10 @@ { "name": "Win32", "includePath": [ - "${workspaceFolder}/include", - "${workspaceFolder}/**" + "${env:includePath}" ], "defines": [ - "STORED_HAVE_ZMQ" + "${env:defines}" ], "cStandard": "c17", "cppStandard": "c++17" @@ -29,11 +41,10 @@ { "name": "Mac", "includePath": [ - "${workspaceFolder}/include", - "${workspaceFolder}/**" + "${env:includePath}" ], "defines": [ - "STORED_HAVE_ZMQ" + "${env:defines}" ], "compilerPath": "/usr/bin/clang", "cStandard": "c17", diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c63d20e..08f998b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,6 +160,12 @@ if(LIBSTORED_HAVE_ZTH) find_package(Zth REQUIRED) endif() +option(LIBSTORED_HAVE_AES "Use AES" ON) + +if(LIBSTORED_HAVE_AES) + find_package(TinyAES REQUIRED) +endif() + set(LIBSTORED_CLANG_TIDY_DEFAULT OFF) if(${CMAKE_VERSION} VERSION_GREATER "3.6.0") diff --git a/README.rst b/README.rst index 56645238..ea9207bf 100644 --- a/README.rst +++ b/README.rst @@ -216,8 +216,8 @@ device. However, once you implemented this data transport, you can access the store, and observe and manipulate it using an Embedded Debugger (PC) client. Moreover, the protocol supports arbitrary streams (like stdout) from the application to the client, and has high-speed tracing of store variables. These -streams are optionally heatshrink_ compressed. libstored provides Python -classes for your custom scripts, a CLI and GUI interface. +streams are optionally heatshrink_ compressed, and AES-256 encrypted via tiny-AES-c_. +libstored provides Python classes for your custom scripts, a CLI and GUI interface. Your application can have one store with one debugging interface, but also multiple stores with one debugging interface, or one store with multiple @@ -457,7 +457,7 @@ generate stuff for you. This is how to integrate it in your project: Before including ``libstored``, you can specify several options (see ``cmake/libstored.cmake``), such as enabling ASan or clang-tidy. - Especially the library dependencies (ZeroMQ, Zth, heatshrink) are + Especially the library dependencies (ZeroMQ, Zth, heatshrink, tiny-AES-c) are relevant to consider. For example, to enable ZeroMQ: .. code:: cmake @@ -514,7 +514,7 @@ approach. In that case: 3. In your project, call ``find_package(Libstored)``, while having the generated ``FindLibstored.cmake`` in your ``CMAKE_MODULE_PATH``. - ``Libstored`` accepts ``ZeroMQ``, ``Zth``, ``Heatshrink`` as ``COMPONENTS``. + ``Libstored`` accepts ``ZeroMQ``, ``Zth``, ``Heatshrink``, and ``AES`` as ``COMPONENTS``. Setting these enables integration of these libraries with libstored. When possible, they are taken from your host system, or built from source. If you want more control over these libraries, you can also use the mechanism @@ -540,6 +540,7 @@ License, v. 2.0, as specified in LICENSE. This project complies to `REUSE`_. .. _documentation: https://demcon.github.io/libstored .. _8_sync: https://github.com/DEMCON/libstored/tree/main/examples/8_sync .. _heatshrink: https://github.com/atomicobject/heatshrink +.. _tiny-AES-c: https://github.com/kokke/tiny-AES-c .. _Kst: https://kst-plot.kde.org/ .. _python: https://github.com/DEMCON/libstored/tree/main/python .. _REUSE: https://reuse.software/ diff --git a/cmake/FindLibstored.cmake.tmpl b/cmake/FindLibstored.cmake.tmpl index f2ca2d77..a4f5cdc1 100644 --- a/cmake/FindLibstored.cmake.tmpl +++ b/cmake/FindLibstored.cmake.tmpl @@ -36,6 +36,9 @@ if(EXISTS ${LIBSTORED_SOURCE_DIR}/cmake/libstored.cmake) elseif("${c}" STREQUAL "Heatshrink") find_package(Heatshrink ${_req}) set(LIBSTORED_HAVE_HEATSHRINK ON CACHE INTERNAL "Enable heatshrink" FORCE) + elseif("${c}" STREQUAL "AES") + find_package(TinyAES ${_req}) + set(LIBSTORED_HAVE_AES ON CACHE INTERNAL "Enable AES" FORCE) endif() endforeach() diff --git a/cmake/LibstoredStoresConfig.cmake.in b/cmake/LibstoredStoresConfig.cmake.in index 46a96c3c..bbd678db 100644 --- a/cmake/LibstoredStoresConfig.cmake.in +++ b/cmake/LibstoredStoresConfig.cmake.in @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +# SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers # # SPDX-License-Identifier: MPL-2.0 @@ -47,6 +47,7 @@ endif() include(${LIBSTORED_STORES_CMAKE_PATH}/libstored.cmake OPTIONAL) include(${LIBSTORED_STORES_CMAKE_PATH}/heatshrink.cmake OPTIONAL) +include(${LIBSTORED_STORES_CMAKE_PATH}/tinyaes.cmake OPTIONAL) find_package(Zth) file(GLOB _stores ${LIBSTORED_STORES_CMAKE_PATH}/*Store.cmake) diff --git a/cmake/libstored.cmake b/cmake/libstored.cmake index 5a74387a..5d2e4f4c 100644 --- a/cmake/libstored.cmake +++ b/cmake/libstored.cmake @@ -34,6 +34,10 @@ option(LIBSTORED_HAVE_LIBZMQ "Use libzmq" OFF) # from source. option(LIBSTORED_HAVE_ZTH "Use Zth" OFF) +# When enabled, TinyAES is used for AES-256 CTR. It is searched via find_package(TinyAES). When +# provided by the system, make sure it is configured for AES-256 CTR mode. +option(LIBSTORED_HAVE_AES "Use AES-256 CTR from TinyAES" OFF) + # ################################################################################################## # Prepare environment @@ -198,6 +202,7 @@ Relationship: SPDXRef-compiler BUILD_DEPENDENCY_OF SPDXRef-libstored ${LIBSTORED_SOURCE_DIR}/include/stored ${LIBSTORED_SOURCE_DIR}/include/stored.h ${LIBSTORED_SOURCE_DIR}/include/stored_config.h + ${LIBSTORED_SOURCE_DIR}/include/libstored/aes.h ${LIBSTORED_SOURCE_DIR}/include/libstored/allocator.h ${LIBSTORED_SOURCE_DIR}/include/libstored/compress.h ${LIBSTORED_SOURCE_DIR}/include/libstored/config.h @@ -212,6 +217,7 @@ Relationship: SPDXRef-compiler BUILD_DEPENDENCY_OF SPDXRef-libstored ${LIBSTORED_SOURCE_DIR}/include/libstored/util.h ${LIBSTORED_SOURCE_DIR}/include/libstored/version.h ${LIBSTORED_SOURCE_DIR}/include/libstored/zmq.h + ${LIBSTORED_SOURCE_DIR}/src/aes.cpp ${LIBSTORED_SOURCE_DIR}/src/compress.cpp ${LIBSTORED_SOURCE_DIR}/src/directory.cpp ${LIBSTORED_SOURCE_DIR}/src/debugger.cpp @@ -442,6 +448,46 @@ Relationship: SPDXRef-libstored DEPENDS_ON SPDXRef-heatshrink ) endif() + if(LIBSTORED_HAVE_AES) + target_compile_definitions(${LIBSTORED_LIB_TARGET} PUBLIC -DSTORED_HAVE_AES=1) + target_link_libraries(${LIBSTORED_LIB_TARGET} PUBLIC tinyaes) + + set(_fields) + + if("${TinyAES_VERSION}" STREQUAL "") + set(_fields + "${_fields} +PackageVersion: preinstalled +PackageDownloadLocation: NOASSERTION +ExternalRef: PACKAGE-MANAGER purl pkg:github/kokke/tiny-AES-c" + ) + else() + set(_fields + "${_fields} +PackageVersion: ${TinyAES_VERSION} +PackageDownloadLocation: https://github.com/kokke/tiny-AES-c/commit/${TinyAES_VERSION} +ExternalRef: PACKAGE-MANAGER purl pkg:github/kokke/tiny-AES-c@${TinyAES_VERSION}" + ) + endif() + + file( + APPEND "${LIBSTORED_LIB_SBOM_CMAKE}" + " + file(APPEND \"${LIBSTORED_LIB_DESTINATION}/doc/sbom.spdx\" \" +PackageName: TinyAES +SPDXID: SPDXRef-TinyAES${_fields} +PackageHomePage: https://github.com/kokke/tiny-AES-c +FilesAnalyzed: false +PackageLicenseConcluded: Unlicense +PackageLicenseDeclared: Unlicense +PackageSummary: This is a small and portable implementation of the AES ECB, CTR and CBC encryption algorithms written in C. +PrimaryPackagePurpose: LIBRARY +Relationship: SPDXRef-libstored DEPENDS_ON SPDXRef-TinyAES +\") + " + ) + endif() + set(DO_CLANG_TIDY "") if(${CMAKE_VERSION} VERSION_GREATER "3.6.0") @@ -621,12 +667,13 @@ function(libstored_copy_dlls target) if(WIN32) get_target_property(target_type ${target} TYPE) get_property(LIBSTORED_RUNTIME_LIBS GLOBAL PROPERTY LIBSTORED_RUNTIME_LIBS) - if(target_type STREQUAL "EXECUTABLE" AND NOT "${LIBSTORED_RUNTIME_LIBS}" STREQUAL "") + if(target_type STREQUAL "EXECUTABLE" AND NOT "${LIBSTORED_RUNTIME_LIBS}" STREQUAL + "" + ) add_custom_command( TARGET ${target} PRE_LINK - COMMAND - ${CMAKE_COMMAND} -E copy ${LIBSTORED_RUNTIME_LIBS} + COMMAND ${CMAKE_COMMAND} -E copy ${LIBSTORED_RUNTIME_LIBS} $/ VERBATIM ) diff --git a/dist/common/FindTinyAES.cmake b/dist/common/FindTinyAES.cmake new file mode 100644 index 00000000..41dfcdb9 --- /dev/null +++ b/dist/common/FindTinyAES.cmake @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers +# +# SPDX-License-Identifier: MIT + +cmake_policy(VERSION 3.10) + +include(ExternalProject) +include(GNUInstallDirs) +find_package(Git) + +if("${TINYAES_GIT_URL}" STREQUAL "") + if(DEFINED ENV{LIBSTORED_GIT_CACHE}) + set(TINYAES_GIT_URL $ENV{LIBSTORED_GIT_CACHE}/tinyaes) + else() + set(TINYAES_GIT_URL "https://github.com/kokke/tiny-AES-c.git") + endif() +endif() + +set(TinyAES_VERSION 23856752fbd139da0b8ca6e471a13d5bcc99a08d) + +ExternalProject_Add( + tinyaes-extern + GIT_REPOSITORY ${TINYAES_GIT_URL} + GIT_TAG ${TinyAES_VERSION} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND ${CMAKE_COMMAND} -E chdir ${GIT_EXECUTABLE} checkout -- . + LOG_CONFIGURE 0 + LOG_BUILD 0 + LOG_TEST 0 + LOG_INSTALL 0 +) + +ExternalProject_Get_Property(tinyaes-extern SOURCE_DIR) + +# The source files are considered generated files. Upon a clean, they are removed. Hence the +# UPDATE_COMMAND to recover them. +add_library(tinyaes STATIC ${SOURCE_DIR}/aes.c) +set_target_properties(tinyaes PROPERTIES PUBLIC_HEADER "${SOURCE_DIR}/aes.h") +add_dependencies(tinyaes tinyaes-extern) + +get_target_property(tinyaes_src tinyaes SOURCES) +set_source_files_properties(${tinyaes_src} PROPERTIES GENERATED 1) + +if(MSVC) + target_compile_options(tinyaes PRIVATE /W1) + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_options(tinyaes PUBLIC /MTd) + else() + target_compile_options(tinyaes PUBLIC /MT) + endif() +endif() + +target_include_directories( + tinyaes PUBLIC $ $ +) + +target_compile_definitions(tinyaes PUBLIC AES256=1 CTR=1 CBC=0 ECB=0) + +install( + TARGETS tinyaes + EXPORT tinyaes + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) + +if(WIN32) + install(EXPORT tinyaes DESTINATION CMake) +else() + install(EXPORT tinyaes DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/libstored/cmake) +endif() + +set(TinyAES_FOUND 1) diff --git a/dist/common/build.sh b/dist/common/build.sh index 650f0db8..5e518fe4 100644 --- a/dist/common/build.sh +++ b/dist/common/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: 2020-2024 Jochem Rutgers +# SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers # # SPDX-License-Identifier: MPL-2.0 @@ -18,19 +18,25 @@ trap gotErr ERR function show_help { echo -e "Usage: $0 [...] [--] []\n" echo "where opt is:" - echo " Debug RelWithDebInfo Release" - echo " Set CMAKE_BUILD_TYPE to this value" - echo " C++98 C++03 C++11 C++14 C++17 C++20" - echo " Set the C++ standard" - echo " conf Configure only, don't build" - echo " dev Enable development-related options" - echo " test Enable building and running tests" - echo " zmq Enable ZeroMQ integration" - echo " nozmq Disable ZeroMQ integration" - echo " zth Enable Zth integration" - echo " fuzz Enable fuzzing with AFL++" - echo " gcov Enable gcov/lcov" - echo " clean Do a clean build" + echo " Debug RelWithDebInfo Release" + echo " Set CMAKE_BUILD_TYPE to this value" + echo " C++98 C++03 C++11 C++14 C++17 C++20" + echo " Set the C++ standard" + echo " conf Configure only, don't build" + echo " dev Enable development-related options" + echo " test Enable building and running tests" + echo " zmq Enable ZeroMQ integration" + echo " nozmq Disable ZeroMQ integration" + echo " heatshrink Enable Heatshrink integration" + echo " noheatshrink Disable Heatshrink integration" + echo " aes Enable TinyAES integration" + echo " noaes Disable TinyAES integration" + echo " san Enable sanitizers" + echo " nosan Disable sanitizers" + echo " zth Enable Zth integration" + echo " fuzz Enable fuzzing with AFL++" + echo " gcov Enable gcov/lcov" + echo " clean Do a clean build" exit 2 } @@ -99,6 +105,10 @@ while [[ ! -z ${1:-} ]]; do cmake_opts="${cmake_opts} -DLIBSTORED_HAVE_HEATSHRINK=ON";; noheatshrink) cmake_opts="${cmake_opts} -DLIBSTORED_HAVE_HEATSHRINK=OFF";; + aes) + cmake_opts="${cmake_opts} -DLIBSTORED_HAVE_AES=ON";; + noaes) + cmake_opts="${cmake_opts} -DLIBSTORED_HAVE_AES=OFF";; san) cmake_opts="${cmake_opts} -DLIBSTORED_ENABLE_ASAN=ON -DLIBSTORED_ENABLE_LSAN=ON -DLIBSTORED_ENABLE_UBSAN=ON";; nosan) diff --git a/dist/ubuntu/test_config.sh b/dist/ubuntu/test_config.sh index a47e1a23..b2d5871e 100755 --- a/dist/ubuntu/test_config.sh +++ b/dist/ubuntu/test_config.sh @@ -38,6 +38,8 @@ config Release nodev C++14 test config Release nodev C++17 test config Debug nodev noheatshrink test config Release noheatshrink test +config Debug nodev noaes test +config Release noaes test config Debug nodev noexamples test config Release nodev noexamples test config Debug nodev san zth test diff --git a/dist/win32/build.cmd b/dist/win32/build.cmd index 03fd61eb..523b8cc8 100644 --- a/dist/win32/build.cmd +++ b/dist/win32/build.cmd @@ -118,6 +118,22 @@ if %1 == nozmq ( set cmake_opts=%cmake_opts% -DLIBSTORED_HAVE_LIBZMQ=OFF goto next_param ) +if %1 == heatshrink ( + set cmake_opts=%cmake_opts% -DLIBSTORED_HAVE_HEATSHRINK=ON + goto next_param +) +if %1 == noheatshrink ( + set cmake_opts=%cmake_opts% -DLIBSTORED_HAVE_HEATSHRINK=OFF + goto next_param +) +if %1 == aes ( + set cmake_opts=%cmake_opts% -DLIBSTORED_HAVE_AES=ON + goto next_param +) +if %1 == noaes ( + set cmake_opts=%cmake_opts% -DLIBSTORED_HAVE_AES=OFF + goto next_param +) if %1 == zth ( set cmake_opts=%cmake_opts% -DLIBSTORED_HAVE_ZTH=ON goto next_param @@ -209,18 +225,22 @@ exit /b 0 echo Usage: %0 [^...] [--] [^] echo. echo where opt is: -echo Debug RelWithDebInfo Release -echo Set CMAKE_BUILD_TYPE to this value -echo gcc Use gcc instead of default compiler -echo C++98 C++03 C++11 C++14 C++17 C++20 -echo Set the C++ standard -echo conf Configure only, don't build -echo dev Enable development-related options -echo test Enable building and running tests -echo zmq Enable ZeroMQ integration -echo nozmq Disable ZeroMQ integration -echo zth Enable Zth integration -echo clean Do a clean build +echo Debug RelWithDebInfo Release +echo Set CMAKE_BUILD_TYPE to this value +echo gcc Use gcc instead of default compiler +echo C++98 C++03 C++11 C++14 C++17 C++20 +echo Set the C++ standard +echo conf Configure only, don't build +echo dev Enable development-related options +echo test Enable building and running tests +echo zmq Enable ZeroMQ integration +echo nozmq Disable ZeroMQ integration +echo heatshrink Enable Heatshrink integration +echo noheatshrink Disable Heatshrink integration +echo aes Enable TinyAES integration +echo noaes Disable TinyAES integration +echo zth Enable Zth integration +echo clean Do a clean build popd exit /b 2 goto silent_error diff --git a/examples/int_submodule/1_hello_submodule/CMakeLists.txt b/examples/int_submodule/1_hello_submodule/CMakeLists.txt index cd5f5846..d1863a51 100644 --- a/examples/int_submodule/1_hello_submodule/CMakeLists.txt +++ b/examples/int_submodule/1_hello_submodule/CMakeLists.txt @@ -15,8 +15,8 @@ list(APPEND CMAKE_MODULE_PATH ${LIBSTORED_CLONE}/cmake) # libstored requires python. Make sure that PYTHON_EXECUTABLE is set if you have a venv, otherwise # it is searched for using find_program(). -# If you want to have ZMQ, heatshrink, and Zth you have to do a find_package() yourself, and set -# LIBSTORED_HAVE_LIBZMQ (and friends) to ON. Refer to the list of options() at the top of of +# If you want to have ZMQ, heatshrink, TinyAES, and Zth you have to do a find_package() yourself, +# and set LIBSTORED_HAVE_LIBZMQ (and friends) to ON. Refer to the list of options() at the top of of # cmake/libstored.cmake for details. # Include all libstored functions. diff --git a/include/libstored/aes.h b/include/libstored/aes.h new file mode 100644 index 00000000..533efff5 --- /dev/null +++ b/include/libstored/aes.h @@ -0,0 +1,144 @@ +#ifndef LIBSTORED_AES_H +#define LIBSTORED_AES_H +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers +// +// SPDX-License-Identifier: MPL-2.0 + +#ifdef __cplusplus + +# include + +# ifdef STORED_HAVE_AES +# include + +namespace stored { + +/*! + * \brief A protocol layer that encrypts data using AES-256 in CTR mode. + * + * The pre-shared key should be provided using #setKey() before connecting. Changing the key during + * an active connection implies a reconnection. + * + * The first message after connecting is a random IV for decryption. After that, both sides send + * encrypted data. The last byte of the decoded data indicates the number of bytes to be stripped + * off (including the last byte itself). + * + * This layer assumes that the data through the stack is properly framed. For example, it runs on + * top of a #stored::ZmqLayer, #stored::TerminalLayer, or #stored::ArqLayer. + * + * The IV is based on a pseudo-random number generator. Make sure to call \c srand() before. + */ +class Aes256BaseLayer : public ProtocolLayer { + STORED_CLASS_NOCOPY(Aes256BaseLayer) +public: + typedef ProtocolLayer base; + + static char const CmdReset = 'R'; + + enum { KeySize = 32, BlockSize = 16 }; + +protected: + Aes256BaseLayer( + void const* key, ProtocolLayer* up = nullptr, ProtocolLayer* down = nullptr); + virtual ~Aes256BaseLayer() override is_default + +public: + virtual void decode(void* buffer, size_t len) override; + virtual void encode(void const* buffer, size_t len, bool last = true) override; +# ifndef DOXYGEN + using base::encode; +# endif + + virtual size_t mtu() const override; + virtual bool flush() override; + virtual void reset() override; + virtual void connected() override; + virtual void disconnected() override; + int lastError() const noexcept; + + void setKey(void const* key); + void fillRandom(uint8_t* buffer, size_t len) noexcept; + +protected: + void sendIV() noexcept; + + /*! + * \brief Low-level initialization. + * \return 0 on success, otherwise an errno + */ + virtual int + init(uint8_t const* key, uint8_t const* iv_enc, uint8_t const* iv_dec) noexcept = 0; + + /*! + * \brief Decrypt data in \p buffer. + * \param buffer the data to decrypt + * \param len the length of \p buffer + * \return 0 on success, otherwise an errno + * + * This function is expected to call \c base::decode() with the decrypted data, without the + * padding bytes. In-place decryption is allowed. + */ + virtual int decrypt(uint8_t* buffer, size_t len) noexcept = 0; + + /*! + * \brief Encrypt data in \p buffer. + * \param buffer the data to encrypt + * \param len the length of \p buffer, which is always a multiple of #BlockSize + * \param last whether this is the last block of data + * \return 0 on success, otherwise an errno + * + * This function is expected to call \c base::encode() with the encrypted data. In-place + * encryption is not allowed. + */ + virtual int encrypt(uint8_t const* buffer, size_t len, bool last) noexcept = 0; + +private: + uint8_t m_key[KeySize]; + uint8_t m_iv_enc[BlockSize]; + uint8_t m_iv_dec[BlockSize]; + uint8_t m_buffer[BlockSize]; + size_t m_bufferLen; + + enum State { + StateDisconnected, + StateConnected, + StateAwaitIV, + StateReady, + StateEncoding, + }; + State m_state; + int m_lastError; +# ifdef STORED_OS_POSIX + unsigned int m_seed; +# endif // STORED_OS_POSIX +}; + +/*! + * \brief A protocol layer that encrypts data using AES-256 in CTR mode using tiny-AES-c. + * + * This is a software implementation of AES-256. You may want to provide a hardware-accelerated + * layer, if your platform supports that. + */ +class Aes256Layer : public Aes256BaseLayer { + STORED_CLASS_NOCOPY(Aes256Layer) +public: + typedef Aes256BaseLayer base; + + Aes256Layer(ProtocolLayer* up = nullptr, ProtocolLayer* down = nullptr); + virtual ~Aes256Layer() override; + +protected: + virtual int + init(uint8_t const* key, uint8_t const* iv_enc, uint8_t const* iv_dec) noexcept override; + virtual int decrypt(uint8_t* buffer, size_t len) noexcept override; + virtual int encrypt(uint8_t const* buffer, size_t len, bool last) noexcept override; + +private: + void* m_ctx_enc; + void* m_ctx_dec; +}; + +} // namespace stored +# endif // STORED_HAVE_AES +#endif // __cplusplus +#endif // LIBSTORED_AES_H diff --git a/include/stored.h b/include/stored.h index 1aeeb4ce..d6cb7376 100644 --- a/include/stored.h +++ b/include/stored.h @@ -1,10 +1,12 @@ #ifndef LIBSTORED_STORED_H #define LIBSTORED_STORED_H -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 #include + +#include #include #include #include diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 3c2d4ac7..6aa67c69 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +# SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers # # SPDX-License-Identifier: MPL-2.0 @@ -49,6 +49,7 @@ set(package_data_ cmake/FindLibstored.cmake.tmpl cmake/LibstoredStoresConfig.cmake.in cmake/libstored.cmake + dist/common/FindTinyAES.cmake dist/common/FindHeatshrink.cmake dist/common/FindZeroMQ.cmake dist/common/FindZth.cmake diff --git a/sphinx/Doxyfile.in b/sphinx/Doxyfile.in index ce9b4986..cbd464cc 100644 --- a/sphinx/Doxyfile.in +++ b/sphinx/Doxyfile.in @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +# SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers # # SPDX-License-Identifier: CC0-1.0 @@ -2083,6 +2083,7 @@ PREDEFINED = DOXYGEN \ _DEBUG \ STORED_HAVE_ZMQ \ STORED_HAVE_HEATSHRINK \ + STORED_HAVE_AES \ STORED_DRAFT_API \ zth_fiber= \ UNUSED(x)= \ diff --git a/sphinx/doc/cpp_protocol.rst b/sphinx/doc/cpp_protocol.rst index 983917f7..c30a9ae0 100644 --- a/sphinx/doc/cpp_protocol.rst +++ b/sphinx/doc/cpp_protocol.rst @@ -94,6 +94,7 @@ The inheritance of the layers is shown below. ProtocolLayer <|-- IdleCheckLayer ProtocolLayer <|-- CallbackLayer ProtocolLayer <|-- MuxLayer + ProtocolLayer <|-- Aes256Layer abstract ArqLayer SegmentationLayer -[hidden]--> ArqLayer @@ -134,6 +135,11 @@ The inheritance of the layers is shown below. FifoLoopback --> FifoLoopback1 +stored::Aes256Layer +------------------- + +.. doxygenclass:: stored::Aes256Layer + stored::AsciiEscapeLayer ------------------------ diff --git a/src/aes.cpp b/src/aes.cpp new file mode 100644 index 00000000..ca13a799 --- /dev/null +++ b/src/aes.cpp @@ -0,0 +1,309 @@ +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers +// +// SPDX-License-Identifier: MPL-2.0 + +#include + +#ifdef STORED_HAVE_AES + +extern "C" { +# include +} // extern "C" + +# ifdef STORED_OS_POSIX +# include +# endif // STORED_OS_POSIX + + +namespace stored { + + + +//////////////////////////////////////////////////////////////// +// Aes256BaseLayer +// + +Aes256BaseLayer::Aes256BaseLayer(void const* key, ProtocolLayer* up, ProtocolLayer* down) + : base(up, down) + , m_key() + , m_iv_enc() + , m_iv_dec() + , m_buffer() + , m_bufferLen() + , m_state(StateDisconnected) + , m_lastError(ENOTCONN) +# ifdef STORED_OS_POSIX + // NOLINTNEXTLINE + , m_seed((unsigned int)(uintptr_t)this ^ (unsigned int)time(nullptr)) +# endif // STORED_OS_POSIX +{ + setKey(key); +} + +bool Aes256BaseLayer::flush() +{ + bool res = false; + + switch(m_state) { + case StateConnected: + sendIV(); + res = true; + m_state = StateAwaitIV; + break; + case StateDisconnected: + case StateAwaitIV: + case StateReady: + case StateEncoding: + default:; + // Nothing to do. + } + + return base::flush() || res; +} + +void Aes256BaseLayer::reset() +{ + m_state = StateDisconnected; + m_lastError = ENOTCONN; + base::reset(); +} + +void Aes256BaseLayer::connected() +{ + m_state = StateConnected; + m_lastError = 0; + m_bufferLen = 0; +} + +void Aes256BaseLayer::disconnected() +{ + m_state = StateDisconnected; + m_lastError = ENOTCONN; + base::disconnected(); +} + +void Aes256BaseLayer::decode(void* buffer, size_t len) +{ + uint8_t* buf = static_cast(buffer); + + switch(m_state) { + case StateDisconnected: + // Ignore data. + return; + case StateConnected: +connected: + sendIV(); + m_state = StateAwaitIV; + STORED_FALLTHROUGH + case StateAwaitIV: + // Expect IV. + if(len != BlockSize + 1 || buf[0] != CmdReset) { + // Invalid command. + m_lastError = EINVAL; + return; + } + + memcpy(m_iv_dec, buf + 1, BlockSize); + if((m_lastError = init(m_key, m_iv_enc, m_iv_dec)) != 0) { + // Initialization error. + m_state = StateDisconnected; + return; + } + + m_state = StateReady; + base::connected(); + STORED_FALLTHROUGH + case StateReady: + case StateEncoding: { + // Decrypt data. + if(len == 0) { + // Nothing to do. + return; + } + if(len == BlockSize + 1 && buf[0] == CmdReset) { + // Re-initialization. + m_state = StateConnected; + goto connected; + } + if(len % BlockSize != 0) { + // Invalid block. + m_lastError = EINVAL; + m_state = StateDisconnected; + base::disconnected(); + return; + } + + if((m_lastError = decrypt(buf + 1, len - 1)) != 0) { + // Decryption error. + m_state = StateDisconnected; + base::disconnected(); + return; + } + + // decrypt() should have called base::decode(). + break; + } + default:; + } +} + +void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) +{ + switch(m_state) { + case StateDisconnected: + // Ignore data. + return; + case StateConnected: + sendIV(); + m_state = StateAwaitIV; + STORED_FALLTHROUGH + case StateAwaitIV: + // Can't send data before IV exchange. + m_lastError = EINVAL; + return; + case StateReady: { + // Encrypt data. + if(len == 0) { + // Nothing to do. + return; + } + + m_state = StateEncoding; + m_bufferLen = 0; + STORED_FALLTHROUGH + } + case StateEncoding: { + uint8_t const* buffer_ = static_cast(buffer); + int res = 0; + while(len && !res) { + if(likely(m_bufferLen == 0)) { + size_t chunk = len & ~(BlockSize - 1); + if(likely(chunk)) { + // Full chunks to encrypt directly. + res = encrypt(buffer_, chunk, false); + len -= chunk; + buffer_ += chunk; + continue; + } + } + + // We have partial data in the buffer. + // Copy first. + size_t copy = BlockSize - m_bufferLen; + if(copy > len) + copy = len; + memcpy(m_buffer + m_bufferLen, buffer_, copy); + m_bufferLen += copy; + len -= copy; + buffer_ += copy; + if(m_bufferLen == BlockSize) { + // Encrypt full buffer. + res = encrypt(m_buffer, BlockSize, false); + m_bufferLen = 0; + continue; + } + } + + if(!res && last) { + // Finalize. + stored_assert(m_bufferLen < BlockSize); + + // Add PKCS#7 padding. + size_t padding = BlockSize - m_bufferLen % BlockSize; + for(size_t i = m_bufferLen; i < BlockSize; ++i) + m_buffer[i] = static_cast(padding); + + res = encrypt(m_buffer, BlockSize, true); + m_state = StateReady; + } + + if(res) { + // Encryption error. + m_lastError = res; + m_state = StateDisconnected; + base::disconnected(); + + if(last) + base::encode(nullptr, 0, true); + } + break; + } + default:; + } +} + +/*! + * \brief Send out the initialization vector for encryption (so for decryption by the peer). + */ +void Aes256BaseLayer::sendIV() noexcept +{ + fillRandom(m_iv_enc, BlockSize); + uint8_t cmd = (uint8_t)CmdReset; + base::encode(&cmd, 1, false); + base::encode(m_iv_enc, BlockSize, true); +} + +/*! + * \brief Set the pre-shared key. + * + * Terminates the current connection if any. + */ +void Aes256BaseLayer::setKey(void const* key) +{ + memcpy(m_key, key, KeySize); + + switch(m_state) { + case StateDisconnected: + // Nothing to do. + break; + case StateReady: + case StateEncoding: + m_state = StateDisconnected; + base::disconnected(); + STORED_FALLTHROUGH + case StateConnected: + case StateAwaitIV: + default: + sendIV(); + m_state = StateAwaitIV; + break; + } +} + +/*! + * \brief Fill \p buffer with \p len pseudo-random bytes. + */ +void Aes256BaseLayer::fillRandom(uint8_t* buffer, size_t len) noexcept +{ + stored_assert(len == 0 || buffer); + + for(size_t i = 0; i < len; ++i) { + buffer[i] = (uint8_t) +# ifdef STORED_OS_POSIX + rand_r(&m_seed); +# else // !STORED_OS_POSIX + rand(); +# endif // !STORED_OS_POSIX + } +} + + + +//////////////////////////////////////////////////////////////// +// Aes256Layer using tiny-AES-c +// + +# if !AES256 +# error "AES256 not defined in aes.h" +# endif + +# if !CTR +# error "CTR not defined in aes.h" +# endif + +static_assert(Aes256Layer::KeySize == AES_KEYLEN, ""); + +} // namespace stored +#else // !STORED_HAVE_AES +char dummy_char_to_make_aes_cpp_non_empty; // NOLINT +#endif // STORED_HAVE_AES diff --git a/tests/test_protocol.cpp b/tests/test_protocol.cpp index efb7e793..e30924cd 100644 --- a/tests/test_protocol.cpp +++ b/tests/test_protocol.cpp @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MPL-2.0 +#include "libstored/aes.h" #include "libstored/compress.h" #include "libstored/fifo.h" #include "libstored/protocol.h" @@ -1505,5 +1506,31 @@ TEST(MuxLayer, Decode) EXPECT_EQ(ch0.decoded().size(), 3); } +TEST(Aes256Layer, EncodeDecode) +{ + std::string key = "some very secure key of 32 bytes"; + ASSERT_EQ(key.size(), stored::Aes256Layer::KeySize); + + stored::Aes256Layer a; + a.setKey((uint8_t const*)key.data()); + LoggingLayer la; + la.stack(a); + + stored::PrintLayer p; + p.wrap(a); + + stored::Aes256Layer b; + b.setKey((uint8_t const*)key.data()); + LoggingLayer lb; + lb.stack(b); + + stored::Loopback l(p, b); + + la.encode("1", 1); + EXPECT_EQ(lb.decoded().at(0), "1"); + + lb.encode("2", 1); + EXPECT_EQ(la.decoded().at(0), "2"); +} } // namespace From 2e9a97506e30a4f96e1a9cb1e898d0b9cee2ed35 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Tue, 2 Dec 2025 17:48:05 +0100 Subject: [PATCH 02/19] actual implement aes --- include/libstored/aes.h | 18 +++++--- src/aes.cpp | 96 ++++++++++++++++++++++++++++++++++++----- src/protocol.cpp | 4 +- tests/test_protocol.cpp | 3 +- 4 files changed, 102 insertions(+), 19 deletions(-) diff --git a/include/libstored/aes.h b/include/libstored/aes.h index 533efff5..3cea2dba 100644 --- a/include/libstored/aes.h +++ b/include/libstored/aes.h @@ -38,8 +38,9 @@ class Aes256BaseLayer : public ProtocolLayer { enum { KeySize = 32, BlockSize = 16 }; protected: - Aes256BaseLayer( - void const* key, ProtocolLayer* up = nullptr, ProtocolLayer* down = nullptr); + explicit Aes256BaseLayer( + void const* key = nullptr, ProtocolLayer* up = nullptr, + ProtocolLayer* down = nullptr); virtual ~Aes256BaseLayer() override is_default public: @@ -49,7 +50,6 @@ class Aes256BaseLayer : public ProtocolLayer { using base::encode; # endif - virtual size_t mtu() const override; virtual bool flush() override; virtual void reset() override; virtual void connected() override; @@ -75,11 +75,13 @@ class Aes256BaseLayer : public ProtocolLayer { * \param len the length of \p buffer * \return 0 on success, otherwise an errno * - * This function is expected to call \c base::decode() with the decrypted data, without the + * This function is expected to call #decodeDecrypted() with the decrypted data, without the * padding bytes. In-place decryption is allowed. */ virtual int decrypt(uint8_t* buffer, size_t len) noexcept = 0; + void decodeDecrypted(void* buffer, size_t len); + /*! * \brief Encrypt data in \p buffer. * \param buffer the data to encrypt @@ -87,11 +89,13 @@ class Aes256BaseLayer : public ProtocolLayer { * \param last whether this is the last block of data * \return 0 on success, otherwise an errno * - * This function is expected to call \c base::encode() with the encrypted data. In-place + * This function is expected to call #encodeEncrypted() with the encrypted data. In-place * encryption is not allowed. */ virtual int encrypt(uint8_t const* buffer, size_t len, bool last) noexcept = 0; + void encodeEncrypted(void const* buffer, size_t len, bool last = true); + private: uint8_t m_key[KeySize]; uint8_t m_iv_enc[BlockSize]; @@ -124,7 +128,9 @@ class Aes256Layer : public Aes256BaseLayer { public: typedef Aes256BaseLayer base; - Aes256Layer(ProtocolLayer* up = nullptr, ProtocolLayer* down = nullptr); + explicit Aes256Layer( + void const* key = nullptr, ProtocolLayer* up = nullptr, + ProtocolLayer* down = nullptr); virtual ~Aes256Layer() override; protected: diff --git a/src/aes.cpp b/src/aes.cpp index ca13a799..27abde86 100644 --- a/src/aes.cpp +++ b/src/aes.cpp @@ -37,7 +37,8 @@ Aes256BaseLayer::Aes256BaseLayer(void const* key, ProtocolLayer* up, ProtocolLay , m_seed((unsigned int)(uintptr_t)this ^ (unsigned int)time(nullptr)) # endif // STORED_OS_POSIX { - setKey(key); + if(key) + setKey(key); } bool Aes256BaseLayer::flush() @@ -72,7 +73,6 @@ void Aes256BaseLayer::connected() { m_state = StateConnected; m_lastError = 0; - m_bufferLen = 0; } void Aes256BaseLayer::disconnected() @@ -86,15 +86,15 @@ void Aes256BaseLayer::decode(void* buffer, size_t len) { uint8_t* buf = static_cast(buffer); +again: switch(m_state) { case StateDisconnected: // Ignore data. return; case StateConnected: -connected: - sendIV(); m_state = StateAwaitIV; - STORED_FALLTHROUGH + sendIV(); + goto again; case StateAwaitIV: // Expect IV. if(len != BlockSize + 1 || buf[0] != CmdReset) { @@ -112,7 +112,7 @@ void Aes256BaseLayer::decode(void* buffer, size_t len) m_state = StateReady; base::connected(); - STORED_FALLTHROUGH + break; case StateReady: case StateEncoding: { // Decrypt data. @@ -123,7 +123,7 @@ void Aes256BaseLayer::decode(void* buffer, size_t len) if(len == BlockSize + 1 && buf[0] == CmdReset) { // Re-initialization. m_state = StateConnected; - goto connected; + goto again; } if(len % BlockSize != 0) { // Invalid block. @@ -133,7 +133,7 @@ void Aes256BaseLayer::decode(void* buffer, size_t len) return; } - if((m_lastError = decrypt(buf + 1, len - 1)) != 0) { + if((m_lastError = decrypt(buf, len)) != 0) { // Decryption error. m_state = StateDisconnected; base::disconnected(); @@ -147,16 +147,33 @@ void Aes256BaseLayer::decode(void* buffer, size_t len) } } +/*! + * \brief Pass decrypted data upstream. + */ +void Aes256BaseLayer::decodeDecrypted(void* buffer, size_t len) +{ + base::decode(buffer, len); +} + +/*! + * \brief Pass encrypted data downstream. + */ +void Aes256BaseLayer::encodeEncrypted(void const* buffer, size_t len, bool last) +{ + base::encode(buffer, len, last); +} + void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) { +again: switch(m_state) { case StateDisconnected: // Ignore data. return; case StateConnected: - sendIV(); m_state = StateAwaitIV; - STORED_FALLTHROUGH + sendIV(); + goto again; case StateAwaitIV: // Can't send data before IV exchange. m_lastError = EINVAL; @@ -250,6 +267,8 @@ void Aes256BaseLayer::sendIV() noexcept */ void Aes256BaseLayer::setKey(void const* key) { + stored_assert(key); + memcpy(m_key, key, KeySize); switch(m_state) { @@ -303,6 +322,63 @@ void Aes256BaseLayer::fillRandom(uint8_t* buffer, size_t len) noexcept static_assert(Aes256Layer::KeySize == AES_KEYLEN, ""); +Aes256Layer::Aes256Layer(void const* key, ProtocolLayer* up, ProtocolLayer* down) + : base(key, up, down) + , m_ctx_enc() + , m_ctx_dec() +{ + m_ctx_enc = new struct AES_ctx; + m_ctx_dec = new struct AES_ctx; +} + +Aes256Layer::~Aes256Layer() +{ + delete static_cast(m_ctx_enc); + delete static_cast(m_ctx_dec); +} + +int Aes256Layer::init(uint8_t const* key, uint8_t const* iv_enc, uint8_t const* iv_dec) noexcept +{ + AES_init_ctx_iv(static_cast(m_ctx_enc), key, iv_enc); + AES_init_ctx_iv(static_cast(m_ctx_dec), key, iv_dec); + return 0; +} + +int Aes256Layer::decrypt(uint8_t* buffer, size_t len) noexcept +{ + if(!len) + return 0; + + stored_assert(len % BlockSize == 0); + stored_assert(buffer); + AES_CTR_xcrypt_buffer(static_cast(m_ctx_dec), buffer, len); + + size_t padding = buffer[len - 1]; + if(padding >= len) + return 0; + + decodeDecrypted(buffer, len - padding); + return 0; +} + +int Aes256Layer::encrypt(uint8_t const* buffer, size_t len, bool last) noexcept +{ + stored_assert(!len || buffer); + stored_assert(len % BlockSize == 0); + + uint8_t buf[BlockSize]; + for(size_t offset = 0; offset < len; offset += BlockSize) { + memcpy(buf, buffer + offset, BlockSize); + AES_CTR_xcrypt_buffer(static_cast(m_ctx_enc), buf, BlockSize); + encodeEncrypted(buf, BlockSize, false); + } + + if(last) + encodeEncrypted(nullptr, 0, last); + + return 0; +} + } // namespace stored #else // !STORED_HAVE_AES char dummy_char_to_make_aes_cpp_non_empty; // NOLINT diff --git a/src/protocol.cpp b/src/protocol.cpp index cea4c143..822dfce7 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -2356,7 +2356,9 @@ void impl::Loopback1::encode(void const* buffer, size_t len, bool last) Loopback::Loopback(ProtocolLayer& a, ProtocolLayer& b) : m_a2b(a, b) , m_b2a(b, a) -{} +{ + a.connected(); +} /*! * \brief Reserve heap memory to assemble partial messages. diff --git a/tests/test_protocol.cpp b/tests/test_protocol.cpp index e30924cd..aeb1e584 100644 --- a/tests/test_protocol.cpp +++ b/tests/test_protocol.cpp @@ -1519,8 +1519,7 @@ TEST(Aes256Layer, EncodeDecode) stored::PrintLayer p; p.wrap(a); - stored::Aes256Layer b; - b.setKey((uint8_t const*)key.data()); + stored::Aes256Layer b((uint8_t const*)key.data()); LoggingLayer lb; lb.stack(b); From d1d44d11fbab58210eaa96b2b4cc6371e0745b9b Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:09:07 +0100 Subject: [PATCH 03/19] fix AES --- CHANGELOG.rst | 1 + examples/lossy_sync/main.cpp | 28 ++++- include/libstored/aes.h | 51 +++++--- include/libstored/synchronizer.h | 1 + sphinx/doc/cpp_protocol.rst | 8 +- src/aes.cpp | 209 ++++++++++++++++--------------- src/synchronizer.cpp | 5 + tests/test_protocol.cpp | 48 +++++++ 8 files changed, 225 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a34be5f9..6f6a74c5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,6 +27,7 @@ Added - Operations such as ``-=`` and ``++`` for store variables in C++. - YAML export of store meta-data. - ``stored::MuxLayer`` to multiplex multiple protocol layers over a single connection. +- ``stored::Aes256Layer`` for encrypted communication using AES-256 CTR. Changed ``````` diff --git a/examples/lossy_sync/main.cpp b/examples/lossy_sync/main.cpp index 119039ef..e559f442 100644 --- a/examples/lossy_sync/main.cpp +++ b/examples/lossy_sync/main.cpp @@ -125,7 +125,9 @@ class ExampleSync : public STORE_T(ExampleSync, stored::Synchronizable, stored:: static void print_help(FILE* out, char const* progname) { - fprintf(out, "Usage: %s [-h] [-v] [-p ] {-s |-c } [-b ]\n", + fprintf(out, + "Usage: %s [-h] [-v] [-p ] {-s |-c } [-b ] [-e " + "]\n", progname); fprintf(out, "where\n"); fprintf(out, " -h Show this help message.\n"); @@ -136,6 +138,7 @@ static void print_help(FILE* out, char const* progname) stored::DebugZmqLayer::DefaultPort); fprintf(out, " -v Verbose output of sync connections.\n"); fprintf(out, " -b Bit error rate (BER) for lossy channel. Default: 0\n"); + fprintf(out, " -e Encrypt communication with specified AES-256 key.\n"); } struct Arguments { @@ -143,6 +146,7 @@ struct Arguments { int debug_port = stored::DebugZmqLayer::DefaultPort; std::string client; std::string server; + std::string key; float ber = 0; }; @@ -154,7 +158,7 @@ static Arguments parse_arguments(int argc, char** argv) int c; // flawfinder: ignore - while((c = getopt(argc, argv, "hs:c:p:vb:")) != -1) { + while((c = getopt(argc, argv, "hs:c:p:vb:e:")) != -1) { switch(c) { case 'p': try { @@ -187,6 +191,10 @@ static Arguments parse_arguments(int argc, char** argv) STORED_throw(std::invalid_argument{e.what()}); } break; + case 'e': + args.key = optarg; + args.key.resize(stored::Aes256Layer::KeySize); + break; case 'h': print_help(stdout, argv[0]); STORED_throw(exit_now()); @@ -280,7 +288,8 @@ class SyncStack { STORED_CLASS_NOCOPY(SyncStack) public: explicit SyncStack( - ExampleSync& store, char const* endpoint, bool server, bool verbose, float ber = 0) + ExampleSync& store, char const* endpoint, bool server, bool verbose = false, + float ber = 0, char const* key = nullptr) : m_zmqLayer(nullptr, endpoint, server) { if((errno = m_zmqLayer.lastError())) { @@ -306,6 +315,10 @@ class SyncStack { m_ch1->wrap(*ch1_print); mux.get()->map(1, *m_ch1); + if(key) + // Encrypt communication. + wrap(key); + // We don't want to do ARQ on large messages, so we segment them to some // appropriate size. wrap(32U); @@ -552,10 +565,13 @@ static void run(Arguments const& args, ExampleSync& store, DebugStack& debugStac std::unique_ptr syncStack; if(!args.client.empty()) { - syncStack.reset(new SyncStack{store, args.client.c_str(), false, args.verbose}); + syncStack.reset(new SyncStack{ + store, args.client.c_str(), false, args.verbose, 0, + args.key.empty() ? nullptr : args.key.c_str()}); } else if(!args.server.empty()) { - syncStack.reset( - new SyncStack{store, args.server.c_str(), true, args.verbose, args.ber}); + syncStack.reset(new SyncStack{ + store, args.server.c_str(), true, args.verbose, args.ber, + args.key.empty() ? nullptr : args.key.c_str()}); } stored::Poller poller; diff --git a/include/libstored/aes.h b/include/libstored/aes.h index 3cea2dba..c78a0a77 100644 --- a/include/libstored/aes.h +++ b/include/libstored/aes.h @@ -41,9 +41,10 @@ class Aes256BaseLayer : public ProtocolLayer { explicit Aes256BaseLayer( void const* key = nullptr, ProtocolLayer* up = nullptr, ProtocolLayer* down = nullptr); - virtual ~Aes256BaseLayer() override is_default public: + virtual ~Aes256BaseLayer() override is_default + virtual void decode(void* buffer, size_t len) override; virtual void encode(void const* buffer, size_t len, bool last = true) override; # ifndef DOXYGEN @@ -56,18 +57,23 @@ class Aes256BaseLayer : public ProtocolLayer { virtual void disconnected() override; int lastError() const noexcept; - void setKey(void const* key); + void setKey(void const* key) noexcept; void fillRandom(uint8_t* buffer, size_t len) noexcept; protected: void sendIV() noexcept; /*! - * \brief Low-level initialization. + * \brief Low-level initialization for #encrypt(). + * \return 0 on success, otherwise an errno + */ + virtual int initEncrypt(uint8_t const* key, uint8_t const* iv) noexcept = 0; + + /*! + * \brief Low-level initialization for #decrypt(). * \return 0 on success, otherwise an errno */ - virtual int - init(uint8_t const* key, uint8_t const* iv_enc, uint8_t const* iv_dec) noexcept = 0; + virtual int initDecrypt(uint8_t const* key, uint8_t const* iv) noexcept = 0; /*! * \brief Decrypt data in \p buffer. @@ -98,19 +104,32 @@ class Aes256BaseLayer : public ProtocolLayer { private: uint8_t m_key[KeySize]; - uint8_t m_iv_enc[BlockSize]; - uint8_t m_iv_dec[BlockSize]; uint8_t m_buffer[BlockSize]; size_t m_bufferLen; - enum State { - StateDisconnected, - StateConnected, - StateAwaitIV, - StateReady, - StateEncoding, + enum EncState +# if STORED_cplusplus >= 201103L + : uint8_t +# endif + { + EncStateDisconnected, + EncStateConnected, + EncStateReady, + EncStateEncoding, + }; + EncState m_encState; + + enum DecState +# if STORED_cplusplus >= 201103L + : uint8_t +# endif + { + DecStateDisconnected, + DecStateConnected, + DecStateReady, }; - State m_state; + DecState m_decState; + int m_lastError; # ifdef STORED_OS_POSIX unsigned int m_seed; @@ -134,8 +153,8 @@ class Aes256Layer : public Aes256BaseLayer { virtual ~Aes256Layer() override; protected: - virtual int - init(uint8_t const* key, uint8_t const* iv_enc, uint8_t const* iv_dec) noexcept override; + virtual int initEncrypt(uint8_t const* key, uint8_t const* iv) noexcept override; + virtual int initDecrypt(uint8_t const* key, uint8_t const* iv) noexcept override; virtual int decrypt(uint8_t* buffer, size_t len) noexcept override; virtual int encrypt(uint8_t const* buffer, size_t len, bool last) noexcept override; diff --git a/include/libstored/synchronizer.h b/include/libstored/synchronizer.h index 82310687..4332e311 100644 --- a/include/libstored/synchronizer.h +++ b/include/libstored/synchronizer.h @@ -560,6 +560,7 @@ class SyncConnection : public ProtocolLayer { virtual void reset() override; virtual void disconnected() override; + virtual void connected() override; protected: Id nextId(); diff --git a/sphinx/doc/cpp_protocol.rst b/sphinx/doc/cpp_protocol.rst index c30a9ae0..304468f2 100644 --- a/sphinx/doc/cpp_protocol.rst +++ b/sphinx/doc/cpp_protocol.rst @@ -236,7 +236,7 @@ stored::SegmentationLayer .. doxygenclass:: stored::SegmentationLayer stored::SerialLayer -------------------------- +------------------- .. doxygenclass:: stored::SerialLayer @@ -248,7 +248,7 @@ stored::StdioLayer stored::SyncZmqLayer -------------------- -.. doxygenclass:: stored::SyncZmqLayer +It is a typedef for ``stored::ZmqLayer``. stored::TerminalLayer --------------------- @@ -256,7 +256,7 @@ stored::TerminalLayer .. doxygenclass:: stored::TerminalLayer stored::XsimLayer ------------------------ +----------------- .. doxygenclass:: stored::XsimLayer @@ -266,7 +266,7 @@ stored::ZmqBaseLayer .. doxygenclass:: stored::ZmqBaseLayer stored::ZmqLayer --------------------- +---------------- .. doxygenclass:: stored::ZmqLayer diff --git a/src/aes.cpp b/src/aes.cpp index 27abde86..b1926ec5 100644 --- a/src/aes.cpp +++ b/src/aes.cpp @@ -6,12 +6,14 @@ #ifdef STORED_HAVE_AES +# include + extern "C" { # include } // extern "C" # ifdef STORED_OS_POSIX -# include +# include # endif // STORED_OS_POSIX @@ -26,35 +28,32 @@ namespace stored { Aes256BaseLayer::Aes256BaseLayer(void const* key, ProtocolLayer* up, ProtocolLayer* down) : base(up, down) , m_key() - , m_iv_enc() - , m_iv_dec() , m_buffer() , m_bufferLen() - , m_state(StateDisconnected) + , m_encState(EncStateDisconnected) + , m_decState(DecStateDisconnected) , m_lastError(ENOTCONN) # ifdef STORED_OS_POSIX // NOLINTNEXTLINE , m_seed((unsigned int)(uintptr_t)this ^ (unsigned int)time(nullptr)) # endif // STORED_OS_POSIX { - if(key) - setKey(key); + setKey(key); } bool Aes256BaseLayer::flush() { bool res = false; - switch(m_state) { - case StateConnected: + switch(m_encState) { + case EncStateConnected: + m_encState = EncStateReady; sendIV(); res = true; - m_state = StateAwaitIV; - break; - case StateDisconnected: - case StateAwaitIV: - case StateReady: - case StateEncoding: + STORED_FALLTHROUGH + case EncStateDisconnected: + case EncStateReady: + case EncStateEncoding: default:; // Nothing to do. } @@ -64,79 +63,72 @@ bool Aes256BaseLayer::flush() void Aes256BaseLayer::reset() { - m_state = StateDisconnected; + m_encState = EncStateDisconnected; + m_decState = DecStateDisconnected; m_lastError = ENOTCONN; base::reset(); } void Aes256BaseLayer::connected() { - m_state = StateConnected; + m_encState = EncStateConnected; + if(m_decState == DecStateDisconnected) + m_decState = DecStateConnected; m_lastError = 0; + base::connected(); } void Aes256BaseLayer::disconnected() { - m_state = StateDisconnected; - m_lastError = ENOTCONN; + m_encState = EncStateDisconnected; + m_decState = DecStateDisconnected; + if(!m_lastError) + m_lastError = ENOTCONN; base::disconnected(); } +int Aes256BaseLayer::lastError() const noexcept +{ + return m_lastError; +} + void Aes256BaseLayer::decode(void* buffer, size_t len) { uint8_t* buf = static_cast(buffer); -again: - switch(m_state) { - case StateDisconnected: + switch(m_decState) { + case DecStateDisconnected: // Ignore data. return; - case StateConnected: - m_state = StateAwaitIV; - sendIV(); - goto again; - case StateAwaitIV: - // Expect IV. - if(len != BlockSize + 1 || buf[0] != CmdReset) { - // Invalid command. - m_lastError = EINVAL; - return; - } - - memcpy(m_iv_dec, buf + 1, BlockSize); - if((m_lastError = init(m_key, m_iv_enc, m_iv_dec)) != 0) { - // Initialization error. - m_state = StateDisconnected; - return; - } - - m_state = StateReady; - base::connected(); - break; - case StateReady: - case StateEncoding: { + case DecStateConnected: + case DecStateReady: { // Decrypt data. if(len == 0) { // Nothing to do. return; } if(len == BlockSize + 1 && buf[0] == CmdReset) { - // Re-initialization. - m_state = StateConnected; - goto again; + // Got IV for decryption. + m_lastError = initDecrypt(m_key, buf + 1); + if(m_lastError) { + // Initialization error. + disconnected(); + return; + } + m_decState = DecStateReady; + return; } - if(len % BlockSize != 0) { + if(m_decState != DecStateReady || len % BlockSize != 0) { // Invalid block. m_lastError = EINVAL; - m_state = StateDisconnected; - base::disconnected(); + disconnected(); return; } - if((m_lastError = decrypt(buf, len)) != 0) { + m_lastError = decrypt(buf, len); + if(m_lastError) { // Decryption error. - m_state = StateDisconnected; - base::disconnected(); + disconnected(); return; } @@ -166,35 +158,31 @@ void Aes256BaseLayer::encodeEncrypted(void const* buffer, size_t len, bool last) void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) { again: - switch(m_state) { - case StateDisconnected: + switch(m_encState) { + case EncStateDisconnected: // Ignore data. return; - case StateConnected: - m_state = StateAwaitIV; + case EncStateConnected: + m_encState = EncStateReady; sendIV(); goto again; - case StateAwaitIV: - // Can't send data before IV exchange. - m_lastError = EINVAL; - return; - case StateReady: { + case EncStateReady: { // Encrypt data. if(len == 0) { // Nothing to do. return; } - m_state = StateEncoding; + m_encState = EncStateEncoding; m_bufferLen = 0; STORED_FALLTHROUGH } - case StateEncoding: { + case EncStateEncoding: { uint8_t const* buffer_ = static_cast(buffer); int res = 0; while(len && !res) { if(likely(m_bufferLen == 0)) { - size_t chunk = len & ~(BlockSize - 1); + size_t chunk = len & ~((size_t)BlockSize - 1U); if(likely(chunk)) { // Full chunks to encrypt directly. res = encrypt(buffer_, chunk, false); @@ -228,20 +216,21 @@ void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) // Add PKCS#7 padding. size_t padding = BlockSize - m_bufferLen % BlockSize; for(size_t i = m_bufferLen; i < BlockSize; ++i) + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index) m_buffer[i] = static_cast(padding); res = encrypt(m_buffer, BlockSize, true); - m_state = StateReady; + m_encState = EncStateReady; } if(res) { // Encryption error. m_lastError = res; - m_state = StateDisconnected; - base::disconnected(); if(last) base::encode(nullptr, 0, true); + + disconnected(); } break; } @@ -254,38 +243,41 @@ void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) */ void Aes256BaseLayer::sendIV() noexcept { - fillRandom(m_iv_enc, BlockSize); - uint8_t cmd = (uint8_t)CmdReset; - base::encode(&cmd, 1, false); - base::encode(m_iv_enc, BlockSize, true); + uint8_t buf[BlockSize + 1]; + buf[0] = (uint8_t)CmdReset; + fillRandom(buf + 1, BlockSize); + + m_lastError = initEncrypt(m_key, buf + 1); + if(m_lastError) { + // Initialization error. + disconnected(); + return; + } + + base::encode(buf, sizeof(buf), true); } /*! * \brief Set the pre-shared key. * - * Terminates the current connection if any. + * Make sure to switch the key at the same time on both sides. */ -void Aes256BaseLayer::setKey(void const* key) +void Aes256BaseLayer::setKey(void const* key) noexcept { - stored_assert(key); - - memcpy(m_key, key, KeySize); + if(!key) + memset(m_key, 0, KeySize); + else + memcpy(m_key, key, KeySize); - switch(m_state) { - case StateDisconnected: + switch(m_encState) { + case EncStateDisconnected: // Nothing to do. break; - case StateReady: - case StateEncoding: - m_state = StateDisconnected; - base::disconnected(); - STORED_FALLTHROUGH - case StateConnected: - case StateAwaitIV: + case EncStateConnected: + case EncStateReady: + case EncStateEncoding: default: - sendIV(); - m_state = StateAwaitIV; - break; + m_encState = EncStateConnected; } } @@ -327,8 +319,17 @@ Aes256Layer::Aes256Layer(void const* key, ProtocolLayer* up, ProtocolLayer* down , m_ctx_enc() , m_ctx_dec() { + // NOLINTNEXTLINE m_ctx_enc = new struct AES_ctx; - m_ctx_dec = new struct AES_ctx; + + try { + // NOLINTNEXTLINE + m_ctx_dec = new struct AES_ctx; + } catch(...) { + delete static_cast(m_ctx_enc); + m_ctx_enc = nullptr; + STORED_rethrow; + } } Aes256Layer::~Aes256Layer() @@ -337,10 +338,17 @@ Aes256Layer::~Aes256Layer() delete static_cast(m_ctx_dec); } -int Aes256Layer::init(uint8_t const* key, uint8_t const* iv_enc, uint8_t const* iv_dec) noexcept +int Aes256Layer::initEncrypt(uint8_t const* key, uint8_t const* iv) noexcept { - AES_init_ctx_iv(static_cast(m_ctx_enc), key, iv_enc); - AES_init_ctx_iv(static_cast(m_ctx_dec), key, iv_dec); + struct AES_ctx* ctx = static_cast(m_ctx_enc); + AES_init_ctx_iv(ctx, key, iv); + return 0; +} + +int Aes256Layer::initDecrypt(uint8_t const* key, uint8_t const* iv) noexcept +{ + struct AES_ctx* ctx = static_cast(m_ctx_dec); + AES_init_ctx_iv(ctx, key, iv); return 0; } @@ -354,6 +362,10 @@ int Aes256Layer::decrypt(uint8_t* buffer, size_t len) noexcept AES_CTR_xcrypt_buffer(static_cast(m_ctx_dec), buffer, len); size_t padding = buffer[len - 1]; + if(padding == 0 || padding > BlockSize) + // Invalid padding. + return EINVAL; + if(padding >= len) return 0; @@ -367,15 +379,12 @@ int Aes256Layer::encrypt(uint8_t const* buffer, size_t len, bool last) noexcept stored_assert(len % BlockSize == 0); uint8_t buf[BlockSize]; - for(size_t offset = 0; offset < len; offset += BlockSize) { - memcpy(buf, buffer + offset, BlockSize); + for(; len; len -= BlockSize, buffer += BlockSize) { + memcpy(buf, buffer, BlockSize); AES_CTR_xcrypt_buffer(static_cast(m_ctx_enc), buf, BlockSize); - encodeEncrypted(buf, BlockSize, false); + encodeEncrypted(buf, BlockSize, last && len == BlockSize); } - if(last) - encodeEncrypted(nullptr, 0, last); - return 0; } diff --git a/src/synchronizer.cpp b/src/synchronizer.cpp index 3dd9c4c0..d4ea77cd 100644 --- a/src/synchronizer.cpp +++ b/src/synchronizer.cpp @@ -848,7 +848,12 @@ void SyncConnection::disconnected() { dropNonSources(); base::disconnected(); +} + +void SyncConnection::connected() +{ helloAgain(); + base::connected(); } /*! diff --git a/tests/test_protocol.cpp b/tests/test_protocol.cpp index aeb1e584..ec4df5f2 100644 --- a/tests/test_protocol.cpp +++ b/tests/test_protocol.cpp @@ -1516,8 +1516,13 @@ TEST(Aes256Layer, EncodeDecode) LoggingLayer la; la.stack(a); +#if 0 + // Print encrypted messages. stored::PrintLayer p; p.wrap(a); +#else + auto& p = a; +#endif stored::Aes256Layer b((uint8_t const*)key.data()); LoggingLayer lb; @@ -1525,11 +1530,54 @@ TEST(Aes256Layer, EncodeDecode) stored::Loopback l(p, b); + // Normal encode/decode. la.encode("1", 1); EXPECT_EQ(lb.decoded().at(0), "1"); + EXPECT_EQ(a.lastError(), 0); + EXPECT_EQ(b.lastError(), 0); lb.encode("2", 1); EXPECT_EQ(la.decoded().at(0), "2"); + + la.encode("3", 1); + EXPECT_EQ(lb.decoded().at(1), "3"); + + la.encode("0123456789abcdef", 16); + EXPECT_EQ(lb.decoded().at(2), "0123456789abcdef"); + + la.encode("0123456789abcdef0", 17); + EXPECT_EQ(lb.decoded().at(3), "0123456789abcdef0"); + + lb.encode("xyz", 3); + EXPECT_EQ(la.decoded().at(1), "xyz"); + EXPECT_EQ(a.lastError(), 0); + EXPECT_EQ(b.lastError(), 0); + + // Reconnect. + la.clear(); + lb.clear(); + a.connected(); + b.connected(); + + la.encode("abc", 3); + EXPECT_EQ(lb.decoded().at(0), "abc"); + EXPECT_EQ(a.lastError(), 0); + EXPECT_EQ(b.lastError(), 0); + + lb.encode("def", 3); + EXPECT_EQ(la.decoded().at(0), "def"); + + // Change key. + key = "another secure key of 32 bytes "; + ASSERT_EQ(key.size(), stored::Aes256Layer::KeySize); + a.setKey((uint8_t const*)key.data()); + b.setKey((uint8_t const*)key.data()); + + la.encode("ghij", 4); + EXPECT_EQ(lb.decoded().at(1), "ghij"); + + lb.encode("klmn", 4); + EXPECT_EQ(la.decoded().at(1), "klmn"); } } // namespace From 1a5fd01e08a156b3cb95ae1a73ebceb4a2025d5d Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Wed, 3 Dec 2025 15:59:24 +0100 Subject: [PATCH 04/19] optimize --- include/libstored/protocol.h | 2 ++ src/aes.cpp | 9 +++++++++ src/protocol.cpp | 25 ++++++++++++++++++------- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/include/libstored/protocol.h b/include/libstored/protocol.h index 354d912d..a5d99921 100644 --- a/include/libstored/protocol.h +++ b/include/libstored/protocol.h @@ -520,10 +520,12 @@ class SegmentationLayer : public ProtocolLayer { size_t mtu() const final; size_t lowerMtu() const; virtual void reset() override; + virtual void connected() override; virtual void disconnected() override; private: size_t m_mtu; + size_t m_lowerMtu; Vector::type m_decode; size_t m_encoded; }; diff --git a/src/aes.cpp b/src/aes.cpp index b1926ec5..42d54aeb 100644 --- a/src/aes.cpp +++ b/src/aes.cpp @@ -71,6 +71,15 @@ void Aes256BaseLayer::reset() void Aes256BaseLayer::connected() { + size_t mtu = this->mtu(); + if(mtu < BlockSize + 1) { + // MTU too small. + m_lastError = EMSGSIZE; + if(m_encState != EncStateDisconnected || m_decState != DecStateDisconnected) + disconnected(); + return; + } + m_encState = EncStateConnected; if(m_decState == DecStateDisconnected) m_decState = DecStateConnected; diff --git a/src/protocol.cpp b/src/protocol.cpp index 822dfce7..4cba5809 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -358,8 +358,11 @@ size_t TerminalLayer::mtu() const SegmentationLayer::SegmentationLayer(size_t mtu, ProtocolLayer* up, ProtocolLayer* down) : base(up, down) , m_mtu(mtu) + , m_lowerMtu() , m_encoded() -{} +{ + lowerMtu(); +} void SegmentationLayer::reset() { @@ -368,6 +371,18 @@ void SegmentationLayer::reset() base::reset(); } +void SegmentationLayer::connected() +{ + m_lowerMtu = lowerMtu(); + if(m_lowerMtu == 0) + m_lowerMtu = std::numeric_limits::max(); + else if(m_lowerMtu == 1) + m_lowerMtu = 2; + + m_encoded = 0; + base::connected(); +} + void SegmentationLayer::disconnected() { m_decode.clear(); @@ -423,14 +438,10 @@ void SegmentationLayer::encode(void const* buffer, size_t len, bool last) { char const* buffer_ = static_cast(buffer); - size_t mtu = lowerMtu(); - if(mtu == 0) - mtu = std::numeric_limits::max(); - else if(mtu == 1) - mtu = 2; + stored_assert(m_lowerMtu > m_encoded); while(len) { - size_t remaining = mtu - m_encoded - 1; + size_t remaining = m_lowerMtu - m_encoded - 1; size_t chunk = std::min(len, remaining); if(chunk) { From 3f14ae50702245eadd83c2b8eda1406458c31b2e Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Wed, 3 Dec 2025 15:59:39 +0100 Subject: [PATCH 05/19] adding aes --- dist/common/requirements-minimal.txt | 1 + python/libstored/protocol/protocol.py | 35 +++++++++++++++++++++++++++ python/setup.cfg | 1 + 3 files changed, 37 insertions(+) diff --git a/dist/common/requirements-minimal.txt b/dist/common/requirements-minimal.txt index bd97a293..e697eb88 100644 --- a/dist/common/requirements-minimal.txt +++ b/dist/common/requirements-minimal.txt @@ -10,6 +10,7 @@ jinja2 matplotlib natsort platformdirs +pycryptodome pyserial >= 3.1 pyzmq textx diff --git a/python/libstored/protocol/protocol.py b/python/libstored/protocol/protocol.py index 7c9270d3..66f73fdb 100644 --- a/python/libstored/protocol/protocol.py +++ b/python/libstored/protocol/protocol.py @@ -6,6 +6,7 @@ import asyncio import crcmod +import Crypto.Cipher.AES import inspect import logging import struct @@ -1105,6 +1106,39 @@ async def timeout(self) -> None: +class Aes256Layer(ProtocolLayer): + ''' + A ProtocolLayer that adds AES-256 encryption/decryption. + ''' + + name = 'aes256' + + def __init__(self, key : bytes | str | None=None, *args, **kwargs): + super().__init__(*args, **kwargs) + + if key is None: + raise ValueError('Key file or binary string must be provided for Aes256Layer') + + if isinstance(key, str): + with open(key, 'rb') as f: + key = f.read() + if len(key) != 32: + raise ValueError('Key must be 32 bytes for AES-256') + + self._key = key + self._encrypt = None + self._decrypt = None + + async def encode(self, data : ProtocolLayer.Packet) -> None: + # Placeholder for actual encryption logic + await super().encode(data) + + async def decode(self, data : ProtocolLayer.Packet) -> None: + # Placeholder for actual decryption logic + await super().decode(data) + + + layer_types : list[typing.Type[ProtocolLayer]] = [ AsciiEscapeLayer, TerminalLayer, @@ -1118,6 +1152,7 @@ async def timeout(self) -> None: LoopbackLayer, RawLayer, MuxLayer, + Aes256Layer, ] def register_layer_type(layer_type : typing.Type[ProtocolLayer]) -> None: diff --git a/python/setup.cfg b/python/setup.cfg index cba12b49..e15085cf 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -32,6 +32,7 @@ install_requires = matplotlib natsort platformdirs + pycryptodome pyserial >= 3.1 pyzmq textx From 3090fb45f7ef3b49e96439df7589ae8e627275bb Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Thu, 4 Dec 2025 00:13:42 +0100 Subject: [PATCH 06/19] read key from file --- examples/lossy_sync/main.cpp | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/examples/lossy_sync/main.cpp b/examples/lossy_sync/main.cpp index e559f442..aa7a14f7 100644 --- a/examples/lossy_sync/main.cpp +++ b/examples/lossy_sync/main.cpp @@ -127,7 +127,7 @@ static void print_help(FILE* out, char const* progname) { fprintf(out, "Usage: %s [-h] [-v] [-p ] {-s |-c } [-b ] [-e " - "]\n", + "]\n", progname); fprintf(out, "where\n"); fprintf(out, " -h Show this help message.\n"); @@ -138,7 +138,9 @@ static void print_help(FILE* out, char const* progname) stored::DebugZmqLayer::DefaultPort); fprintf(out, " -v Verbose output of sync connections.\n"); fprintf(out, " -b Bit error rate (BER) for lossy channel. Default: 0\n"); - fprintf(out, " -e Encrypt communication with specified AES-256 key.\n"); + fprintf(out, + " -e Encrypt communication with the %zu-byte AES-256 key, read from the file.\n", + (size_t)stored::Aes256Layer::KeySize); } struct Arguments { @@ -191,10 +193,25 @@ static Arguments parse_arguments(int argc, char** argv) STORED_throw(std::invalid_argument{e.what()}); } break; - case 'e': - args.key = optarg; + case 'e': { + FILE* f = fopen(optarg, "rb"); args.key.resize(stored::Aes256Layer::KeySize); + + if(!f) { + log("Cannot open key file '%s'; %s\n", optarg, strerror(errno)); + STORED_throw(std::invalid_argument{"Cannot open key file"}); + } + + if(fread(&args.key[0], stored::Aes256Layer::KeySize, 1, f) != 1) { + log("Cannot read key file '%s'; %s\n", optarg, strerror(errno)); + fclose(f); + STORED_throw(std::invalid_argument{"Cannot read key file"}); + } + + fclose(f); + log("Read AES-256 key from '%s'\n", optarg); break; + } case 'h': print_help(stdout, argv[0]); STORED_throw(exit_now()); From 015bad144a107faa0f4570f2e2079b0349e84eea Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Thu, 4 Dec 2025 00:13:49 +0100 Subject: [PATCH 07/19] fix mtu check --- src/aes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aes.cpp b/src/aes.cpp index 42d54aeb..ffca8562 100644 --- a/src/aes.cpp +++ b/src/aes.cpp @@ -72,7 +72,7 @@ void Aes256BaseLayer::reset() void Aes256BaseLayer::connected() { size_t mtu = this->mtu(); - if(mtu < BlockSize + 1) { + if(mtu && mtu < BlockSize + 1) { // MTU too small. m_lastError = EMSGSIZE; if(m_encState != EncStateDisconnected || m_decState != DecStateDisconnected) From cc3eee125c37b8abcaa16066bc899f98b9053585 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Thu, 4 Dec 2025 00:13:57 +0100 Subject: [PATCH 08/19] fix aes --- python/libstored/protocol/protocol.py | 56 ++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/python/libstored/protocol/protocol.py b/python/libstored/protocol/protocol.py index 66f73fdb..f07b1b0a 100644 --- a/python/libstored/protocol/protocol.py +++ b/python/libstored/protocol/protocol.py @@ -7,6 +7,8 @@ import asyncio import crcmod import Crypto.Cipher.AES +import Crypto.Random +import Crypto.Util.Padding import inspect import logging import struct @@ -1115,13 +1117,19 @@ class Aes256Layer(ProtocolLayer): def __init__(self, key : bytes | str | None=None, *args, **kwargs): super().__init__(*args, **kwargs) + self._encrypt = None + self._decrypt = None if key is None: raise ValueError('Key file or binary string must be provided for Aes256Layer') + self.set_key(key) + + def set_key(self, key : bytes | str) -> None: if isinstance(key, str): with open(key, 'rb') as f: key = f.read() + if len(key) != 32: raise ValueError('Key must be 32 bytes for AES-256') @@ -1130,13 +1138,57 @@ def __init__(self, key : bytes | str | None=None, *args, **kwargs): self._decrypt = None async def encode(self, data : ProtocolLayer.Packet) -> None: - # Placeholder for actual encryption logic + if isinstance(data, str): + data = data.encode() + elif isinstance(data, memoryview): + data = data.cast('B') + + if self._encrypt is None: + await self._send_iv() + + assert self._encrypt is not None + data = Crypto.Util.Padding.pad(data, 16) + data = self._encrypt.encrypt(data) await super().encode(data) async def decode(self, data : ProtocolLayer.Packet) -> None: - # Placeholder for actual decryption logic + if isinstance(data, str): + data = data.encode() + elif isinstance(data, memoryview): + data = data.cast('B') + + if len(data) == 17 and data[0:1] == b'R': + # Received IV for decryption + iv = data[1:17] + self._decrypt = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, iv) + self.logger.debug('Received IV for decryption') + return + + if self._decrypt is None: + self.logger.debug('Got data before IV, waiting for IV') + self._decrypt = None + return + + if len(data) % 16 != 0: + self.logger.debug('Data length not multiple of 16, dropped') + self._decrypt = None + return + + data = self._decrypt.decrypt(data) + try: + data = Crypto.Util.Padding.unpad(data, 16) + except ValueError: + self.logger.debug('Invalid padding, dropped') + self._decrypt = None + return + await super().decode(data) + async def _send_iv(self) -> None: + iv = Crypto.Random.get_random_bytes(16) + self._encrypt = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, iv) + await super().encode(b'R' + iv) + layer_types : list[typing.Type[ProtocolLayer]] = [ From 19583d95a6a43335db0cdb34040f33fc0775076e Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:24:06 +0100 Subject: [PATCH 09/19] add unified mode --- examples/lossy_sync/main.cpp | 18 ++++- include/libstored/aes.h | 52 +++++++++++--- src/aes.cpp | 130 ++++++++++++++++++++++------------- tests/test_protocol.cpp | 8 +++ 4 files changed, 148 insertions(+), 60 deletions(-) diff --git a/examples/lossy_sync/main.cpp b/examples/lossy_sync/main.cpp index aa7a14f7..a2bdff77 100644 --- a/examples/lossy_sync/main.cpp +++ b/examples/lossy_sync/main.cpp @@ -248,7 +248,8 @@ class disconnected : public std::exception {}; class DebugStack { STORED_CLASS_NOCOPY(DebugStack) public: - explicit DebugStack(ExampleSync& store, int port, char const* name = nullptr) + explicit DebugStack( + ExampleSync& store, int port, char const* name = nullptr, char const* key = nullptr) : m_debugLayer{nullptr, port} { if((errno = m_debugLayer.lastError())) { @@ -263,7 +264,16 @@ class DebugStack { m_debugger.setIdentification(m_id.c_str()); m_debugger.map(store); - m_debugLayer.wrap(m_debugger); + + if(key) { + // Encrypted debug channel. + m_aes.reset(new stored::Aes256Layer{key}); + m_aes->wrap(m_debugger); + m_debugLayer.wrap(*m_aes); + } else { + m_debugLayer.wrap(m_debugger); + } + logger_callback = [&](char const* msg) { m_debugger.stream('l', msg); }; } @@ -295,6 +305,7 @@ class DebugStack { std::string m_id; stored::Debugger m_debugger; stored::DebugZmqLayer m_debugLayer; + std::unique_ptr m_aes; stored::PollableZmqSocket m_pollable{m_debugLayer.socket(), stored::Pollable::PollIn}; }; @@ -642,7 +653,8 @@ int main(int argc, char** argv) ExampleSync store; DebugStack debugStack{ - store, args.debug_port, args.client.empty() ? "server" : "client"}; + store, args.debug_port, args.client.empty() ? "server" : "client", + args.key.empty() ? nullptr : args.key.c_str()}; while(true) { try { diff --git a/include/libstored/aes.h b/include/libstored/aes.h index c78a0a77..94d813cb 100644 --- a/include/libstored/aes.h +++ b/include/libstored/aes.h @@ -19,9 +19,14 @@ namespace stored { * The pre-shared key should be provided using #setKey() before connecting. Changing the key during * an active connection implies a reconnection. * - * The first message after connecting is a random IV for decryption. After that, both sides send - * encrypted data. The last byte of the decoded data indicates the number of bytes to be stripped - * off (including the last byte itself). + * The first (part of the) message after connecting is a random IV for decryption. After that, both + * sides send encrypted data. The last byte of the decoded data indicates the number of bytes to be + * stripped off (including the last byte itself). + * + * The unified mode uses the same key stream for all decrypt and encrypt operations. This would work + * fine in case of a REQ/REP (decode/encode) pattern, where the REQ/REP may start with a new IV. + * The non-unified mode uses two different key streams for decode (decrypt) and encode (encrypt). + * The modes are not mixed, without initializing the IV again. * * This layer assumes that the data through the stack is properly framed. For example, it runs on * top of a #stored::ZmqLayer, #stored::TerminalLayer, or #stored::ArqLayer. @@ -33,7 +38,8 @@ class Aes256BaseLayer : public ProtocolLayer { public: typedef ProtocolLayer base; - static char const CmdReset = 'R'; + static char const CmdBidirectional = 'B'; + static char const CmdUnified = 'U'; enum { KeySize = 32, BlockSize = 16 }; @@ -51,17 +57,18 @@ class Aes256BaseLayer : public ProtocolLayer { using base::encode; # endif - virtual bool flush() override; virtual void reset() override; virtual void connected() override; virtual void disconnected() override; int lastError() const noexcept; void setKey(void const* key) noexcept; + void unified(bool enable) noexcept; + bool unified() const noexcept; void fillRandom(uint8_t* buffer, size_t len) noexcept; protected: - void sendIV() noexcept; + void sendIV(bool unified, bool last = true) noexcept; /*! * \brief Low-level initialization for #encrypt(). @@ -75,16 +82,29 @@ class Aes256BaseLayer : public ProtocolLayer { */ virtual int initDecrypt(uint8_t const* key, uint8_t const* iv) noexcept = 0; + /*! + * \brief Low-level initialization for #encrypt() and #decrypt(), using the same IV and key + * stream. + * \return 0 on success, otherwise an errno + */ + virtual int initUnified(uint8_t const* key, uint8_t const* iv) noexcept = 0; + + /*! + * \brief Update the IV on the unified encrypt/decrypt key stream. + */ + virtual int updateUnified(uint8_t const* iv) noexcept = 0; + /*! * \brief Decrypt data in \p buffer. * \param buffer the data to decrypt * \param len the length of \p buffer + * \param unified when \c true, use the same key stream for encrypt and decrypt * \return 0 on success, otherwise an errno * * This function is expected to call #decodeDecrypted() with the decrypted data, without the * padding bytes. In-place decryption is allowed. */ - virtual int decrypt(uint8_t* buffer, size_t len) noexcept = 0; + virtual int decrypt(uint8_t* buffer, size_t len, bool unified) noexcept = 0; void decodeDecrypted(void* buffer, size_t len); @@ -93,12 +113,14 @@ class Aes256BaseLayer : public ProtocolLayer { * \param buffer the data to encrypt * \param len the length of \p buffer, which is always a multiple of #BlockSize * \param last whether this is the last block of data + * \param unified when \c true, use the same key stream for encrypt and decrypt * \return 0 on success, otherwise an errno * * This function is expected to call #encodeEncrypted() with the encrypted data. In-place * encryption is not allowed. */ - virtual int encrypt(uint8_t const* buffer, size_t len, bool last) noexcept = 0; + virtual int + encrypt(uint8_t const* buffer, size_t len, bool last, bool unified) noexcept = 0; void encodeEncrypted(void const* buffer, size_t len, bool last = true); @@ -130,6 +152,8 @@ class Aes256BaseLayer : public ProtocolLayer { }; DecState m_decState; + bool m_unified; + int m_lastError; # ifdef STORED_OS_POSIX unsigned int m_seed; @@ -155,11 +179,17 @@ class Aes256Layer : public Aes256BaseLayer { protected: virtual int initEncrypt(uint8_t const* key, uint8_t const* iv) noexcept override; virtual int initDecrypt(uint8_t const* key, uint8_t const* iv) noexcept override; - virtual int decrypt(uint8_t* buffer, size_t len) noexcept override; - virtual int encrypt(uint8_t const* buffer, size_t len, bool last) noexcept override; + virtual int initUnified(uint8_t const* key, uint8_t const* iv) noexcept override; + virtual int updateUnified(uint8_t const* iv) noexcept override; + virtual int decrypt(uint8_t* buffer, size_t len, bool unified) noexcept override; + virtual int + encrypt(uint8_t const* buffer, size_t len, bool last, bool unified) noexcept override; private: - void* m_ctx_enc; + union { + void* m_ctx_enc; + void* m_ctx_uni; + }; void* m_ctx_dec; }; diff --git a/src/aes.cpp b/src/aes.cpp index ffca8562..8c18d3bc 100644 --- a/src/aes.cpp +++ b/src/aes.cpp @@ -32,6 +32,7 @@ Aes256BaseLayer::Aes256BaseLayer(void const* key, ProtocolLayer* up, ProtocolLay , m_bufferLen() , m_encState(EncStateDisconnected) , m_decState(DecStateDisconnected) + , m_unified() , m_lastError(ENOTCONN) # ifdef STORED_OS_POSIX // NOLINTNEXTLINE @@ -41,26 +42,6 @@ Aes256BaseLayer::Aes256BaseLayer(void const* key, ProtocolLayer* up, ProtocolLay setKey(key); } -bool Aes256BaseLayer::flush() -{ - bool res = false; - - switch(m_encState) { - case EncStateConnected: - m_encState = EncStateReady; - sendIV(); - res = true; - STORED_FALLTHROUGH - case EncStateDisconnected: - case EncStateReady: - case EncStateEncoding: - default:; - // Nothing to do. - } - - return base::flush() || res; -} - void Aes256BaseLayer::reset() { m_encState = EncStateDisconnected; @@ -103,8 +84,9 @@ int Aes256BaseLayer::lastError() const noexcept void Aes256BaseLayer::decode(void* buffer, size_t len) { - uint8_t* buf = static_cast(buffer); + uint8_t* buffer_ = static_cast(buffer); +again: switch(m_decState) { case DecStateDisconnected: // Ignore data. @@ -116,16 +98,35 @@ void Aes256BaseLayer::decode(void* buffer, size_t len) // Nothing to do. return; } - if(len == BlockSize + 1 && buf[0] == CmdReset) { + if(len > BlockSize && len % BlockSize == 1) { // Got IV for decryption. - m_lastError = initDecrypt(m_key, buf + 1); + switch(buffer_[0]) { + case CmdBidirectional: + if(unified()) + unified(false); + m_lastError = initDecrypt(m_key, buffer_ + 1); + break; + case CmdUnified: + m_unified = true; + if(m_encState == EncStateConnected) + m_encState = EncStateReady; + m_lastError = initUnified(m_key, buffer_ + 1); + break; + default: + // Invalid command. + m_lastError = EINVAL; + } + if(m_lastError) { // Initialization error. disconnected(); return; } + m_decState = DecStateReady; - return; + buffer_ += BlockSize + 1; + len -= BlockSize + 1; + goto again; } if(m_decState != DecStateReady || len % BlockSize != 0) { // Invalid block. @@ -134,7 +135,7 @@ void Aes256BaseLayer::decode(void* buffer, size_t len) return; } - m_lastError = decrypt(buf, len); + m_lastError = decrypt(buffer_, len, unified()); if(m_lastError) { // Decryption error. disconnected(); @@ -173,7 +174,13 @@ void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) return; case EncStateConnected: m_encState = EncStateReady; - sendIV(); + if(unified()) { + if(m_decState == DecStateConnected) + m_decState = DecStateReady; + sendIV(true, false); + } else { + sendIV(false, false); + } goto again; case EncStateReady: { // Encrypt data. @@ -194,7 +201,7 @@ void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) size_t chunk = len & ~((size_t)BlockSize - 1U); if(likely(chunk)) { // Full chunks to encrypt directly. - res = encrypt(buffer_, chunk, false); + res = encrypt(buffer_, chunk, false, unified()); len -= chunk; buffer_ += chunk; continue; @@ -212,7 +219,7 @@ void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) buffer_ += copy; if(m_bufferLen == BlockSize) { // Encrypt full buffer. - res = encrypt(m_buffer, BlockSize, false); + res = encrypt(m_buffer, BlockSize, false, unified()); m_bufferLen = 0; continue; } @@ -228,7 +235,7 @@ void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index) m_buffer[i] = static_cast(padding); - res = encrypt(m_buffer, BlockSize, true); + res = encrypt(m_buffer, BlockSize, true, unified()); m_encState = EncStateReady; } @@ -250,20 +257,20 @@ void Aes256BaseLayer::encode(void const* buffer, size_t len, bool last) /*! * \brief Send out the initialization vector for encryption (so for decryption by the peer). */ -void Aes256BaseLayer::sendIV() noexcept +void Aes256BaseLayer::sendIV(bool unified, bool last) noexcept { uint8_t buf[BlockSize + 1]; - buf[0] = (uint8_t)CmdReset; + buf[0] = (uint8_t)(unified ? CmdUnified : CmdBidirectional); fillRandom(buf + 1, BlockSize); - m_lastError = initEncrypt(m_key, buf + 1); + m_lastError = unified ? initUnified(m_key, buf + 1) : initEncrypt(m_key, buf + 1); if(m_lastError) { // Initialization error. disconnected(); return; } - base::encode(buf, sizeof(buf), true); + base::encode(buf, sizeof(buf), last); } /*! @@ -278,16 +285,30 @@ void Aes256BaseLayer::setKey(void const* key) noexcept else memcpy(m_key, key, KeySize); - switch(m_encState) { - case EncStateDisconnected: - // Nothing to do. - break; - case EncStateConnected: - case EncStateReady: - case EncStateEncoding: - default: + if(m_encState != EncStateDisconnected) m_encState = EncStateConnected; - } + if(unified() && m_decState != DecStateDisconnected) + m_decState = DecStateConnected; +} + +/*! + * \brief Configure unified mode. + */ +void Aes256BaseLayer::unified(bool enable) noexcept +{ + m_unified = enable; + + stored_assert(m_encState != EncStateEncoding); + + if(m_encState != EncStateDisconnected) + m_encState = EncStateConnected; + if(m_decState != DecStateDisconnected) + m_decState = DecStateConnected; +} + +bool Aes256BaseLayer::unified() const noexcept +{ + return m_unified; } /*! @@ -361,14 +382,29 @@ int Aes256Layer::initDecrypt(uint8_t const* key, uint8_t const* iv) noexcept return 0; } -int Aes256Layer::decrypt(uint8_t* buffer, size_t len) noexcept +int Aes256Layer::initUnified(uint8_t const* key, uint8_t const* iv) noexcept +{ + struct AES_ctx* ctx = static_cast(m_ctx_uni); + AES_init_ctx_iv(ctx, key, iv); + return 0; +} + +int Aes256Layer::updateUnified(uint8_t const* iv) noexcept +{ + struct AES_ctx* ctx = static_cast(m_ctx_uni); + AES_ctx_set_iv(ctx, iv); + return 0; +} + +int Aes256Layer::decrypt(uint8_t* buffer, size_t len, bool unified) noexcept { if(!len) return 0; stored_assert(len % BlockSize == 0); stored_assert(buffer); - AES_CTR_xcrypt_buffer(static_cast(m_ctx_dec), buffer, len); + AES_CTR_xcrypt_buffer( + static_cast(unified ? m_ctx_uni : m_ctx_dec), buffer, len); size_t padding = buffer[len - 1]; if(padding == 0 || padding > BlockSize) @@ -382,7 +418,7 @@ int Aes256Layer::decrypt(uint8_t* buffer, size_t len) noexcept return 0; } -int Aes256Layer::encrypt(uint8_t const* buffer, size_t len, bool last) noexcept +int Aes256Layer::encrypt(uint8_t const* buffer, size_t len, bool last, bool unified) noexcept { stored_assert(!len || buffer); stored_assert(len % BlockSize == 0); @@ -390,7 +426,9 @@ int Aes256Layer::encrypt(uint8_t const* buffer, size_t len, bool last) noexcept uint8_t buf[BlockSize]; for(; len; len -= BlockSize, buffer += BlockSize) { memcpy(buf, buffer, BlockSize); - AES_CTR_xcrypt_buffer(static_cast(m_ctx_enc), buf, BlockSize); + AES_CTR_xcrypt_buffer( + static_cast(unified ? m_ctx_uni : m_ctx_enc), buf, + BlockSize); encodeEncrypted(buf, BlockSize, last && len == BlockSize); } diff --git a/tests/test_protocol.cpp b/tests/test_protocol.cpp index ec4df5f2..d669a723 100644 --- a/tests/test_protocol.cpp +++ b/tests/test_protocol.cpp @@ -1578,6 +1578,14 @@ TEST(Aes256Layer, EncodeDecode) lb.encode("klmn", 4); EXPECT_EQ(la.decoded().at(1), "klmn"); + + // Set unified. + a.unified(true); + la.encode("op", 2); + EXPECT_EQ(lb.decoded().at(2), "op"); + + lb.encode("qr", 2); + EXPECT_EQ(la.decoded().at(2), "qr"); } } // namespace From 2e75d41c494176d31d16d93b281856799269ca7a Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:24:54 +0100 Subject: [PATCH 10/19] improve args --- python/libstored/wrapper/serial.py | 14 +++++++------- python/libstored/wrapper/stdio.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/python/libstored/wrapper/serial.py b/python/libstored/wrapper/serial.py index 0044ab2b..d9b59dd5 100644 --- a/python/libstored/wrapper/serial.py +++ b/python/libstored/wrapper/serial.py @@ -16,15 +16,15 @@ def main(): parser = argparse.ArgumentParser(description='serial wrapper to ZMQ server', formatter_class=argparse.ArgumentDefaultsHelpFormatter, prog=__package__) - parser.add_argument('-V', action='version', version=__version__) - parser.add_argument('-l', dest='zmqlisten', type=str, default='*', help='ZMQ listen address') - parser.add_argument('-p', dest='zmqport', type=int, default=lprot.default_port, help='ZMQ port') + parser.add_argument('-V', '--version', action='version', version=__version__) + parser.add_argument('-l', '--listen', dest='zmqlisten', type=str, default='*', help='ZMQ listen address') + parser.add_argument('-p', '--port', dest='zmqport', type=int, default=lprot.default_port, help='ZMQ port') parser.add_argument('port', help='serial port') parser.add_argument('baud', nargs='?', type=int, default=115200, help='baud rate') - parser.add_argument('-r', dest='rtscts', default=False, help='RTS/CTS flow control', action='store_true') - parser.add_argument('-x', dest='xonxoff', default=False, help='XON/XOFF flow control', action='store_true') - parser.add_argument('-v', dest='verbose', default=0, help='Enable verbose output', action='count') - parser.add_argument('-S', dest='stack', type=str, default='ascii,pubterm,stdin', help='protocol stack') + parser.add_argument('-r', '--rtscts', dest='rtscts', default=False, help='RTS/CTS flow control', action='store_true') + parser.add_argument('-x', '--xonxoff', dest='xonxoff', default=False, help='XON/XOFF flow control', action='store_true') + parser.add_argument('-v', '--verbose', dest='verbose', default=0, help='Enable verbose output', action='count') + parser.add_argument('-S', '--stack', dest='stack', type=str, default='ascii,pubterm,stdin', help='protocol stack') args = parser.parse_args() diff --git a/python/libstored/wrapper/stdio.py b/python/libstored/wrapper/stdio.py index 545a0b13..b3d92baf 100644 --- a/python/libstored/wrapper/stdio.py +++ b/python/libstored/wrapper/stdio.py @@ -16,11 +16,11 @@ def main(): parser = argparse.ArgumentParser(description='stdin/stdout wrapper to ZMQ server', formatter_class=argparse.ArgumentDefaultsHelpFormatter, prog=__package__) - parser.add_argument('-V', action='version', version=__version__) - parser.add_argument('-l', dest='listen', type=str, default='*', help='listen address') - parser.add_argument('-p', dest='port', type=int, default=lprot.default_port, help='port') - parser.add_argument('-S', dest='stack', type=str, default='ascii,pubterm,stdin', help='protocol stack') - parser.add_argument('-v', dest='verbose', default=0, help='Enable verbose output', action='count') + parser.add_argument('-V', '--version', action='version', version=__version__) + parser.add_argument('-l', '--listen', dest='listen', type=str, default='*', help='listen address') + parser.add_argument('-p', '--port', dest='port', type=int, default=lprot.default_port, help='port') + parser.add_argument('-S', '--stack', dest='stack', type=str, default='ascii,pubterm,stdin', help='protocol stack') + parser.add_argument('-v', '--verbose', dest='verbose', default=0, help='Enable verbose output', action='count') parser.add_argument('command') parser.add_argument('args', nargs='*') From cecc5ca7caa67b224dd48906aa4012e9935ca6e6 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:25:24 +0100 Subject: [PATCH 11/19] integrate aes --- python/libstored/asyncio/zmq.py | 71 ++++++++++++++++++++++-- python/libstored/cli/__main__.py | 16 ++++-- python/libstored/cmake/__main__.py | 4 +- python/libstored/generator/__main__.py | 16 +++--- python/libstored/gui/__main__.py | 36 ++++++++----- python/libstored/log/__main__.py | 30 ++++++----- python/libstored/protocol/protocol.py | 75 ++++++++++++++++++++++---- sphinx/doc/py_py.rst | 2 + 8 files changed, 197 insertions(+), 53 deletions(-) diff --git a/python/libstored/asyncio/zmq.py b/python/libstored/asyncio/zmq.py index 1dedcf53..e8bf2f76 100644 --- a/python/libstored/asyncio/zmq.py +++ b/python/libstored/asyncio/zmq.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MPL-2.0 from __future__ import annotations +import logging import aiofiles import asyncio @@ -1308,6 +1309,7 @@ class ZmqClient(Work): def __init__(self, host : str='localhost', port : int=lprot.default_port, multi : bool=False, timeout : float | None=None, context : None | zmq.asyncio.Context=None, t : str | None = None, use_state : str | None=None, + stack : str | lprot.ProtocolLayer | None=None, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1323,6 +1325,18 @@ def __init__(self, host : str='localhost', port : int=lprot.default_port, self._timestamp_to_time = lambda t: float(t) self._use_state = use_state + if isinstance(stack, str): + self._stack = lprot.build_stack(stack) + elif isinstance(stack, lprot.ProtocolLayer): + self._stack = stack + else: + self._stack = lprot.ProtocolLayer() + self._stack_encoded : bytearray | None = None + self._stack_decoded : bytearray | None = None + self._stack.up = self._stack_up + self._stack.down = self._stack_down + + self._reset() # Events @@ -1666,16 +1680,65 @@ async def req(self, msg : bytes | str) -> bytes | str: finally: self._req_task = None + def _stack_clear(self) -> None: + self._stack_encoded = None + self._stack_decoded = None + + def _stack_up(self, data : lprot.ProtocolLayer.Packet) -> None: + if isinstance(data, str): + data = data.encode() + elif isinstance(data, memoryview): + data = data.cast('B') + + if self._stack_decoded is None: + self._stack_decoded = bytearray(data) + else: + self._stack_decoded.extend(data) + + def _stack_down(self, data : lprot.ProtocolLayer.Packet) -> None: + if isinstance(data, str): + data = data.encode() + elif isinstance(data, memoryview): + data = data.cast('B') + + if self._stack_encoded is None: + self._stack_encoded = bytearray(data) + else: + self._stack_encoded.extend(data) + async def _req(self, msg : bytes) -> bytes: if not self.is_connected(): raise lexc.InvalidState('Not connected') assert self._socket is not None - self.logger.debug('req %s', msg) - await self._socket.send(msg) + + self._stack_clear() + await self._stack.encode(msg) + if self._stack_encoded is None: + raise lexc.OperationFailed('Stack did not produce data') + + if self.logger.getEffectiveLevel() <= logging.DEBUG: + if self._stack_encoded != msg: + self.logger.debug('req %s -> %s', msg, bytes(self._stack_encoded)) + else: + self.logger.debug('req %s', msg) + + await self._socket.send(self._stack_encoded) + rep = b''.join(await self._socket.recv_multipart()) - self.logger.debug('rep %s', rep) - return rep + await self._stack.decode(rep) + if rep and self._stack_decoded is None: + raise lexc.InvalidResponse('Stack did not decode data') + + decoded = bytes(self._stack_decoded) if self._stack_decoded is not None else b'' + + if self.logger.getEffectiveLevel() <= logging.DEBUG: + if self._stack_decoded != rep: + self.logger.debug('rep %s <- %s', decoded, rep) + else: + self.logger.debug('rep %s', decoded) + + return decoded diff --git a/python/libstored/cli/__main__.py b/python/libstored/cli/__main__.py index d9d7740a..0375be8e 100644 --- a/python/libstored/cli/__main__.py +++ b/python/libstored/cli/__main__.py @@ -16,7 +16,11 @@ @run_sync async def async_main(args : argparse.Namespace): - async with ZmqClient(args.server, args.port, multi=True) as client: + stack = None + if args.encrypted: + stack = lprot.Aes256Layer(args.encrypted, unified=True) + + async with ZmqClient(args.server, args.port, multi=True, stack=stack) as client: prefix = '> ' await aiofiles.stdout.write(prefix) await aiofiles.stdout.flush() @@ -29,10 +33,12 @@ async def async_main(args : argparse.Namespace): def main(): parser = argparse.ArgumentParser(description='ZMQ command line client', prog=__package__) - parser.add_argument('-V', action='version', version=__version__) - parser.add_argument('-s', dest='server', type=str, default='localhost', help='ZMQ server to connect to') - parser.add_argument('-p', dest='port', type=int, default=lprot.default_port, help='port') - parser.add_argument('-v', dest='verbose', default=0, help='Enable verbose output', action='count') + parser.add_argument('-V', '--version', action='version', version=__version__) + parser.add_argument('-s', '--server', dest='server', type=str, default='localhost', help='ZMQ server to connect to') + parser.add_argument('-p', '--port', dest='port', type=int, default=lprot.default_port, help='port') + parser.add_argument('-v', '--verbose', dest='verbose', default=0, help='Enable verbose output', action='count') + parser.add_argument('-e', '--encrypt', dest='encrypted', type=str, default=None, + help='Enable AES-256 CTR encryption with the given pre-shared key file', metavar='file') args = parser.parse_args() diff --git a/python/libstored/cmake/__main__.py b/python/libstored/cmake/__main__.py index cb047a15..6cf0ada9 100644 --- a/python/libstored/cmake/__main__.py +++ b/python/libstored/cmake/__main__.py @@ -52,9 +52,9 @@ def main(): description='Generator for find_package(Libstored) in CMake', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-v', dest='verbose', default=0, help='enable verbose output', action='count') + parser.add_argument('-V', '--version', action='version', version=__version__) + parser.add_argument('-v', '--verbose', dest='verbose', default=0, help='enable verbose output', action='count') parser.add_argument('-D', dest='define', metavar='key[=value]', default=[], nargs=1, action='append', help='CMake defines') - parser.add_argument('-V', action='version', version=__version__) parser.add_argument('filename', default='FindLibstored.cmake', nargs='?', type=str, help='Output filename') args = parser.parse_args() diff --git a/python/libstored/generator/__main__.py b/python/libstored/generator/__main__.py index 39a7eed3..e392ac50 100644 --- a/python/libstored/generator/__main__.py +++ b/python/libstored/generator/__main__.py @@ -209,7 +209,7 @@ def pyliteral(x): return f'int({x})' else: return repr(x) - + def pyinit(o): if o.init is None: return None @@ -249,7 +249,7 @@ def yamlstring(s): """ if s is None: return 'null' - + if isinstance(s, str): # Escape backslashes and quotes inside the string esc = s.replace('\\', '\\\\').replace('"', '\\"') @@ -510,10 +510,10 @@ def generate_cmake(libprefix, model_files, output_dir): def main(): parser = argparse.ArgumentParser(description='Store generator', prog=__package__) - parser.add_argument('-V', action='version', version=__version__) - parser.add_argument('-p', type=str, help='libstored prefix for cmake library target') - parser.add_argument('-b', help='generate for big-endian device (default=little)', action='store_true') - parser.add_argument('-v', dest='verbose', default=0, help='Enable verbose output', action='count') + parser.add_argument('-V', '--version',action='version', version=__version__) + parser.add_argument('-p', '--prefix', type=str, help='libstored prefix for cmake library target') + parser.add_argument('-b', '--big', help='generate for big-endian device (default=little)', action='store_true') + parser.add_argument('-v', '--verbose', dest='verbose', default=0, help='Enable verbose output', action='count') parser.add_argument('store_file', type=str, nargs='+', help='store description to parse') parser.add_argument('output_dir', type=str, help='output directory for generated files') @@ -530,9 +530,9 @@ def main(): logger = logging.getLogger('libstored') for f in args.store_file: - generate_store(f, args.output_dir, not args.b) + generate_store(f, args.output_dir, not args.big) - generate_cmake(args.p, args.store_file, args.output_dir) + generate_cmake(args.prefix, args.store_file, args.output_dir) if __name__ == '__main__': main() diff --git a/python/libstored/gui/__main__.py b/python/libstored/gui/__main__.py index 7a8991f1..42e02eb9 100644 --- a/python/libstored/gui/__main__.py +++ b/python/libstored/gui/__main__.py @@ -682,7 +682,7 @@ def _on_linux_scroll(self, event): return "break" @staticmethod - def _children(widget : tk.Widget) -> set[tk.Widget]: + def _children(widget : tk.BaseWidget) -> set[tk.BaseWidget]: return set(widget.winfo_children()).union(*(ScrollableFrame._children(w) for w in widget.winfo_children())) def bind_scroll(self): @@ -1212,20 +1212,25 @@ def default_poll(self) -> float: # def main(): - parser = argparse.ArgumentParser(prog=__package__, description='ZMQ GUI client', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-V', action='version', version=__version__) - parser.add_argument('-s', dest='server', type=str, default='localhost', help='ZMQ server to connect to') - parser.add_argument('-p', dest='port', type=int, default=lprot.default_port, help='port') - parser.add_argument('-v', dest='verbose', default=0, help='Enable verbose output', action='count') - parser.add_argument('-m', dest='multi', default=False, + parser = argparse.ArgumentParser(prog=__package__, description='ZMQ GUI client', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-V', '--version', action='version', version=__version__) + parser.add_argument('-s', '--server', dest='server', type=str, default='localhost', help='ZMQ server to connect to') + parser.add_argument('-p', '--port', dest='port', type=int, default=lprot.default_port, help='port') + parser.add_argument('-v', '--verbose', dest='verbose', default=0, help='Enable verbose output', action='count') + parser.add_argument('-m', '--multi', dest='multi', default=False, help='Enable multi-mode; allow multiple simultaneous connections to the same target, ' + 'but it is less efficient.', action='store_true') - parser.add_argument('-c', dest='clear_state', default=False, help='Clear previously saved state', action='store_true') - parser.add_argument('-D', dest='deadlock', default=0, nargs='?', help='Enable deadlock checks after x seconds', type=float, const=10.0) - parser.add_argument('-f', dest='csv', default=None, nargs='?', + parser.add_argument('-c', '--clearstate', dest='clear_state', default=False, + help='Clear previously saved state', action='store_true') + parser.add_argument('-D', '--deadlock', dest='deadlock', default=0, nargs='?', + help='Enable deadlock checks after x seconds', type=float, const=10.0) + parser.add_argument('-f', '--csv', dest='csv', default=None, nargs='?', help='Log auto-refreshed data to csv file. ' + 'The file is truncated upon startup and when the set of auto-refreshed objects change. ' + 'The file name may include strftime() format codes.', const='log.csv') + parser.add_argument('-e', '--encrypt', dest='encrypted', type=str, default=None, + help='Enable AES-256 CTR encryption with the given pre-shared key file', metavar='file') args = parser.parse_args() @@ -1234,6 +1239,8 @@ def main(): 'datefmt': '%H:%M:%S', } + logger = logging.getLogger(__package__) + if args.verbose == 0: logging_config['level'] = logging.WARNING elif args.verbose == 1: @@ -1243,7 +1250,7 @@ def main(): logging.basicConfig(**logging_config) if args.deadlock > 0: - logging.getLogger().info(f'Enable deadlock checks after {args.deadlock} seconds') + logger.info(f'Enable deadlock checks after {args.deadlock} seconds') lexc.DeadlockChecker.default_timeout_s = args.deadlock csv = None @@ -1251,7 +1258,12 @@ def main(): assert isinstance(args.csv, str) csv = laio_csv.CsvExport(laio_csv.generate_filename(args.csv)) - client = laio_zmq.ZmqClient(host=args.server, port=args.port, multi=args.multi, use_state='gui') + stack = None + if args.encrypted: + stack = lprot.Aes256Layer(args.encrypted, unified=True) + logger.info(f'Enable AES-256 encryption with key file {args.encrypted}') + + client = laio_zmq.ZmqClient(host=args.server, port=args.port, multi=args.multi, use_state='gui', stack=stack) GUIClient.run(worker=client.worker, client=client, clear_state=args.clear_state, csv=csv) if __name__ == '__main__': diff --git a/python/libstored/log/__main__.py b/python/libstored/log/__main__.py index 187cbe3a..68831156 100644 --- a/python/libstored/log/__main__.py +++ b/python/libstored/log/__main__.py @@ -22,7 +22,11 @@ async def async_main(args : argparse.Namespace) -> int: if filename != '-': filename = generate_filename(filename, add_timestamp=args.timestamp, unique=args.unique) - async with ZmqClient(args.server, args.port, multi=args.multi) as client: + stack = None + if args.encrypted: + stack = lprot.Aes256Layer(args.encrypted, unified=True) + + async with ZmqClient(args.server, args.port, multi=args.multi, stack=stack) as client: objs = [] for o in args.objects: @@ -78,22 +82,24 @@ def main(): parser = argparse.ArgumentParser(prog=__package__, description='ZMQ command line logging client', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-V', action='version', version=__version__) - parser.add_argument('-s', dest='server', type=str, default='localhost', help='ZMQ server to connect to') - parser.add_argument('-p', dest='port', type=int, default=lprot.default_port, help='port') - parser.add_argument('-v', dest='verbose', default=0, help='Enable verbose output', action='count') - parser.add_argument('-f', dest='csv', default='-', + parser.add_argument('-V', '--version', action='version', version=__version__) + parser.add_argument('-s', '--server', dest='server', type=str, default='localhost', help='ZMQ server to connect to') + parser.add_argument('-p', '--port', dest='port', type=int, default=lprot.default_port, help='port') + parser.add_argument('-v', '--verbose', dest='verbose', default=0, help='Enable verbose output', action='count') + parser.add_argument('-f', '--csv', dest='csv', default='-', help='File to log to. The file name may include strftime() format codes.') - parser.add_argument('-t', dest='timestamp', default=False, help='Append time stamp in csv file name', action='store_true') - parser.add_argument('-u', dest='unique', default=False, + parser.add_argument('-t', '--timestamp', dest='timestamp', default=False, help='Append time stamp in csv file name', action='store_true') + parser.add_argument('-u', '--unique', dest='unique', default=False, help='Make sure that the log filename is unique by appending a suffix', action='store_true') - parser.add_argument('-m', dest='multi', default=False, + parser.add_argument('-m', '--multi', dest='multi', default=False, help='Enable multi-mode; allow multiple simultaneous connections to the same target, ' + 'but it is less efficient.', action='store_true') - parser.add_argument('-i', dest='interval', type=float, default=1, help='Poll interval (s)') - parser.add_argument('-d', dest='duration', type=float, default=None, help='Poll duration (s)') + parser.add_argument('-i', '--interval', dest='interval', type=float, default=1, help='Poll interval (s)') + parser.add_argument('-d', '--duration', dest='duration', type=float, default=None, help='Poll duration (s)') parser.add_argument('objects', metavar='obj', type=str, nargs='*', help='Object to poll') - parser.add_argument('-o', dest='objectfile', type=str, action='append', help='File with list of objects to poll') + parser.add_argument('-o', '--objectfile', dest='objectfile', type=str, action='append', help='File with list of objects to poll') + parser.add_argument('-e', '--encrypt', dest='encrypted', type=str, default=None, + help='Enable AES-256 CTR encryption with the given pre-shared key file', metavar='file') args = parser.parse_args() diff --git a/python/libstored/protocol/protocol.py b/python/libstored/protocol/protocol.py index f07b1b0a..cce6ca61 100644 --- a/python/libstored/protocol/protocol.py +++ b/python/libstored/protocol/protocol.py @@ -1115,10 +1115,12 @@ class Aes256Layer(ProtocolLayer): name = 'aes256' - def __init__(self, key : bytes | str | None=None, *args, **kwargs): + def __init__(self, key : bytes | str | None=None, *args, unified : bool=False, reqrep : bool=False, **kwargs): super().__init__(*args, **kwargs) self._encrypt = None self._decrypt = None + self._reqrep : bool = reqrep + self._unified : bool = unified or reqrep if key is None: raise ValueError('Key file or binary string must be provided for Aes256Layer') @@ -1126,6 +1128,12 @@ def __init__(self, key : bytes | str | None=None, *args, **kwargs): self.set_key(key) def set_key(self, key : bytes | str) -> None: + ''' + Change the AES-256 key. + + The argument can be either a 32 byte binary string, or a filename containing the key. + ''' + if isinstance(key, str): with open(key, 'rb') as f: key = f.read() @@ -1137,18 +1145,42 @@ def set_key(self, key : bytes | str) -> None: self._encrypt = None self._decrypt = None + @property + def unified(self) -> bool: + ''' + Return if unified mode is enabled. + ''' + return self._unified or self._reqrep + + @unified.setter + def unified(self, enable : bool=True) -> None: + ''' + Set unified mode. + ''' + + if self._reqrep: + return + + self._unified = enable + self._encrypt = None + if enable: + self._decrypt = None + async def encode(self, data : ProtocolLayer.Packet) -> None: if isinstance(data, str): data = data.encode() elif isinstance(data, memoryview): data = data.cast('B') + prefix = None if self._encrypt is None: - await self._send_iv() + prefix = self._iv(self.unified) assert self._encrypt is not None data = Crypto.Util.Padding.pad(data, 16) data = self._encrypt.encrypt(data) + if prefix is not None: + data = prefix + data await super().encode(data) async def decode(self, data : ProtocolLayer.Packet) -> None: @@ -1157,12 +1189,24 @@ async def decode(self, data : ProtocolLayer.Packet) -> None: elif isinstance(data, memoryview): data = data.cast('B') - if len(data) == 17 and data[0:1] == b'R': + if len(data) > 16 and len(data) % 16 == 1: # Received IV for decryption iv = data[1:17] - self._decrypt = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, iv) - self.logger.debug('Received IV for decryption') - return + cypher = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, nonce=b'', initial_value=iv) + if data[0:1] == b'U': + self.logger.debug('Received IV for unified operation') + self._decrypt = cypher + self._encrypt = cypher + self._unified = True + elif data[0:1] == b'B': + self.logger.debug('Received IV for decryption') + self._decrypt = cypher + self._unified = False + else: + self.logger.debug('Invalid IV prefix') + self._decrypt = None + + data = data[17:] if self._decrypt is None: self.logger.debug('Got data before IV, waiting for IV') @@ -1174,7 +1218,12 @@ async def decode(self, data : ProtocolLayer.Packet) -> None: self._decrypt = None return - data = self._decrypt.decrypt(data) + if self.unified: + # Encryption is the same as decryption for AES, but the API does not allow mixing the calls. + data = self._decrypt.encrypt(data) + else: + data = self._decrypt.decrypt(data) + try: data = Crypto.Util.Padding.unpad(data, 16) except ValueError: @@ -1184,10 +1233,16 @@ async def decode(self, data : ProtocolLayer.Packet) -> None: await super().decode(data) - async def _send_iv(self) -> None: + def _iv(self, unified : bool) -> bytes: iv = Crypto.Random.get_random_bytes(16) - self._encrypt = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, iv) - await super().encode(b'R' + iv) + cipher = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, nonce=b'', initial_value=iv) + cipher.nonce + self._encrypt = cipher + if unified: + self._decrypt = cipher + return b'U' + iv + else: + return b'B' + iv diff --git a/sphinx/doc/py_py.rst b/sphinx/doc/py_py.rst index 0cac45fb..c4d0a979 100644 --- a/sphinx/doc/py_py.rst +++ b/sphinx/doc/py_py.rst @@ -22,6 +22,8 @@ Protocol layers :members: :undoc-members: +.. autoclass:: libstored.protocol.Aes256Layer + .. autoclass:: libstored.protocol.AsciiEscapeLayer .. autoclass:: libstored.protocol.TerminalLayer From 20a082cfbc3a382107d308b891bb34a28d93484a Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Thu, 4 Dec 2025 22:24:28 +0100 Subject: [PATCH 12/19] fix aes on debug channel --- CHANGELOG.rst | 7 +++- python/libstored/cli/__main__.py | 2 +- python/libstored/gui/__main__.py | 2 +- python/libstored/log/__main__.py | 2 +- python/libstored/protocol/protocol.py | 47 ++++++++++++++++++++++----- src/aes.cpp | 2 ++ 6 files changed, 49 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6f6a74c5..de57cf6c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,7 +27,12 @@ Added - Operations such as ``-=`` and ``++`` for store variables in C++. - YAML export of store meta-data. - ``stored::MuxLayer`` to multiplex multiple protocol layers over a single connection. -- ``stored::Aes256Layer`` for encrypted communication using AES-256 CTR. +- ``stored::Aes256Layer`` and ``libstored.protocol.Aes256Layer`` for encrypted communication using + AES-256 CTR. +- Allow adding a protocol layer stack to ``ZmqClient`` on top of the ZeroMQ socket. If putting + the ``Aes256Layer`` in the stack, encrypted debug communication is enabled with the target. +- ``lossy_sync`` example that shows how to use the Synchronizer over a lossy protocol, and how to + use encryption in both the Synchronizer and Debugger stacks. Changed ``````` diff --git a/python/libstored/cli/__main__.py b/python/libstored/cli/__main__.py index 0375be8e..7a9f5201 100644 --- a/python/libstored/cli/__main__.py +++ b/python/libstored/cli/__main__.py @@ -18,7 +18,7 @@ async def async_main(args : argparse.Namespace): stack = None if args.encrypted: - stack = lprot.Aes256Layer(args.encrypted, unified=True) + stack = lprot.Aes256Layer(args.encrypted, reqrep=True) async with ZmqClient(args.server, args.port, multi=True, stack=stack) as client: prefix = '> ' diff --git a/python/libstored/gui/__main__.py b/python/libstored/gui/__main__.py index 42e02eb9..e0ff6b17 100644 --- a/python/libstored/gui/__main__.py +++ b/python/libstored/gui/__main__.py @@ -1260,7 +1260,7 @@ def main(): stack = None if args.encrypted: - stack = lprot.Aes256Layer(args.encrypted, unified=True) + stack = lprot.Aes256Layer(args.encrypted, reqrep=True) logger.info(f'Enable AES-256 encryption with key file {args.encrypted}') client = laio_zmq.ZmqClient(host=args.server, port=args.port, multi=args.multi, use_state='gui', stack=stack) diff --git a/python/libstored/log/__main__.py b/python/libstored/log/__main__.py index 68831156..78fc7ed8 100644 --- a/python/libstored/log/__main__.py +++ b/python/libstored/log/__main__.py @@ -24,7 +24,7 @@ async def async_main(args : argparse.Namespace) -> int: stack = None if args.encrypted: - stack = lprot.Aes256Layer(args.encrypted, unified=True) + stack = lprot.Aes256Layer(args.encrypted, reqrep=True) async with ZmqClient(args.server, args.port, multi=args.multi, stack=stack) as client: objs = [] diff --git a/python/libstored/protocol/protocol.py b/python/libstored/protocol/protocol.py index cce6ca61..9b62979e 100644 --- a/python/libstored/protocol/protocol.py +++ b/python/libstored/protocol/protocol.py @@ -1111,6 +1111,13 @@ async def timeout(self) -> None: class Aes256Layer(ProtocolLayer): ''' A ProtocolLayer that adds AES-256 encryption/decryption. + + The unified mode allows using the same cipher state for encryption and decryption, which uses a + single key stream for all data. + + A specific mode is the reqrep mode, which is a unified mode, but changes the IV for every + request/reply pair. This is useful for a ZeroMQ REQ/REP pattern, where the server may handle + multiple clients simultaneously, which does not allow having a single cipher state. ''' name = 'aes256' @@ -1158,9 +1165,27 @@ def unified(self, enable : bool=True) -> None: Set unified mode. ''' - if self._reqrep: - return + self._unified = enable + self._encrypt = None + if enable: + self._decrypt = None + else: + self._reqrep = False + @property + def reqrep(self) -> bool: + ''' + Return if reqrep mode is enabled. + ''' + return self._reqrep + + @reqrep.setter + def reqrep(self, enable : bool=True) -> None: + ''' + Set reqrep mode. + ''' + + self._reqrep = enable self._unified = enable self._encrypt = None if enable: @@ -1172,13 +1197,15 @@ async def encode(self, data : ProtocolLayer.Packet) -> None: elif isinstance(data, memoryview): data = data.cast('B') + data = Crypto.Util.Padding.pad(data, 16) + prefix = None - if self._encrypt is None: + if self._encrypt is None or self.reqrep: prefix = self._iv(self.unified) + assert self._encrypt is not None - assert self._encrypt is not None - data = Crypto.Util.Padding.pad(data, 16) data = self._encrypt.encrypt(data) + if prefix is not None: data = prefix + data await super().encode(data) @@ -1198,10 +1225,12 @@ async def decode(self, data : ProtocolLayer.Packet) -> None: self._decrypt = cypher self._encrypt = cypher self._unified = True + self._reqrep = False elif data[0:1] == b'B': self.logger.debug('Received IV for decryption') self._decrypt = cypher self._unified = False + self._reqrep = False else: self.logger.debug('Invalid IV prefix') self._decrypt = None @@ -1235,11 +1264,11 @@ async def decode(self, data : ProtocolLayer.Packet) -> None: def _iv(self, unified : bool) -> bytes: iv = Crypto.Random.get_random_bytes(16) - cipher = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, nonce=b'', initial_value=iv) - cipher.nonce - self._encrypt = cipher + # Make sure not to wrap around the counter soon. + iv = bytes([iv[0] & 0x0f]) + iv[1:] + self._encrypt = Crypto.Cipher.AES.new(self._key, Crypto.Cipher.AES.MODE_CTR, nonce=b'', initial_value=iv) if unified: - self._decrypt = cipher + self._decrypt = self._encrypt return b'U' + iv else: return b'B' + iv diff --git a/src/aes.cpp b/src/aes.cpp index 8c18d3bc..dada5f65 100644 --- a/src/aes.cpp +++ b/src/aes.cpp @@ -262,6 +262,8 @@ void Aes256BaseLayer::sendIV(bool unified, bool last) noexcept uint8_t buf[BlockSize + 1]; buf[0] = (uint8_t)(unified ? CmdUnified : CmdBidirectional); fillRandom(buf + 1, BlockSize); + // Make sure not to wrap around the counter soon. + buf[1] = (uint8_t)(buf[1] & 0x0fU); m_lastError = unified ? initUnified(m_key, buf + 1) : initEncrypt(m_key, buf + 1); if(m_lastError) { From d8626690ff023e42feed56627b78abc0a25b4905 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Fri, 5 Dec 2025 09:56:10 +0100 Subject: [PATCH 13/19] compile fix --- cmake/libstored.cmake | 4 ++-- python/CMakeLists.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/libstored.cmake b/cmake/libstored.cmake index 5d2e4f4c..b5a222b7 100644 --- a/cmake/libstored.cmake +++ b/cmake/libstored.cmake @@ -409,7 +409,7 @@ Relationship: SPDXRef-libstored DEPENDS_ON SPDXRef-libzmq target_compile_definitions( ${LIBSTORED_LIB_TARGET} PUBLIC -DSTORED_HAVE_HEATSHRINK=1 ) - target_link_libraries(${LIBSTORED_LIB_TARGET} PUBLIC heatshrink) + target_link_libraries(${LIBSTORED_LIB_TARGET} PRIVATE heatshrink) set(_fields) @@ -450,7 +450,7 @@ Relationship: SPDXRef-libstored DEPENDS_ON SPDXRef-heatshrink if(LIBSTORED_HAVE_AES) target_compile_definitions(${LIBSTORED_LIB_TARGET} PUBLIC -DSTORED_HAVE_AES=1) - target_link_libraries(${LIBSTORED_LIB_TARGET} PUBLIC tinyaes) + target_link_libraries(${LIBSTORED_LIB_TARGET} PRIVATE tinyaes) set(_fields) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6aa67c69..6de7cd45 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -67,6 +67,7 @@ set(package_data_ fpga/rtl/variable.vhd fpga/tb/tb_pkg.vhd fpga/vivado/vivado.tcl.tmpl + include/libstored/aes.h include/libstored/allocator.h include/libstored/components.h include/libstored/compress.h @@ -89,6 +90,7 @@ set(package_data_ include/stored.h include/stored_config.h include/stored + src/aes.cpp src/compress.cpp src/debugger.cpp src/directory.cpp From a70814345e4f9644c859cfea06888dcf7d9818eb Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Fri, 5 Dec 2025 10:31:33 +0100 Subject: [PATCH 14/19] compile fix --- examples/lib/src/getopt_mini.cpp | 4 +- examples/lossy_sync/main.cpp | 2 + src/aes.cpp | 6 +- tests/test_debugger.cpp | 12 +- tests/test_directory.cpp | 10 +- tests/test_fifo.cpp | 62 ++++---- tests/test_function.cpp | 6 +- tests/test_pipes.cpp | 14 +- tests/test_poller.cpp | 6 +- tests/test_protocol.cpp | 264 +++++++++++++++---------------- tests/test_spm.cpp | 62 ++++---- tests/test_synchronizer.cpp | 98 ++++++------ tests/test_types.cpp | 56 +++---- 13 files changed, 304 insertions(+), 298 deletions(-) diff --git a/examples/lib/src/getopt_mini.cpp b/examples/lib/src/getopt_mini.cpp index 6f346de6..e4f99281 100644 --- a/examples/lib/src/getopt_mini.cpp +++ b/examples/lib/src/getopt_mini.cpp @@ -61,4 +61,6 @@ int getopt(int argc, char* const* argv, char const* options) return optopt; } -#endif // !POSIX +#else // POSIX +char dummy_char_to_make_getopt_mini_cpp_non_empty; // NOLINT +#endif // POSIX diff --git a/examples/lossy_sync/main.cpp b/examples/lossy_sync/main.cpp index a2bdff77..ff1f15b6 100644 --- a/examples/lossy_sync/main.cpp +++ b/examples/lossy_sync/main.cpp @@ -194,6 +194,7 @@ static Arguments parse_arguments(int argc, char** argv) } break; case 'e': { + // flawfinder: ignore FILE* f = fopen(optarg, "rb"); args.key.resize(stored::Aes256Layer::KeySize); @@ -540,6 +541,7 @@ class SyncStack { auto h = m_heartbeat++; if(connected()) { + // flawfinder: ignore char buf[32]; snprintf(buf, sizeof(buf), "ping %u", h); m_ch1->encode(buf, strlen(buf), true); diff --git a/src/aes.cpp b/src/aes.cpp index dada5f65..a6c738ff 100644 --- a/src/aes.cpp +++ b/src/aes.cpp @@ -344,13 +344,15 @@ void Aes256BaseLayer::fillRandom(uint8_t* buffer, size_t len) noexcept # error "CTR not defined in aes.h" # endif -static_assert(Aes256Layer::KeySize == AES_KEYLEN, ""); - +// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) Aes256Layer::Aes256Layer(void const* key, ProtocolLayer* up, ProtocolLayer* down) : base(key, up, down) , m_ctx_enc() , m_ctx_dec() { + static_assert(Aes256Layer::KeySize == AES_KEYLEN, ""); + static_assert(Aes256Layer::BlockSize == AES_BLOCKLEN, ""); + // NOLINTNEXTLINE m_ctx_enc = new struct AES_ctx; diff --git a/tests/test_debugger.cpp b/tests/test_debugger.cpp index 85231864..454a8e1a 100644 --- a/tests/test_debugger.cpp +++ b/tests/test_debugger.cpp @@ -24,7 +24,7 @@ TEST(Debugger, Capabilities) ll.wrap(d); DECODE(d, "?"); - EXPECT_GT(ll.encoded().at(0).size(), 1); + EXPECT_GT(ll.encoded().at(0).size(), 1U); } TEST(Debugger, Identification) @@ -97,7 +97,7 @@ TEST(Debugger, List) d.list([&](char const* name, stored::DebugVariant&) { names.push_back(name); }); // We should find something. - EXPECT_GT(names.size(), 10); + EXPECT_GT(names.size(), 10U); // Check a few names. EXPECT_TRUE(std::find(names.begin(), names.end(), "/default int8") != names.end()); @@ -152,7 +152,7 @@ TEST(Debugger, ListMulti) d.list([&](char const* name, stored::DebugVariant&) { names.push_back(name); }); // We should find something. - EXPECT_GT(names.size(), 10); + EXPECT_GT(names.size(), 10U); // Check a few names. EXPECT_TRUE(std::find(names.begin(), names.end(), "/first/default int8") != names.end()); @@ -307,15 +307,15 @@ TEST(Debugger, WriteMem) LoggingLayer ll; ll.wrap(d); - uint32_t i = 0x12345678; + uint32_t i = 0x12345678U; char buf[32]; snprintf(buf, sizeof(buf), "W%" PRIxPTR " abcdef01", (uintptr_t)&i); d.decode(buf, strlen(buf)); EXPECT_EQ(ll.encoded().at(0), "!"); #ifdef STORED_LITTLE_ENDIAN - EXPECT_EQ(i, 0x01efcdab); + EXPECT_EQ(i, 0x01efcdabU); #else - EXPECT_EQ(i, 0xabcdef01); + EXPECT_EQ(i, 0xabcdef01U); #endif } diff --git a/tests/test_directory.cpp b/tests/test_directory.cpp index 2780273a..a2ad19f2 100644 --- a/tests/test_directory.cpp +++ b/tests/test_directory.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -113,7 +113,7 @@ TEST(Directory, List) }); // We should find something. - EXPECT_GT(names.size(), 10); + EXPECT_GT(names.size(), 10U); // Check a few names. EXPECT_TRUE(std::find(names.begin(), names.end(), "/default int8") != names.end()); @@ -131,12 +131,12 @@ TEST(Directory, List) TEST(Directory, Constexpr) { #ifdef STORED_COMPILER_GCC -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Waddress" +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Waddress" #endif static_assert(stored::TestStoreData::shortDirectory() != nullptr, ""); #ifdef STORED_COMPILER_GCC -# pragma GCC diagnostic pop +# pragma GCC diagnostic pop #endif constexpr auto v = diff --git a/tests/test_fifo.cpp b/tests/test_fifo.cpp index 2663b835..1b872dd2 100644 --- a/tests/test_fifo.cpp +++ b/tests/test_fifo.cpp @@ -48,27 +48,27 @@ TEST(Fifo, UnboundedFifo) f.push_back(1); EXPECT_FALSE(f.empty()); EXPECT_FALSE(f.full()); - EXPECT_EQ(f.size(), 1u); - EXPECT_EQ(f.front(), 1u); + EXPECT_EQ(f.size(), 1U); + EXPECT_EQ(f.front(), 1); EXPECT_EQ_VIEW(f); f.push_back(2); - EXPECT_EQ(f.size(), 2u); - EXPECT_EQ(f.front(), 1u); + EXPECT_EQ(f.size(), 2U); + EXPECT_EQ(f.front(), 1); EXPECT_EQ_VIEW(f); f.push_back(3); f.push_back(4); - EXPECT_EQ(f.size(), 4u); + EXPECT_EQ(f.size(), 4U); EXPECT_EQ_VIEW(f); f.pop_front(); - EXPECT_EQ(f.front(), 2u); + EXPECT_EQ(f.front(), 2); EXPECT_EQ_VIEW(f); f.push_back(5); // Not empty, should still be growing. - EXPECT_EQ(f.size(), 5u); + EXPECT_EQ(f.size(), 5U); EXPECT_EQ_VIEW(f); f.pop_front(); @@ -82,8 +82,8 @@ TEST(Fifo, UnboundedFifo) f.push_back(7); f.push_back(8); // Restarted at beginning of buffer. - EXPECT_EQ(f.size(), 5u); - EXPECT_EQ(f.front(), 6u); + EXPECT_EQ(f.size(), 5U); + EXPECT_EQ(f.front(), 6); EXPECT_EQ_VIEW(f); } @@ -94,45 +94,45 @@ TEST(Fifo, BoundedFifo) EXPECT_TRUE(f.bounded()); EXPECT_TRUE(f.empty()); EXPECT_FALSE(f.full()); - EXPECT_LE(f.size(), 5u); + EXPECT_LE(f.size(), 5U); EXPECT_EQ_VIEW(f); f.push_back(1); EXPECT_FALSE(f.empty()); EXPECT_FALSE(f.full()); - EXPECT_LE(f.size(), 5u); - EXPECT_EQ(f.front(), 1u); + EXPECT_LE(f.size(), 5U); + EXPECT_EQ(f.front(), 1); EXPECT_EQ_VIEW(f); f.push_back(2); - EXPECT_LE(f.size(), 5u); - EXPECT_EQ(f.front(), 1u); + EXPECT_LE(f.size(), 5U); + EXPECT_EQ(f.front(), 1); EXPECT_EQ_VIEW(f); f.push_back(3); f.push_back(4); EXPECT_TRUE(f.full()); - EXPECT_LE(f.size(), 5u); + EXPECT_LE(f.size(), 5U); EXPECT_EQ_VIEW(f); f.pop_front(); EXPECT_FALSE(f.full()); - EXPECT_EQ(f.front(), 2u); + EXPECT_EQ(f.front(), 2); EXPECT_EQ_VIEW(f); f.push_back(5); EXPECT_TRUE(f.full()); - EXPECT_LE(f.size(), 5u); + EXPECT_LE(f.size(), 5U); EXPECT_EQ_VIEW(f); f.pop_front(); - EXPECT_EQ(f.front(), 3u); + EXPECT_EQ(f.front(), 3); EXPECT_EQ_VIEW(f); f.pop_front(); - EXPECT_EQ(f.front(), 4u); + EXPECT_EQ(f.front(), 4); EXPECT_EQ_VIEW(f); f.pop_front(); - EXPECT_EQ(f.front(), 5u); + EXPECT_EQ(f.front(), 5); EXPECT_EQ_VIEW(f); f.pop_front(); EXPECT_TRUE(f.empty()); @@ -158,9 +158,9 @@ TEST(Fifo, IterateFifo) } // Should stop at the content when the iterator was created. - EXPECT_EQ(f.front(), 11u); + EXPECT_EQ(f.front(), 11); f.pop_front(); - EXPECT_EQ(f.front(), 12u); + EXPECT_EQ(f.front(), 12); f.pop_front(); EXPECT_TRUE(f.empty()); } @@ -181,14 +181,14 @@ TEST(Fifo, UnboundedMessageFifo) EXPECT_TRUE(f.empty()); EXPECT_TRUE(f.push_back("abc", 3)); EXPECT_FALSE(f.empty()); - EXPECT_EQ(f.available(), 1u); - EXPECT_EQ(f.size(), 3u); + EXPECT_EQ(f.available(), 1U); + EXPECT_EQ(f.size(), 3U); EXPECT_EQ_MSG(f.front(), "abc"); EXPECT_TRUE(f.push_back("defg", 4)); - EXPECT_EQ(f.available(), 2u); - EXPECT_EQ(f.size(), 7u); + EXPECT_EQ(f.available(), 2U); + EXPECT_EQ(f.size(), 7U); EXPECT_EQ_MSG(f.front(), "abc"); f.pop_front(); EXPECT_EQ_MSG(f.front(), "defg"); @@ -199,7 +199,7 @@ TEST(Fifo, UnboundedMessageFifo) EXPECT_TRUE(f.push_back(stored::MessageView{"hi", 2})); EXPECT_EQ_MSG(f.front(), "hi"); EXPECT_FALSE(f.empty()); - EXPECT_EQ(f.size(), 7u); + EXPECT_EQ(f.size(), 7U); EXPECT_TRUE(f.append_back("jk", 2)); EXPECT_EQ_MSG(f.front(), "hi"); @@ -218,16 +218,16 @@ TEST(Fifo, BoundedMessageFifo) stored::MessageFifo<16, 4> f; EXPECT_TRUE(f.bounded()); - EXPECT_EQ(f.space(), 15u); + EXPECT_EQ(f.space(), 15U); EXPECT_FALSE(f.full()); - EXPECT_EQ(f.push_back({{"abc", 3}, {"defg", 4}, {"ghijk", 5}, {"lmn", 3}}), 4u); - EXPECT_EQ(f.space(), 0u); + EXPECT_EQ(f.push_back({{"abc", 3}, {"defg", 4}, {"ghijk", 5}, {"lmn", 3}}), 4U); + EXPECT_EQ(f.space(), 0U); // Does not fit EXPECT_FALSE(f.push_back({"h", 1})); EXPECT_EQ_MSG(f.front(), "abc"); f.pop_front(); - EXPECT_EQ(f.space(), 2u); + EXPECT_EQ(f.space(), 2U); // Too long message. EXPECT_FALSE(f.push_back({"hijl", 4})); diff --git a/tests/test_function.cpp b/tests/test_function.cpp index 870d5e8e..da6e0e08 100644 --- a/tests/test_function.cpp +++ b/tests/test_function.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -96,8 +96,8 @@ TEST(Function, WriteOnly) { FunctionTestStore store; char buffer[] = "hi all!"; - EXPECT_EQ(store.f_write_only.get(buffer, sizeof(buffer)), 0); - EXPECT_EQ(store.f_write_only.set(buffer, strlen(buffer)), 4u); + EXPECT_EQ(store.f_write_only.get(buffer, sizeof(buffer)), 0U); + EXPECT_EQ(store.f_write_only.set(buffer, strlen(buffer)), 4U); } TEST(Function, FreeFunction) diff --git a/tests/test_pipes.cpp b/tests/test_pipes.cpp index 0a38ff3e..fa036481 100644 --- a/tests/test_pipes.cpp +++ b/tests/test_pipes.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -656,8 +656,8 @@ TEST(Pipes, IndexMap) auto p4 = Entry{} >> Map({10L, 20L, 30L, 40L}, comp{}) >> Cap{}; - EXPECT_EQ(p4.entry_cast(29L), 2); - EXPECT_EQ(p4.entry_cast(25L), 0); + EXPECT_EQ(p4.entry_cast(29L), 2U); + EXPECT_EQ(p4.entry_cast(25L), 0U); } TEST(Pipes, OrderedMap) @@ -795,13 +795,13 @@ TEST(Pipes, Ref) auto& p1 = Entry{} >> Ref{}; p1.inject(0); - EXPECT_EQ(gc.size(), 1); + EXPECT_EQ(gc.size(), 1U); gc.destroy(); Group g; Entry{} >> Ref{g}; - EXPECT_EQ(gc.size(), 0); - EXPECT_EQ(g.size(), 1); + EXPECT_EQ(gc.size(), 0U); + EXPECT_EQ(g.size(), 1U); g.destroy(); int cnt = 0; @@ -815,7 +815,7 @@ TEST(Pipes, Ref) 2 >> p3; EXPECT_EQ(cnt, 4); - EXPECT_EQ(gc.size(), 2); + EXPECT_EQ(gc.size(), 2U); } } // namespace diff --git a/tests/test_poller.cpp b/tests/test_poller.cpp index 8a57ff6f..b22c14e9 100644 --- a/tests/test_poller.cpp +++ b/tests/test_poller.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -52,13 +52,13 @@ TEST(Poller, PollableZmqSocket) EXPECT_EQ(poller.add(preq), 0); auto const* res = &poller.poll(0); - EXPECT_EQ(res->size(), 0); + EXPECT_EQ(res->size(), 0U); EXPECT_EQ(errno, EAGAIN); zmq_send(req, "Hi", 2, 0); res = &poller.poll(0); - ASSERT_EQ(res->size(), 1); + ASSERT_EQ(res->size(), 1U); EXPECT_EQ(res->at(0)->revents, stored::Pollable::PollIn + 0); zmq_close(req); diff --git a/tests/test_protocol.cpp b/tests/test_protocol.cpp index d669a723..8c4453a1 100644 --- a/tests/test_protocol.cpp +++ b/tests/test_protocol.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2024 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -29,12 +29,12 @@ TEST(AsciiEscapeLayer, Encode) ll.encoded().clear(); l.encode("123", 3); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123"); ll.encoded().clear(); l.encode("123\x00", 4); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123\x7f@"); ll.encoded().clear(); @@ -42,7 +42,7 @@ TEST(AsciiEscapeLayer, Encode) "123\r" "4", 5); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ( ll.encoded().at(0), "123\x7f" @@ -50,7 +50,7 @@ TEST(AsciiEscapeLayer, Encode) ll.encoded().clear(); l.encode("123\x7f", 4); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123\x7f\x7f"); ll.encoded().clear(); @@ -58,7 +58,7 @@ TEST(AsciiEscapeLayer, Encode) "\x7f" "123\r", 5); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ( ll.encoded().at(0), "\x7f\x7f" @@ -75,19 +75,19 @@ TEST(AsciiEscapeLayer, Decode) DECODE(l, "123\x7f" "F"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123\x06"); ll.decoded().clear(); DECODE(l, "123\x7f"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123\x7f"); ll.decoded().clear(); DECODE(l, "\x7f" "A12\r3"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ( ll.decoded().at(0), "\x01" @@ -102,30 +102,30 @@ TEST(SegmentationLayer, SingleChunkEncode) ll.encoded().clear(); l.encode("123", 3); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123E"); ll.encoded().clear(); l.encode("", 0); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "E"); ll.encoded().clear(); l.encode("1234567", 7); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234567E"); ll.encoded().clear(); l.encode("1234", 4, false); l.encode("567", 3, true); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234567E"); ll.encoded().clear(); l.encode("1234", 4, false); l.encode("567", 3, false); l.encode(); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234567E"); } @@ -137,19 +137,19 @@ TEST(SegmentationLayer, MultiChunkEncode) ll.encoded().clear(); l.encode("1234", 4); - EXPECT_EQ(ll.encoded().size(), 2); + EXPECT_EQ(ll.encoded().size(), 2U); EXPECT_EQ(ll.encoded().at(0), "123C"); EXPECT_EQ(ll.encoded().at(1), "4E"); ll.encoded().clear(); l.encode("12345", 5); - EXPECT_EQ(ll.encoded().size(), 2); + EXPECT_EQ(ll.encoded().size(), 2U); EXPECT_EQ(ll.encoded().at(0), "123C"); EXPECT_EQ(ll.encoded().at(1), "45E"); ll.encoded().clear(); l.encode("1234567890", 10); - EXPECT_EQ(ll.encoded().size(), 4); + EXPECT_EQ(ll.encoded().size(), 4U); EXPECT_EQ(ll.encoded().at(0), "123C"); EXPECT_EQ(ll.encoded().at(1), "456C"); EXPECT_EQ(ll.encoded().at(2), "789C"); @@ -160,7 +160,7 @@ TEST(SegmentationLayer, MultiChunkEncode) l.encode("67", 2, false); l.encode("89", 2, false); l.encode(); - EXPECT_EQ(ll.encoded().size(), 3); + EXPECT_EQ(ll.encoded().size(), 3U); EXPECT_EQ(ll.encoded().at(0), "123C"); EXPECT_EQ(ll.encoded().at(1), "456C"); EXPECT_EQ(ll.encoded().at(2), "789E"); @@ -174,17 +174,17 @@ TEST(SegmentationLayer, SingleChunkDecode) ll.decoded().clear(); DECODE(l, "123E"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123"); ll.decoded().clear(); DECODE(l, "E"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), ""); ll.decoded().clear(); DECODE(l, ""); - EXPECT_EQ(ll.decoded().size(), 0); + EXPECT_EQ(ll.decoded().size(), 0U); } TEST(SegmentationLayer, MultiChunkDecode) @@ -195,31 +195,31 @@ TEST(SegmentationLayer, MultiChunkDecode) ll.decoded().clear(); DECODE(l, "12345E"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "12345"); ll.decoded().clear(); DECODE(l, "1234567890E"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "1234567890"); ll.decoded().clear(); DECODE(l, "123C"); DECODE(l, "45E"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "12345"); ll.decoded().clear(); DECODE(l, "123C"); DECODE(l, "456789E"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123456789"); ll.decoded().clear(); DECODE(l, "123C"); DECODE(l, "456789C"); DECODE(l, "E"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123456789"); } @@ -236,10 +236,10 @@ TEST(DebugArqLayer, SingleChunk) DECODE(l, "\x01" "123"); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(top.decoded().at(0), "123"); top.encode("abc", 3); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ( bottom.encoded().at(0), std::string( "\x81" @@ -251,10 +251,10 @@ TEST(DebugArqLayer, SingleChunk) DECODE(l, "\x02" "123"); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(top.decoded().at(0), "123"); top.encode("abc", 3); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ( bottom.encoded().at(0), std::string( "\x02" @@ -267,10 +267,10 @@ TEST(DebugArqLayer, SingleChunk) DECODE(l, "\x01" "123"); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(top.decoded().at(0), "123"); top.encode("abc", 3); - EXPECT_EQ(bottom.encoded().size(), 2); + EXPECT_EQ(bottom.encoded().size(), 2U); EXPECT_EQ(bottom.encoded().at(0), std::string("\x80", 1)); EXPECT_EQ( bottom.encoded().at(1), std::string( @@ -284,11 +284,11 @@ TEST(DebugArqLayer, SingleChunk) DECODE(l, "\x40\x13" "123"); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(top.decoded().at(0), "123"); top.encode("abc", 3, false); top.encode("def", 3); - EXPECT_EQ(bottom.encoded().size(), 2); + EXPECT_EQ(bottom.encoded().size(), 2U); EXPECT_EQ(bottom.encoded().at(0), "\x80"); EXPECT_EQ( bottom.encoded().at(1), @@ -313,12 +313,12 @@ TEST(DebugArqLayer, MultiChunk) DECODE(l, "\x04" "456"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); EXPECT_EQ(top.decoded().at(0), "123"); EXPECT_EQ(top.decoded().at(1), "456"); top.encode("abc", 3); top.encode("defg", 4); - EXPECT_EQ(bottom.encoded().size(), 3); + EXPECT_EQ(bottom.encoded().size(), 3U); EXPECT_EQ(bottom.encoded().at(0), "\x80"); EXPECT_EQ( bottom.encoded().at(1), @@ -337,13 +337,13 @@ TEST(DebugArqLayer, MultiChunk) DECODE(l, "\x06" "456"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); EXPECT_EQ(top.decoded().at(0), "123"); EXPECT_EQ(top.decoded().at(1), "456"); top.encode("abc", 3, false); top.encode("defg", 4); top.encode("hi", 2); - EXPECT_EQ(bottom.encoded().size(), 2); + EXPECT_EQ(bottom.encoded().size(), 2U); EXPECT_EQ( bottom.encoded().at(0), std::string( "\x03" @@ -371,7 +371,7 @@ TEST(DebugArqLayer, LostRequest) DECODE(l, "\x02" "456"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); EXPECT_EQ(top.decoded().at(0), "123"); EXPECT_EQ(top.decoded().at(1), "456"); // Assume last part is lost. @@ -379,15 +379,15 @@ TEST(DebugArqLayer, LostRequest) DECODE(l, "\x02" "456"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); DECODE(l, "\x04" "zzz"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); DECODE(l, "\x20" "..."); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); // Reset and retransmit full request DECODE(l, "\x80"); DECODE(l, @@ -399,13 +399,13 @@ TEST(DebugArqLayer, LostRequest) DECODE(l, "\x03" "789"); - EXPECT_EQ(top.decoded().size(), 5); + EXPECT_EQ(top.decoded().size(), 5U); EXPECT_EQ(top.decoded().at(2), "123"); EXPECT_EQ(top.decoded().at(3), "456"); EXPECT_EQ(top.decoded().at(4), "789"); top.encode("abc", 3); - EXPECT_EQ(bottom.encoded().size(), 2); + EXPECT_EQ(bottom.encoded().size(), 2U); EXPECT_EQ(bottom.encoded().at(0), "\x80"); EXPECT_EQ( bottom.encoded().at(1), std::string( @@ -416,7 +416,7 @@ TEST(DebugArqLayer, LostRequest) top.decoded().clear(); bottom.encoded().clear(); DECODE(l, "\x80"); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ(bottom.encoded().at(0), "\x80"); DECODE(l, "\x01" @@ -424,7 +424,7 @@ TEST(DebugArqLayer, LostRequest) DECODE(l, "\x02" "456"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); EXPECT_EQ(top.decoded().at(0), "123"); EXPECT_EQ(top.decoded().at(1), "456"); // Do some retransmit @@ -443,7 +443,7 @@ TEST(DebugArqLayer, LostRequest) DECODE(l, "\x03" "567"); - EXPECT_EQ(top.decoded().size(), 3); + EXPECT_EQ(top.decoded().size(), 3U); EXPECT_EQ(top.decoded().at(2), "567"); } @@ -458,17 +458,17 @@ TEST(DebugArqLayer, LostResponse) top.decoded().clear(); bottom.encoded().clear(); DECODE(l, "\x8F"); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ(bottom.encoded().at(0), "\x80"); bottom.encoded().clear(); DECODE(l, "\x10" "123"); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(top.decoded().at(0), "123"); top.encode("abc", 3); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ( bottom.encoded().at(0), std::string( "\x01" @@ -479,8 +479,8 @@ TEST(DebugArqLayer, LostResponse) DECODE(l, "\x10" "123"); - EXPECT_EQ(top.decoded().size(), 1); - EXPECT_EQ(bottom.encoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 1U); + EXPECT_EQ(bottom.encoded().size(), 2U); EXPECT_EQ( bottom.encoded().at(1), std::string( "\x01" @@ -490,12 +490,12 @@ TEST(DebugArqLayer, LostResponse) DECODE(l, "\x11" "456"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); EXPECT_EQ(top.decoded().at(1), "456"); top.encode("def", 3, false); top.encode("g", 1); top.encode("hi", 2); - EXPECT_EQ(bottom.encoded().size(), 4); + EXPECT_EQ(bottom.encoded().size(), 4U); EXPECT_EQ( bottom.encoded().at(2), std::string( "\x02" @@ -508,7 +508,7 @@ TEST(DebugArqLayer, LostResponse) DECODE(l, "\x11" "456"); - EXPECT_EQ(bottom.encoded().size(), 6); + EXPECT_EQ(bottom.encoded().size(), 6U); EXPECT_EQ( bottom.encoded().at(4), std::string( "\x02" @@ -531,18 +531,18 @@ TEST(DebugArqLayer, Purgeable) top.decoded().clear(); bottom.encoded().clear(); DECODE(l, "\x80"); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ(bottom.encoded().at(0), "\x80"); bottom.encoded().clear(); DECODE(l, "\x01" "123"); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(top.decoded().at(0), "123"); top.setPurgeableResponse(); top.encode("abc", 3); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ( bottom.encoded().at(0), std::string( "\x01" @@ -553,11 +553,11 @@ TEST(DebugArqLayer, Purgeable) DECODE(l, "\x01" "123"); - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); EXPECT_EQ(top.decoded().at(1), "123"); top.setPurgeableResponse(); top.encode("def", 3); - EXPECT_EQ(bottom.encoded().size(), 2); + EXPECT_EQ(bottom.encoded().size(), 2U); EXPECT_EQ( bottom.encoded().at(1), std::string( "\x82" @@ -568,11 +568,11 @@ TEST(DebugArqLayer, Purgeable) DECODE(l, "\x01" "123"); - EXPECT_EQ(top.decoded().size(), 3); + EXPECT_EQ(top.decoded().size(), 3U); EXPECT_EQ(top.decoded().at(2), "123"); // Default to precious, but reset flag remains. top.encode("ghi", 3); - EXPECT_EQ(bottom.encoded().size(), 3); + EXPECT_EQ(bottom.encoded().size(), 3U); EXPECT_EQ( bottom.encoded().at(2), std::string( "\x83" @@ -582,9 +582,9 @@ TEST(DebugArqLayer, Purgeable) DECODE(l, "\x01" "123"); - EXPECT_EQ(top.decoded().size(), 3); + EXPECT_EQ(top.decoded().size(), 3U); // Default to precious, but reset flag remains. - EXPECT_EQ(bottom.encoded().size(), 4); + EXPECT_EQ(bottom.encoded().size(), 4U); EXPECT_EQ( bottom.encoded().at(3), std::string( "\x83" @@ -594,10 +594,10 @@ TEST(DebugArqLayer, Purgeable) DECODE(l, "\x02" "123"); - EXPECT_EQ(top.decoded().size(), 4); + EXPECT_EQ(top.decoded().size(), 4U); // Default to precious. top.encode("jkl", 3); - EXPECT_EQ(bottom.encoded().size(), 5); + EXPECT_EQ(bottom.encoded().size(), 5U); EXPECT_EQ( bottom.encoded().at(4), std::string( "\x04" @@ -616,15 +616,15 @@ TEST(DebugArqLayer, Overflow) top.decoded().clear(); bottom.encoded().clear(); DECODE(l, "\x80"); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ(bottom.encoded().at(0), "\x80"); bottom.encoded().clear(); DECODE(l, "\x01" "123"); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); top.encode("abcde", 5); - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); EXPECT_EQ( bottom.encoded().at(0), std::string( "\x01" @@ -635,9 +635,9 @@ TEST(DebugArqLayer, Overflow) "\x01" "123"); // Behave like purgeable. - EXPECT_EQ(top.decoded().size(), 2); + EXPECT_EQ(top.decoded().size(), 2U); top.encode("fghij", 5); - EXPECT_EQ(bottom.encoded().size(), 2); + EXPECT_EQ(bottom.encoded().size(), 2U); EXPECT_EQ( bottom.encoded().at(1), std::string( "\x82" @@ -647,9 +647,9 @@ TEST(DebugArqLayer, Overflow) DECODE(l, "\x02" "456"); - EXPECT_EQ(top.decoded().size(), 3); + EXPECT_EQ(top.decoded().size(), 3U); top.encode("klm", 3); - EXPECT_EQ(bottom.encoded().size(), 3); + EXPECT_EQ(bottom.encoded().size(), 3U); EXPECT_EQ( bottom.encoded().at(2), std::string( "\x03" @@ -659,8 +659,8 @@ TEST(DebugArqLayer, Overflow) DECODE(l, "\x02" "456"); - EXPECT_EQ(top.decoded().size(), 3); - EXPECT_EQ(bottom.encoded().size(), 4); + EXPECT_EQ(top.decoded().size(), 3U); + EXPECT_EQ(bottom.encoded().size(), 4U); EXPECT_EQ( bottom.encoded().at(3), std::string( "\x03" @@ -676,22 +676,22 @@ TEST(Crc8Layer, Encode) ll.encoded().clear(); l.encode(); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "\xff"); ll.encoded().clear(); l.encode("1", 1); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1\x5e"); ll.encoded().clear(); l.encode("12", 2); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "12\x54"); ll.encoded().clear(); l.encode("123", 3); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123\xfc"); } @@ -703,33 +703,33 @@ TEST(Crc8Layer, Decode) ll.decoded().clear(); DECODE(l, "\xff"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), ""); ll.decoded().clear(); DECODE(l, "1\x5e"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "1"); ll.decoded().clear(); DECODE(l, "12\x54"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "12"); ll.decoded().clear(); DECODE(l, "123\xfc"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123"); ll.decoded().clear(); DECODE(l, "1234\xfc"); - EXPECT_EQ(ll.decoded().size(), 0); + EXPECT_EQ(ll.decoded().size(), 0U); ll.decoded().clear(); DECODE(l, "\x00" "123\xfc"); - EXPECT_EQ(ll.decoded().size(), 0); + EXPECT_EQ(ll.decoded().size(), 0U); } TEST(Crc16Layer, Encode) @@ -740,22 +740,22 @@ TEST(Crc16Layer, Encode) ll.encoded().clear(); l.encode(); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "\xff\xff"); ll.encoded().clear(); l.encode("1", 1); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1\x49\xD6"); ll.encoded().clear(); l.encode("12", 2); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "12\x77\xA2"); ll.encoded().clear(); l.encode("123", 3); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123\x1C\x84"); } @@ -767,33 +767,33 @@ TEST(Crc16Layer, Decode) ll.decoded().clear(); DECODE(l, "\xff\xff"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), ""); ll.decoded().clear(); DECODE(l, "1\x49\xd6"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "1"); ll.decoded().clear(); DECODE(l, "12\x77\xa2"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "12"); ll.decoded().clear(); DECODE(l, "123\x1c\x84"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123"); ll.decoded().clear(); DECODE(l, "1234\x1c\x84"); - EXPECT_EQ(ll.decoded().size(), 0); + EXPECT_EQ(ll.decoded().size(), 0U); ll.decoded().clear(); DECODE(l, "\x00" "123\x1c\x84"); - EXPECT_EQ(ll.decoded().size(), 0); + EXPECT_EQ(ll.decoded().size(), 0U); } TEST(Crc32Layer, Encode) @@ -804,7 +804,7 @@ TEST(Crc32Layer, Encode) ll.encoded().clear(); l.encode(); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0)[0], '\x00'); EXPECT_EQ(ll.encoded().at(0)[1], '\x00'); EXPECT_EQ(ll.encoded().at(0)[2], '\x00'); @@ -812,22 +812,22 @@ TEST(Crc32Layer, Encode) ll.encoded().clear(); l.encode("1", 1); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1\x83\xDC\xEF\xB7"); ll.encoded().clear(); l.encode("12", 2); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "12OSD\xCD"); ll.encoded().clear(); l.encode("123", 3); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123\x88Hc\xD2"); ll.encoded().clear(); l.encode("1234", 4); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234\x9B\xE3\xE0\xA3"); } @@ -839,37 +839,37 @@ TEST(Crc32Layer, Decode) ll.decoded().clear(); DECODE(l, "\0\0\0\0"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), ""); ll.decoded().clear(); DECODE(l, "1\x83\xDC\xEF\xB7"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "1"); ll.decoded().clear(); DECODE(l, "12OSD\xCD"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "12"); ll.decoded().clear(); DECODE(l, "123\x88Hc\xD2"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "123"); ll.decoded().clear(); DECODE(l, "1234\x9B\xE3\xE0\xA3"); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); ll.decoded().clear(); DECODE(l, "123\x9B\xE3\xE0\xA3"); - EXPECT_EQ(ll.decoded().size(), 0); + EXPECT_EQ(ll.decoded().size(), 0U); ll.decoded().clear(); DECODE(l, "\x00" "1234\x9B\xE3\xE0\xA3"); - EXPECT_EQ(ll.decoded().size(), 0); + EXPECT_EQ(ll.decoded().size(), 0U); } TEST(BufferLayer, Encode) @@ -880,47 +880,47 @@ TEST(BufferLayer, Encode) ll.encoded().clear(); l.encode("123", 3); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123"); ll.encoded().clear(); l.encode("12", 2, false); l.encode("3", 1); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123"); ll.encoded().clear(); l.encode("12", 2, false); l.encode("3", 1, false); l.encode(); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "123"); ll.encoded().clear(); l.encode("1234", 4, true); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234"); ll.encoded().clear(); l.encode("1234", 4, false); l.encode(); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234"); ll.encoded().clear(); l.encode("12345", 5, true); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "12345"); ll.encoded().clear(); l.encode("12345", 5, false); l.encode("67", 2, true); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234567"); ll.encoded().clear(); l.encode("1234567890", 10, true); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "1234567890"); } @@ -981,7 +981,7 @@ TEST(ArqLayer, Retransmit) DECODE(bottom, "\x80"); top.flush(); // no retransmit - EXPECT_EQ(bottom.encoded().size(), 3); + EXPECT_EQ(bottom.encoded().size(), 3U); top.clear(); bottom.clear(); @@ -990,7 +990,7 @@ TEST(ArqLayer, Retransmit) EXPECT_EQ(bottom.encoded().at(0), "\x01 1"); top.encode(" 2", 2); // Does not retransmit. - EXPECT_EQ(bottom.encoded().size(), 1); + EXPECT_EQ(bottom.encoded().size(), 1U); l.keepAlive(); // triggers retransmit of 1 EXPECT_EQ(bottom.encoded().at(1), "\x01 1"); @@ -1003,7 +1003,7 @@ TEST(ArqLayer, Retransmit) // Wrong ack DECODE(bottom, "\x83"); // ignored - EXPECT_EQ(bottom.encoded().size(), 4); + EXPECT_EQ(bottom.encoded().size(), 4U); DECODE(bottom, "\x82"); top.clear(); @@ -1044,7 +1044,7 @@ TEST(ArqLayer, KeepAlive) DECODE(bottom, "\x82"); DECODE(bottom, "\x41"); - EXPECT_EQ(top.decoded().size(), 0); + EXPECT_EQ(top.decoded().size(), 0U); EXPECT_EQ(bottom.encoded().at(3), "\x81"); } @@ -1242,7 +1242,7 @@ TEST(FileLayer, NamedPipe) l.encode("Zip-a-Dee-Doo-Dah", 17); char buf[32] = {}; - EXPECT_EQ(read(fd, buf, sizeof(buf)), 17); + EXPECT_EQ(read(fd, buf, sizeof(buf)), 17U); EXPECT_EQ(std::string(buf), "Zip-a-Dee-Doo-Dah"); close(fd); @@ -1322,7 +1322,7 @@ TEST(FifoLoopback1, FifoLoopback1) l.encode("night", 5); EXPECT_EQ(l.recv(), 0); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(top.decoded().at(0), "This is the night"); l.encode("It's a beautiful night", 22); @@ -1330,11 +1330,11 @@ TEST(FifoLoopback1, FifoLoopback1) l.encode("bella notte", 11, false); l.encode(); - EXPECT_EQ(top.decoded().size(), 1); + EXPECT_EQ(top.decoded().size(), 1U); EXPECT_EQ(l.recv(), 0); EXPECT_EQ(l.recv(), 0); EXPECT_EQ(l.recv(), EAGAIN); - EXPECT_EQ(top.decoded().size(), 3); + EXPECT_EQ(top.decoded().size(), 3U); } TEST(FifoLoopback, FifoLoopback) @@ -1348,15 +1348,15 @@ TEST(FifoLoopback, FifoLoopback) a.encode("the ", 4); EXPECT_EQ(l.b2a().recv(), EAGAIN); b.encode("skies", 5); - EXPECT_EQ(a.decoded().size(), 0); - EXPECT_EQ(b.decoded().size(), 0); + EXPECT_EQ(a.decoded().size(), 0U); + EXPECT_EQ(b.decoded().size(), 0U); EXPECT_EQ(l.a2b().recv(), 0); - EXPECT_EQ(b.decoded().size(), 1); + EXPECT_EQ(b.decoded().size(), 1U); EXPECT_EQ(b.decoded().at(0), "Look the "); EXPECT_EQ(l.b2a().recv(), 0); - EXPECT_EQ(a.decoded().size(), 1); + EXPECT_EQ(a.decoded().size(), 1U); EXPECT_EQ(a.decoded().at(0), "at skies"); EXPECT_EQ(l.a2b().lastError(), 0); @@ -1423,11 +1423,11 @@ TEST(TerminalLayer, Encode) ll.encoded().clear(); l.encode("You can learn a lot", 19); - EXPECT_EQ(ll.encoded().size(), 1); + EXPECT_EQ(ll.encoded().size(), 1U); EXPECT_EQ(ll.encoded().at(0), "\x1b_You can learn a lot\x1b\\"); l.nonDebugEncode("of things", 9); - EXPECT_EQ(ll.encoded().size(), 2); + EXPECT_EQ(ll.encoded().size(), 2U); EXPECT_EQ(ll.encoded().at(1), "of things"); } @@ -1441,7 +1441,7 @@ TEST(TerminalLayer, Decode) DECODE(l, "from the \x1b_flowers\x1b\\..."); EXPECT_EQ(nonDebug, "from the ..."); - EXPECT_EQ(ll.decoded().size(), 1); + EXPECT_EQ(ll.decoded().size(), 1U); EXPECT_EQ(ll.decoded().at(0), "flowers"); } @@ -1477,7 +1477,7 @@ TEST(MuxLayer, Decode) stored::MuxLayer l{{ch0, ch1, ch2}}; DECODE(l, "ch?"); - EXPECT_EQ(ch0.decoded().size(), 0); + EXPECT_EQ(ch0.decoded().size(), 0U); DECODE(l, "\x10\x00 ch0"); EXPECT_EQ(ch0.decoded().at(0), " ch0"); @@ -1503,7 +1503,7 @@ TEST(MuxLayer, Decode) EXPECT_EQ(ch0.decoded().at(2), " 0 1"); DECODE(l, "\x10\x03 ch?"); - EXPECT_EQ(ch0.decoded().size(), 3); + EXPECT_EQ(ch0.decoded().size(), 3U); } TEST(Aes256Layer, EncodeDecode) diff --git a/tests/test_spm.cpp b/tests/test_spm.cpp index f2509e0b..e0836821 100644 --- a/tests/test_spm.cpp +++ b/tests/test_spm.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -11,14 +11,14 @@ TEST(ScrachPad, Alloc) { stored::ScratchPad<> spm; - EXPECT_EQ(spm.chunks(), 0); - EXPECT_EQ(spm.size(), 0); - EXPECT_EQ(spm.max(), 0); + EXPECT_EQ(spm.chunks(), 0U); + EXPECT_EQ(spm.size(), 0U); + EXPECT_EQ(spm.max(), 0U); // First chunk alloc. void** a = spm.alloc(); EXPECT_NE(a, nullptr); - EXPECT_EQ(spm.chunks(), 1); + EXPECT_EQ(spm.chunks(), 1U); EXPECT_EQ(spm.size(), sizeof(void*)); EXPECT_EQ(spm.max(), sizeof(void*)); EXPECT_GE(spm.capacity(), sizeof(void*)); @@ -27,9 +27,9 @@ TEST(ScrachPad, Alloc) void** b = spm.alloc(); EXPECT_NE(b, nullptr); EXPECT_NE(a, b); - EXPECT_EQ(spm.size(), sizeof(void*) * 2); - EXPECT_EQ(spm.max(), sizeof(void*) * 2); - EXPECT_GE(spm.capacity(), sizeof(void*) * 2); + EXPECT_EQ(spm.size(), sizeof(void*) * 2U); + EXPECT_EQ(spm.max(), sizeof(void*) * 2U); + EXPECT_GE(spm.capacity(), sizeof(void*) * 2U); // Random allocs. EXPECT_NE(spm.alloc(10), nullptr); @@ -51,9 +51,9 @@ TEST(ScratchPad, Reset) // Empty reset. spm.reset(); - EXPECT_EQ(spm.chunks(), 0); - EXPECT_EQ(spm.size(), 0); - EXPECT_EQ(spm.max(), 0); + EXPECT_EQ(spm.chunks(), 0U); + EXPECT_EQ(spm.size(), 0U); + EXPECT_EQ(spm.max(), 0U); // First chunk reset. int* i = spm.alloc(); @@ -61,8 +61,8 @@ TEST(ScratchPad, Reset) *i = 42; spm.reset(); spmInfo(spm); - EXPECT_EQ(spm.chunks(), 1); - EXPECT_EQ(spm.size(), 0); + EXPECT_EQ(spm.chunks(), 1U); + EXPECT_EQ(spm.size(), 0U); EXPECT_EQ(spm.max(), sizeof(int)); // Force an additional chunk. @@ -73,11 +73,11 @@ TEST(ScratchPad, Reset) size_t total = spm.size(); spmInfo(spm); EXPECT_NE(p, nullptr); - EXPECT_EQ(spm.chunks(), 2); + EXPECT_EQ(spm.chunks(), 2U); spm.reset(); spmInfo(spm); - EXPECT_EQ(spm.chunks(), 1); - EXPECT_EQ(spm.size(), 0); + EXPECT_EQ(spm.chunks(), 1U); + EXPECT_EQ(spm.size(), 0U); EXPECT_GE(spm.capacity(), total); } @@ -93,8 +93,8 @@ TEST(ScratchPad, Alignment) // Add padding bytes for int int* i = spm.alloc(); EXPECT_NE(i, nullptr); - EXPECT_EQ((uintptr_t)i & (sizeof(int) - 1), 0); - EXPECT_EQ(spm.size(), sizeof(int) * 2); + EXPECT_EQ((uintptr_t)i & (sizeof(int) - 1), 0U); + EXPECT_EQ(spm.size(), sizeof(int) * 2U); // Another few bytes c = spm.alloc(); @@ -107,7 +107,7 @@ TEST(ScratchPad, Alignment) // More padding for double double* d = spm.alloc(); EXPECT_NE(d, nullptr); - EXPECT_EQ((uintptr_t)d & (sizeof(double) - 1), 0); + EXPECT_EQ((uintptr_t)d & (sizeof(double) - 1), 0U); EXPECT_EQ(spm.size(), sizeof(int) * 2 + sizeof(void*) + sizeof(double)); } @@ -118,29 +118,29 @@ TEST(ScratchPad, Snapshot) auto c = spm.alloc(); EXPECT_NE(c, nullptr); - EXPECT_EQ(spm.size(), 1); + EXPECT_EQ(spm.size(), 1U); // Rollback within same chunk. auto s1 = spm.snapshot(); c = spm.alloc(); EXPECT_NE(c, nullptr); - EXPECT_EQ(spm.size(), 2); + EXPECT_EQ(spm.size(), 2U); s1.rollback(); - EXPECT_EQ(spm.size(), 1); + EXPECT_EQ(spm.size(), 1U); auto d = spm.alloc(); EXPECT_NE(d, nullptr); EXPECT_EQ(spm.size(), sizeof(void*) + sizeof(double)); s1.rollback(); - EXPECT_EQ(spm.size(), 1); + EXPECT_EQ(spm.size(), 1U); // Rollback to previous chunk. c = spm.alloc(spm.capacity() - spm.size() + 1); EXPECT_NE(c, nullptr); - EXPECT_EQ(spm.chunks(), 2); + EXPECT_EQ(spm.chunks(), 2U); s1.rollback(); - EXPECT_EQ(spm.size(), 1); - EXPECT_EQ(spm.chunks(), 1); + EXPECT_EQ(spm.size(), 1U); + EXPECT_EQ(spm.chunks(), 1U); } TEST(ScratchPad, Shrink) @@ -160,16 +160,16 @@ TEST(ScratchPad, Shrink) auto c = spm.alloc(spm.capacity() + 1); EXPECT_NE(c, nullptr); - EXPECT_EQ(spm.chunks(), 2); + EXPECT_EQ(spm.chunks(), 2U); s.rollback(); s.reset(); - EXPECT_EQ(spm.chunks(), 1); + EXPECT_EQ(spm.chunks(), 1U); spm.reset(); - EXPECT_EQ(spm.chunks(), 1); + EXPECT_EQ(spm.chunks(), 1U); spm.shrink_to_fit(); - EXPECT_EQ(spm.chunks(), 0); - EXPECT_EQ(spm.capacity(), 0); + EXPECT_EQ(spm.chunks(), 0U); + EXPECT_EQ(spm.capacity(), 0U); } TEST(ScratchPad, Stress) diff --git a/tests/test_synchronizer.cpp b/tests/test_synchronizer.cpp index 1cda3af0..308d1f6e 100644 --- a/tests/test_synchronizer.cpp +++ b/tests/test_synchronizer.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -25,20 +25,20 @@ namespace { TEST(Synchronizer, Endianness) { - EXPECT_EQ(stored::swap_endian(1), 1); - EXPECT_EQ(stored::swap_endian(0x1234), 0x3412); - EXPECT_EQ(stored::swap_endian(0x12345678), 0x78563412); + EXPECT_EQ(stored::swap_endian(1), 1U); + EXPECT_EQ(stored::swap_endian(0x1234), 0x3412U); + EXPECT_EQ(stored::swap_endian(0x12345678), 0x78563412U); uint8_t b[] = {1, 2, 3}; stored::swap_endian_<3>(b); - EXPECT_EQ(b[0], 3); - EXPECT_EQ(b[1], 2); - EXPECT_EQ(b[2], 1); + EXPECT_EQ(b[0], 3U); + EXPECT_EQ(b[1], 2U); + EXPECT_EQ(b[2], 1U); stored::swap_endian(b, 2); - EXPECT_EQ(b[0], 2); - EXPECT_EQ(b[1], 3); - EXPECT_EQ(b[2], 1); + EXPECT_EQ(b[0], 2U); + EXPECT_EQ(b[1], 3U); + EXPECT_EQ(b[2], 1U); } TEST(Synchronizer, Instantiate) @@ -66,7 +66,7 @@ TEST(Synchronizer, ShortSeq) { TestJournal j("123", nullptr, 0u); - EXPECT_EQ(j.seq(), 1); + EXPECT_EQ(j.seq(), 1U); j.changed(1, 0); EXPECT_TRUE(j.hasChanged(1, 1)); @@ -74,29 +74,29 @@ TEST(Synchronizer, ShortSeq) for(int i = 1; i < 50; i++) j.bumpSeq(true); - EXPECT_EQ(j.seq(), 50); + EXPECT_EQ(j.seq(), 50U); EXPECT_FALSE(j.hasChanged(1, 2)); - EXPECT_EQ(j.toShort(50), 50); - EXPECT_EQ(j.toShort(49), 49); - EXPECT_EQ(j.toShort(1), 1); + EXPECT_EQ(j.toShort(50), 50U); + EXPECT_EQ(j.toShort(49), 49U); + EXPECT_EQ(j.toShort(1), 1U); - EXPECT_EQ(j.toLong(50), 50); - EXPECT_EQ(j.toLong(49), 49); - EXPECT_EQ(j.toLong(1), 1); + EXPECT_EQ(j.toLong(50), 50U); + EXPECT_EQ(j.toLong(49), 49U); + EXPECT_EQ(j.toLong(1), 1U); for(int i = 0; i < 0x10000; i++) j.bumpSeq(true); - EXPECT_EQ(j.toShort(0x10032), 50); - EXPECT_EQ(j.toShort(0x10031), 49); - EXPECT_EQ(j.toShort(0x10001), 1); - EXPECT_EQ(j.toShort(51), 51); + EXPECT_EQ(j.toShort(0x10032), 50U); + EXPECT_EQ(j.toShort(0x10031), 49U); + EXPECT_EQ(j.toShort(0x10001), 1U); + EXPECT_EQ(j.toShort(51), 51U); - EXPECT_EQ(j.toLong(51), 51); - EXPECT_EQ(j.toLong(50), 0x10032); - EXPECT_EQ(j.toLong(49), 0x10031); - EXPECT_EQ(j.toLong(1), 0x10001); + EXPECT_EQ(j.toLong(51), 51U); + EXPECT_EQ(j.toLong(50), 0x10032U); + EXPECT_EQ(j.toLong(49), 0x10031U); + EXPECT_EQ(j.toLong(1), 0x10001U); EXPECT_TRUE(j.hasChanged( 1, j.seq() - TestJournal::ShortSeqWindow + TestJournal::SeqLowerMargin)); @@ -113,7 +113,7 @@ TEST(Synchronizer, Changes) size_t c = 0; store.journal().iterateChanged(0, [&](stored::StoreJournal::Key) { c++; }); - EXPECT_EQ(c, 0); + EXPECT_EQ(c, 0U); stored::StoreJournal::Key key_u8 = (stored::StoreJournal::Key)u8.key(); EXPECT_FALSE(store.journal().hasChanged(key_u8, now)); @@ -123,7 +123,7 @@ TEST(Synchronizer, Changes) c = 0; store.journal().iterateChanged(0, [&](stored::StoreJournal::Key) { c++; }); - EXPECT_EQ(c, 1); + EXPECT_EQ(c, 1U); now = store.journal().seq(); store.default_uint8 = 2; @@ -144,29 +144,29 @@ TEST(Synchronizer, Changes) c = 0; store.journal().iterateChanged(0, [&](stored::StoreJournal::Key) { c++; }); - EXPECT_EQ(c, 2); + EXPECT_EQ(c, 2U); } -#define EXPECT_SYNCED(store1, store2) \ - do { \ - auto _map1 = (store1).map(); \ - auto _map2 = (store2).map(); \ - for(auto& _o : _map1) \ - EXPECT_EQ(_o.second.get(), _map2[_o.first].get()); \ - } while(0) - -#define EXPECT_NOT_SYNCED(store1, store2) \ - do { \ - auto _map1 = (store1).map(); \ - auto _map2 = (store2).map(); \ - bool _synced = true; \ - for(auto& _o : _map1) \ - if(_o.second.get() != _map2[_o.first].get()) { \ - _synced = false; \ - break; \ - } \ - EXPECT_FALSE(_synced); \ - } while(0) +#define EXPECT_SYNCED(store1, store2) \ + do { \ + auto _map1 = (store1).map(); \ + auto _map2 = (store2).map(); \ + for(auto& _o : _map1) \ + EXPECT_EQ(_o.second.get(), _map2[_o.first].get()); \ + } while(0) + +#define EXPECT_NOT_SYNCED(store1, store2) \ + do { \ + auto _map1 = (store1).map(); \ + auto _map2 = (store2).map(); \ + bool _synced = true; \ + for(auto& _o : _map1) \ + if(_o.second.get() != _map2[_o.first].get()) { \ + _synced = false; \ + break; \ + } \ + EXPECT_FALSE(_synced); \ + } while(0) TEST(Synchronizer, Sync2) { diff --git a/tests/test_types.cpp b/tests/test_types.cpp index 6615ae64..cd727b24 100644 --- a/tests/test_types.cpp +++ b/tests/test_types.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2020-2023 Jochem Rutgers +// SPDX-FileCopyrightText: 2020-2025 Jochem Rutgers // // SPDX-License-Identifier: MPL-2.0 @@ -8,12 +8,12 @@ #include "gtest/gtest.h" #ifdef STORED_OS_WINDOWS -# include -# ifndef alloca -# define alloca(s) _malloca(s) -# endif +# include +# ifndef alloca +# define alloca(s) _malloca(s) +# endif #else -# include +# include #endif namespace { @@ -55,47 +55,47 @@ TEST(Types, Int64) TEST(Types, Uint8) { stored::TestStore store; - EXPECT_EQ(store.default_uint8.get(), 0); - store.default_uint8 = 42; - EXPECT_EQ(store.default_uint8.get(), 42); + EXPECT_EQ(store.default_uint8.get(), 0U); + store.default_uint8 = 42U; + EXPECT_EQ(store.default_uint8.get(), 42U); } TEST(Types, Uint16) { stored::TestStore store; - EXPECT_EQ(store.default_uint16.get(), 0); - store.default_uint16 = 0x1234; - EXPECT_EQ(store.default_uint16.get(), 0x1234); + EXPECT_EQ(store.default_uint16.get(), 0U); + store.default_uint16 = 0x1234U; + EXPECT_EQ(store.default_uint16.get(), 0x1234U); } TEST(Types, Uint32) { stored::TestStore store; - EXPECT_EQ(store.default_uint32.get(), 0); - store.default_uint32 = 0x8abcdef0; - EXPECT_EQ(store.default_uint32.get(), 0x8abcdef0); + EXPECT_EQ(store.default_uint32.get(), 0U); + store.default_uint32 = 0x8abcdef0U; + EXPECT_EQ(store.default_uint32.get(), 0x8abcdef0U); } TEST(Types, Uint64) { stored::TestStore store; - EXPECT_EQ(store.default_uint64.get(), 0); - store.default_uint64 = 0xf123456789abcdefull; - EXPECT_EQ(store.default_uint64.get(), 0xf123456789abcdefull); + EXPECT_EQ(store.default_uint64.get(), 0ULL); + store.default_uint64 = 0xf123456789abcdefULL; + EXPECT_EQ(store.default_uint64.get(), 0xf123456789abcdefULL); } TEST(Types, Float) { stored::TestStore store; - EXPECT_EQ(store.default_float.get(), 0); - store.default_float = 3.14f; - EXPECT_FLOAT_EQ(store.default_float.get(), 3.14f); + EXPECT_EQ(store.default_float.get(), 0.F); + store.default_float = 3.14F; + EXPECT_FLOAT_EQ(store.default_float.get(), 3.14F); } TEST(Types, Double) { stored::TestStore store; - EXPECT_EQ(store.default_double.get(), 0); + EXPECT_EQ(store.default_double.get(), 0.); store.default_double = 3.14; EXPECT_DOUBLE_EQ(store.default_double.get(), 3.14); } @@ -150,7 +150,7 @@ TEST(Types, String) memset(buffer1, 0, s + 1); char* buffer2 = (char*)alloca(s + 1); memset(buffer2, 0, s + 1); - EXPECT_EQ(store.default_string.get(buffer2, s), 0); + EXPECT_EQ(store.default_string.get(buffer2, s), 0U); for(size_t i = 0; i < s + 1; i++) buffer1[i] = 'a'; @@ -166,8 +166,8 @@ TEST(Types, String) ASSERT_TRUE(s >= 4); memcpy(buffer1, "a\0b\0", 4); - EXPECT_EQ(store.default_string.set(buffer1, s), 1); - EXPECT_EQ(store.default_string.get(buffer2, s), 1); + EXPECT_EQ(store.default_string.set(buffer1, s), 1U); + EXPECT_EQ(store.default_string.get(buffer2, s), 1U); } TEST(Types, FreeVariable) @@ -179,10 +179,10 @@ TEST(Types, FreeVariable) auto v = f.apply(store); v = 10; - EXPECT_EQ(store.default_uint8.get(), 10); + EXPECT_EQ(store.default_uint8.get(), 10U); - store.default_uint8 = 11; - EXPECT_EQ(v.get(), 11); + store.default_uint8 = 11U; + EXPECT_EQ(v.get(), 11U); } } // namespace From 6a989d27eee7702da533f02a747ca8f19256d903 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:10:13 +0100 Subject: [PATCH 15/19] compile fix --- examples/lib/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/lib/CMakeLists.txt b/examples/lib/CMakeLists.txt index 4760e42b..d76fac45 100644 --- a/examples/lib/CMakeLists.txt +++ b/examples/lib/CMakeLists.txt @@ -4,3 +4,12 @@ add_library(example_lib STATIC include/getopt_mini.h src/getopt_mini.cpp) target_include_directories(example_lib PUBLIC include) + +if(MSVC) + target_compile_options(example_lib PRIVATE /W1) + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_options(example_lib PUBLIC /MTd) + else() + target_compile_options(example_lib PUBLIC /MT) + endif() +endif() From 56e7abc101926e37f58ddb9b2ee9dd5c39ccfa4e Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:24:50 +0100 Subject: [PATCH 16/19] compile fix --- examples/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index d9ec1b9d..da8d4223 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -4,6 +4,12 @@ if(MSVC) add_compile_options(/Wall /WX) + + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + add_compile_options(/MTd) + else() + add_compile_options(/MT) + endif() else() add_compile_options( -Wall From 5ac439e38a24fa599c9ed12379a3fc4fc3074a73 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:32:16 +0100 Subject: [PATCH 17/19] compile fix --- dist/common/FindHeatshrink.cmake | 5 ----- dist/common/FindTinyAES.cmake | 5 ----- examples/lossy_sync/main.cpp | 5 ++++- include/libstored/protocol.h | 6 +++++- tests/CMakeLists.txt | 6 ------ 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/dist/common/FindHeatshrink.cmake b/dist/common/FindHeatshrink.cmake index 3ba459d9..bc8a2d95 100644 --- a/dist/common/FindHeatshrink.cmake +++ b/dist/common/FindHeatshrink.cmake @@ -50,11 +50,6 @@ set_source_files_properties(${heatshrink_src} PROPERTIES GENERATED 1) if(MSVC) target_compile_options(heatshrink PRIVATE /W1) - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - target_compile_options(heatshrink PUBLIC /MTd) - else() - target_compile_options(heatshrink PUBLIC /MT) - endif() endif() target_include_directories( diff --git a/dist/common/FindTinyAES.cmake b/dist/common/FindTinyAES.cmake index 41dfcdb9..5f005260 100644 --- a/dist/common/FindTinyAES.cmake +++ b/dist/common/FindTinyAES.cmake @@ -45,11 +45,6 @@ set_source_files_properties(${tinyaes_src} PROPERTIES GENERATED 1) if(MSVC) target_compile_options(tinyaes PRIVATE /W1) - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - target_compile_options(tinyaes PUBLIC /MTd) - else() - target_compile_options(tinyaes PUBLIC /MT) - endif() endif() target_include_directories( diff --git a/examples/lossy_sync/main.cpp b/examples/lossy_sync/main.cpp index ff1f15b6..241a81c2 100644 --- a/examples/lossy_sync/main.cpp +++ b/examples/lossy_sync/main.cpp @@ -8,6 +8,8 @@ * channel. */ +#define _CRT_SECURE_NO_WARNINGS 1 + #include "ExampleSync.h" #include @@ -196,13 +198,14 @@ static Arguments parse_arguments(int argc, char** argv) case 'e': { // flawfinder: ignore FILE* f = fopen(optarg, "rb"); - args.key.resize(stored::Aes256Layer::KeySize); if(!f) { log("Cannot open key file '%s'; %s\n", optarg, strerror(errno)); STORED_throw(std::invalid_argument{"Cannot open key file"}); } + args.key.resize(stored::Aes256Layer::KeySize); + if(fread(&args.key[0], stored::Aes256Layer::KeySize, 1, f) != 1) { log("Cannot read key file '%s'; %s\n", optarg, strerror(errno)); fclose(f); diff --git a/include/libstored/protocol.h b/include/libstored/protocol.h index a5d99921..d54fb121 100644 --- a/include/libstored/protocol.h +++ b/include/libstored/protocol.h @@ -957,7 +957,11 @@ class Crc32Layer : public ProtocolLayer { public: typedef ProtocolLayer base; - enum STORED_ANONYMOUS { + enum STORED_ANONYMOUS +# if STORED_cplusplus >= 201103L + : uint32_t +# endif + { polynomial = 0x04c11db7, init = 0xffffffff, final_xor = 0xffffffff diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 88e431e5..2be6a444 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,12 +15,6 @@ set(LIBSTORED_DRAFT_API "ON") if(MSVC) add_compile_options(/W1 /WX) - - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - add_compile_options(/MTd) - else() - add_compile_options(/MT) - endif() else() add_compile_options( -Wall From ba0a73457d69d03a6391bfd1fb54df8b12352004 Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:35:05 +0100 Subject: [PATCH 18/19] prepare release --- CHANGELOG.rst | 14 +++++++++++++- version/version.txt | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index de57cf6c..847925d9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,18 @@ The format is based on `Keep a Changelog`_, and this project adheres to Added ````` +... + +.. _Unreleased: https://github.com/DEMCON/libstored/compare/v2.1.0...HEAD + + + +`2.1.0`_ - 2025-12-05 +--------------------- + +Added +````` + - Operations such as ``-=`` and ``++`` for store variables in C++. - YAML export of store meta-data. - ``stored::MuxLayer`` to multiplex multiple protocol layers over a single connection. @@ -46,7 +58,7 @@ Fixed - Init value of CRC32 in Python ``libstored.protocol.Crc32Layer``. -.. _Unreleased: https://github.com/DEMCON/libstored/compare/v2.0.0...HEAD +.. _2.1.0: https://github.com/DEMCON/libstored/releases/tag/v2.1.0 diff --git a/version/version.txt b/version/version.txt index ccd4e8c1..7ec1d6db 100644 --- a/version/version.txt +++ b/version/version.txt @@ -1 +1 @@ -2.1.0-alpha +2.1.0 From 62b3fe133e7d2c73717e2a1c1952c0a09506d49d Mon Sep 17 00:00:00 2001 From: Jochem Rutgers <68805714+jhrutgers@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:02:56 +0100 Subject: [PATCH 19/19] fix config --- dist/win32/build.cmd | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dist/win32/build.cmd b/dist/win32/build.cmd index 523b8cc8..db544b3a 100644 --- a/dist/win32/build.cmd +++ b/dist/win32/build.cmd @@ -18,6 +18,7 @@ set do_build=1 set support_test=1 set do_test= set msvc=1 +set config= :parse_param @@ -35,14 +36,17 @@ if %1 == --help ( ) if %1 == Debug ( set cmake_opts=%cmake_opts% -DCMAKE_BUILD_TYPE=%1 + set config=--config %1 goto next_param ) if %1 == RelWithDebInfo ( set cmake_opts=%cmake_opts% -DCMAKE_BUILD_TYPE=%1 + set config=--config %1 goto next_param ) if %1 == Release ( set cmake_opts=%cmake_opts% -DCMAKE_BUILD_TYPE=%1 + set config=--config %1 goto next_param ) if %1 == msvc ( @@ -201,18 +205,18 @@ if errorlevel 1 goto error if not "%do_build%" == "1" goto done -cmake --build . %par% +cmake --build . %config% %par% if errorlevel 1 goto error -cmake --build . --target install %par% +cmake --build . %config% --target install %par% if errorlevel 1 goto error if not "%do_test%" == "1" goto done set CTEST_OUTPUT_ON_FAILURE=1 if %msvc% == 1 ( - cmake --build . --target RUN_TESTS + cmake --build . %config% --target RUN_TESTS if errorlevel 1 goto error ) else ( - cmake --build . --target test + cmake --build . %config% --target test if errorlevel 1 goto error )